ref: 70789e6f4361267c478556a454e7660a241a737a
parent: 4cf2b2705a461d0eff56e82a5a74f00ef912538c
author: Jean-Marc Valin <jmvalin@jmvalin.ca>
date: Tue Jul 31 14:37:27 EDT 2018
audio-domain synthesis
--- a/dnn/lpcnet.py
+++ b/dnn/lpcnet.py
@@ -4,6 +4,7 @@
from keras.models import Model
from keras.layers import Input, LSTM, CuDNNGRU, Dense, Embedding, Reshape, Concatenate, Lambda, Conv1D, Multiply, Bidirectional, MaxPooling1D, Activation
from keras import backend as K
+from keras.initializers import Initializer
from mdense import MDense
import numpy as np
import h5py
@@ -14,7 +15,31 @@
pcm_levels = 2**pcm_bits
nb_used_features = 38
+class PCMInit(Initializer):
+ def __init__(self, gain=.1, seed=None):
+ self.gain = gain
+ self.seed = seed
+ def __call__(self, shape, dtype=None):
+ num_rows = 1
+ for dim in shape[:-1]:
+ num_rows *= dim
+ num_cols = shape[-1]
+ flat_shape = (num_rows, num_cols)
+ if self.seed is not None:
+ np.random.seed(self.seed)
+ a = np.random.uniform(-1.7321, 1.7321, flat_shape)
+ #a[:,0] = math.sqrt(12)*np.arange(-.5*num_rows+.5,.5*num_rows-.4)/num_rows
+ #a[:,1] = .5*a[:,0]*a[:,0]*a[:,0]
+ a = a + np.reshape(math.sqrt(12)*np.arange(-.5*num_rows+.5,.5*num_rows-.4)/num_rows, (num_rows, 1))
+ return self.gain * a
+
+ def get_config(self):
+ return {+ 'gain': self.gain,
+ 'seed': self.seed
+ }
+
def new_wavernn_model():
pcm = Input(shape=(None, 1))
pitch = Input(shape=(None, 1))
@@ -34,6 +59,10 @@
else:
cpcm = pcm
cpitch = pitch
+
+ embed = Embedding(256, 128, embeddings_initializer=PCMInit())
+ cpcm = Reshape((-1, 128))(embed(pcm))
+
cfeat = fconv2(fconv1(feat))
--- /dev/null
+++ b/dnn/test_wavenet_audio.py
@@ -1,0 +1,103 @@
+#!/usr/bin/python3
+
+import wavenet
+import lpcnet
+import sys
+import numpy as np
+from keras.optimizers import Adam
+from keras.callbacks import ModelCheckpoint
+from ulaw import ulaw2lin, lin2ulaw
+import keras.backend as K
+import h5py
+
+#import tensorflow as tf
+#from keras.backend.tensorflow_backend import set_session
+#config = tf.ConfigProto()
+#config.gpu_options.per_process_gpu_memory_fraction = 0.44
+#set_session(tf.Session(config=config))
+
+nb_epochs = 40
+batch_size = 64
+
+#model = wavenet.new_wavenet_model(fftnet=True)
+model, enc, dec = lpcnet.new_wavernn_model()
+
+model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])
+model.summary()
+
+pcmfile = sys.argv[1]
+feature_file = sys.argv[2]
+frame_size = 160
+nb_features = 54
+nb_used_features = wavenet.nb_used_features
+feature_chunk_size = 15
+pcm_chunk_size = frame_size*feature_chunk_size
+
+data = np.fromfile(pcmfile, dtype='int16')
+data = np.minimum(127, lin2ulaw(data[80:]/32768.))
+nb_frames = len(data)//pcm_chunk_size
+
+features = np.fromfile(feature_file, dtype='float32')
+
+data = data[:nb_frames*pcm_chunk_size]
+features = features[:nb_frames*feature_chunk_size*nb_features]
+
+in_data = np.concatenate([data[0:1], data[:-1]]);
+
+features = np.reshape(features, (nb_frames*feature_chunk_size, nb_features))
+pitch = 1.*data
+pitch[:320] = 0
+for i in range(2, nb_frames*feature_chunk_size):
+ period = int(50*features[i,36]+100)
+ period = period - 4
+ pitch[i*frame_size:(i+1)*frame_size] = data[i*frame_size-period:(i+1)*frame_size-period]
+in_pitch = np.reshape(pitch/16., (nb_frames, pcm_chunk_size, 1))
+
+in_data = np.reshape(in_data, (nb_frames, pcm_chunk_size, 1))
+in_data = (in_data.astype('int16')+128).astype('uint8')+out_data = np.reshape(data, (nb_frames, pcm_chunk_size, 1))
+out_data = (out_data.astype('int16')+128).astype('uint8')+features = np.reshape(features, (nb_frames, feature_chunk_size, nb_features))
+features = features[:, :, :nb_used_features]
+
+
+
+in_data = np.reshape(in_data, (nb_frames*pcm_chunk_size, 1))
+out_data = np.reshape(data, (nb_frames*pcm_chunk_size, 1))
+
+
+model.load_weights('wavenet3e_30.h5')+
+order = 16
+
+pcm = 0.*out_data
+exc = out_data-0
+pitch = np.zeros((1, 1, 1), dtype='float32')
+fexc = np.zeros((1, 1, 1), dtype='float32')
+iexc = np.zeros((1, 1, 1), dtype='int16')
+state = np.zeros((1, lpcnet.rnn_units), dtype='float32')
+for c in range(1, nb_frames):
+ cfeat = enc.predict(features[c:c+1, :, :nb_used_features])
+ for fr in range(1, feature_chunk_size):
+ f = c*feature_chunk_size + fr
+ a = features[c, fr, nb_used_features:]
+
+ #print(a)
+ gain = 1.;
+ period = int(50*features[c, fr, 36]+100)
+ period = period - 4
+ for i in range(frame_size):
+ pitch[0, 0, 0] = exc[f*frame_size + i - period, 0]
+ fexc[0, 0, 0] = iexc + 128
+ #fexc[0, 0, 0] = in_data[f*frame_size + i, 0]
+ #print(cfeat.shape)
+ p, state = dec.predict([fexc, cfeat[:, fr:fr+1, :], state])
+ p = p/(1e-5 + np.sum(p))
+ #print(np.sum(p))
+ iexc[0, 0, 0] = np.argmax(np.random.multinomial(1, p[0,0,:], 1))-128
+ exc[f*frame_size + i] = iexc[0, 0, 0]/16.
+ #out_data[f*frame_size + i, 0] = iexc[0, 0, 0]
+ pcm[f*frame_size + i, 0] = 32768*ulaw2lin(iexc[0, 0, 0]*1.0)
+ print(iexc[0, 0, 0], out_data[f*frame_size + i, 0], pcm[f*frame_size + i, 0])
+
+
--- a/dnn/train_wavenet_audio.py
+++ b/dnn/train_wavenet_audio.py
@@ -1,6 +1,7 @@
#!/usr/bin/python3
import wavenet
+import lpcnet
import sys
import numpy as np
from keras.optimizers import Adam
@@ -18,7 +19,9 @@
nb_epochs = 40
batch_size = 64
-model = wavenet.new_wavenet_model(fftnet=True)
+#model = wavenet.new_wavenet_model(fftnet=True)
+model, _, _ = lpcnet.new_wavernn_model()
+
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])
model.summary()
@@ -64,7 +67,7 @@
# f.create_dataset('data', data=in_data[:50000, :, :]) # f.create_dataset('feat', data=features[:50000, :, :])-checkpoint = ModelCheckpoint('wavenet3c_{epoch:02d}.h5')+checkpoint = ModelCheckpoint('wavenet3e_{epoch:02d}.h5') #model.load_weights('wavernn1c_01.h5')model.compile(optimizer=Adam(0.001, amsgrad=True, decay=2e-4), loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])
--- a/dnn/ulaw.py
+++ b/dnn/ulaw.py
@@ -5,7 +5,7 @@
def ulaw2lin(u):
s = np.sign(u)
u = np.abs(u)
- return s*(np.exp(u/128*math.log(256))-1)/255
+ return s*(np.exp(u/128.*math.log(256))-1)/255
def lin2ulaw(x):
--
⑨