shithub: opus

Info  •  Files  •  Log  •  Branches

pitch embedding

--- a/dnn/lpcnet.py

+++ b/dnn/lpcnet.py

@@ -46,6 +46,7 @@

     exc = Input(shape=(None, 1))

     pitch = Input(shape=(None, 1))

     feat = Input(shape=(None, nb_used_features))

+    pitch = Input(shape=(None, 1))

     dec_feat = Input(shape=(None, 32))

     dec_state = Input(shape=(rnn_units,))

@@ -67,7 +68,10 @@

     embed2 = Embedding(256, embed_size, embeddings_initializer=PCMInit())

     cexc = Reshape((-1, embed_size))(embed2(exc))

-    cfeat = fconv2(fconv1(feat))

+    pembed = Embedding(256, 64)

+    cat_feat = Concatenate()([feat, Reshape((-1, 64))(pembed(pitch))])

+    

+    cfeat = fconv2(fconv1(cat_feat))

     rep = Lambda(lambda x: K.repeat_elements(x, 160, 1))

@@ -77,8 +81,8 @@

     gru_out, state = rnn(rnn_in)

     ulaw_prob = md(gru_out)

-    model = Model([pcm, exc, feat], ulaw_prob)

-    encoder = Model(feat, cfeat)

+    model = Model([pcm, exc, feat, pitch], ulaw_prob)

+    encoder = Model([feat, pitch], cfeat)

     dec_rnn_in = Concatenate()([cpcm, cexc, dec_feat])

     dec_gru_out, state = rnn(dec_rnn_in, initial_state=dec_state)

--- a/dnn/test_wavenet_audio.py

+++ b/dnn/test_wavenet_audio.py

@@ -60,7 +60,7 @@

 features = np.reshape(features, (nb_frames, feature_chunk_size, nb_features))

 features = features[:, :, :]

-

+periods = (50*features[:,:,36:37]+100).astype('int16')

 in_data = np.reshape(in_data, (nb_frames*pcm_chunk_size, 1))

 out_data = np.reshape(data, (nb_frames*pcm_chunk_size, 1))

--- a/dnn/train_wavenet_audio.py

+++ b/dnn/train_wavenet_audio.py

@@ -94,6 +94,7 @@

 features = features[:, :, :nb_used_features]

 pred = np.reshape(pred, (nb_frames, pcm_chunk_size, 1))

 pred = (pred.astype('int16')+128).astype('uint8')

+periods = (50*features[:,:,36:37]+100).astype('int16')

 in_data = np.concatenate([in_data, pred], axis=-1)

@@ -103,8 +104,8 @@

 # f.create_dataset('data', data=in_data[:50000, :, :])

 # f.create_dataset('feat', data=features[:50000, :, :])

-checkpoint = ModelCheckpoint('wavenet4d3_{epoch:02d}.h5')

+checkpoint = ModelCheckpoint('wavenet4e_{epoch:02d}.h5')

 #model.load_weights('wavernn1c_01.h5')

 model.compile(optimizer=Adam(0.001, amsgrad=True, decay=2e-4), loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])

-model.fit([in_data, in_exc, features], out_data, batch_size=batch_size, epochs=30, validation_split=0.2, callbacks=[checkpoint])

+model.fit([in_data, in_exc, features, periods], out_data, batch_size=batch_size, epochs=30, validation_split=0.2, callbacks=[checkpoint])

-- 

⑨