ref: ea02ef7e024a98c6cc41bc790e44643f74f7f10e
parent: d75a4aec7218239aa41fb265f58dba5e58e40113
author: Jean-Marc Valin <jmvalin@jmvalin.ca>
date: Thu Dec 27 21:52:58 EST 2018
Computing signals in C
--- a/dnn/dump_data.c
+++ b/dnn/dump_data.c
@@ -58,6 +58,9 @@
float pitch_buf[PITCH_BUF_SIZE];
float last_gain;
int last_period;
+ float lpc[LPC_ORDER];
+ float sig_mem[LPC_ORDER];
+ int exc_mem;
} DenoiseState;
static int rnnoise_get_size() {@@ -108,7 +111,6 @@
int i;
float E = 0;
float Ly[NB_BANDS];
- float lpc[LPC_ORDER];
float p[WINDOW_SIZE];
float pitch_buf[PITCH_BUF_SIZE];
int pitch_index;
@@ -150,7 +152,7 @@
}
dct(features, Ly);
features[0] -= 4;
- g = lpc_from_cepstrum(lpc, features);
+ g = lpc_from_cepstrum(st->lpc, features);
#if 0
for (i=0;i<NB_BANDS;i++) printf("%f ", Ly[i]); printf("\n");@@ -158,7 +160,7 @@
features[2*NB_BANDS] = .01*(pitch_index-200);
features[2*NB_BANDS+1] = gain;
features[2*NB_BANDS+2] = log10(g);
- for (i=0;i<LPC_ORDER;i++) features[2*NB_BANDS+3+i] = lpc[i];
+ for (i=0;i<LPC_ORDER;i++) features[2*NB_BANDS+3+i] = st->lpc[i];
#if 0
for (i=0;i<NB_FEATURES;i++) printf("%f ", features[i]); printf("\n");@@ -198,6 +200,36 @@
b[1] = .75*uni_rand();
}
+void write_audio(DenoiseState *st, const short *pcm, float noise_std, FILE *file) {+ int i;
+ unsigned char data[4*FRAME_SIZE];
+ for (i=0;i<FRAME_SIZE;i++) {+ int noise;
+ float p=0;
+ float e;
+ int j;
+ for (j=0;j<LPC_ORDER;j++) p -= st->lpc[j]*st->sig_mem[j];
+ e = lin2ulaw(pcm[i] - p);
+ /* Signal. */
+ data[4*i] = lin2ulaw(st->sig_mem[0]);
+ /* Prediction. */
+ data[4*i+1] = lin2ulaw(p);
+ /* Excitation in. */
+ data[4*i+2] = st->exc_mem;
+ /* Excitation out. */
+ data[4*i+3] = e;
+ /* Simulate error on excitation. */
+ noise = (int)floor(.5 + noise_std*.707*(log((float)rand()/RAND_MAX)-log((float)rand()/RAND_MAX)));
+ e += noise;
+ e = IMIN(255, IMAX(0, e));
+
+ RNN_MOVE(&st->sig_mem[1], &st->sig_mem[0], LPC_ORDER-1);
+ st->sig_mem[0] = p + ulaw2lin(e);
+ st->exc_mem = e;
+ }
+ fwrite(data, 4*FRAME_SIZE, 1, file);
+}
+
int main(int argc, char **argv) {int i;
int count=0;
@@ -221,6 +253,7 @@
float old_speech_gain = 1;
int one_pass_completed = 0;
DenoiseState *st;
+ float noise_std=0;
int training = -1;
st = rnnoise_create();
if (argc == 5 && strcmp(argv[1], "-train")==0) training = 1;
@@ -287,6 +320,7 @@
if (rand()%100==0) speech_gain = 0;
gain_change_count = 0;
rand_resp(a_sig, b_sig);
+ noise_std = 3*(float)rand()/RAND_MAX;
}
biquad(x, mem_hp_x, x, b_hp, a_hp, FRAME_SIZE);
biquad(x, mem_resp_x, x, b_sig, a_sig, FRAME_SIZE);
@@ -302,7 +336,8 @@
fwrite(features, sizeof(float), NB_FEATURES, ffeat);
/* PCM is delayed by 1/2 frame to make the features centered on the frames. */
for (i=0;i<FRAME_SIZE-TRAINING_OFFSET;i++) pcm[i+TRAINING_OFFSET] = float2short(x[i]);
- if (fpcm) fwrite(pcm, sizeof(short), FRAME_SIZE, fpcm);
+ if (fpcm) write_audio(st, pcm, noise_std, fpcm);
+ //if (fpcm) fwrite(pcm, sizeof(short), FRAME_SIZE, fpcm);
for (i=0;i<TRAINING_OFFSET;i++) pcm[i] = float2short(x[i+FRAME_SIZE-TRAINING_OFFSET]);
old_speech_gain = speech_gain;
count++;
--- a/dnn/train_lpcnet.py
+++ b/dnn/train_lpcnet.py
@@ -66,85 +66,39 @@
# u for unquantised, load 16 bit PCM samples and convert to mu-law
-udata = np.fromfile(pcm_file, dtype='int16')
-data = lin2ulaw(udata)
-nb_frames = len(data)//pcm_chunk_size
+data = np.fromfile(pcm_file, dtype='uint8')
+nb_frames = len(data)//(4*pcm_chunk_size)
features = np.fromfile(feature_file, dtype='float32')
# limit to discrete number of frames
-data = data[:nb_frames*pcm_chunk_size]
-udata = udata[:nb_frames*pcm_chunk_size]
+data = data[:nb_frames*4*pcm_chunk_size]
features = features[:nb_frames*feature_chunk_size*nb_features]
-# Noise injection: the idea is that the real system is going to be
-# predicting samples based on previously predicted samples rather than
-# from the original. Since the previously predicted samples aren't
-# expected to be so good, I add noise to the training data. Exactly
-# how the noise is added makes a huge difference
+features = np.reshape(features, (nb_frames*feature_chunk_size, nb_features))
-in_data = np.concatenate([data[0:1], data[:-1]]);
-noise = np.concatenate([np.zeros((len(data)*1//5)), np.random.randint(-3, 3, len(data)*1//5), np.random.randint(-2, 2, len(data)*1//5), np.random.randint(-1, 1, len(data)*2//5)])
-#noise = np.round(np.concatenate([np.zeros((len(data)*1//5)), np.random.laplace(0, 1.2, len(data)*1//5), np.random.laplace(0, .77, len(data)*1//5), np.random.laplace(0, .33, len(data)*1//5), np.random.randint(-1, 1, len(data)*1//5)]))
+sig = np.reshape(data[0::4], (nb_frames, pcm_chunk_size, 1))
+pred = np.reshape(data[1::4], (nb_frames, pcm_chunk_size, 1))
+in_exc = np.reshape(data[2::4], (nb_frames, pcm_chunk_size, 1))
+out_exc = np.reshape(data[3::4], (nb_frames, pcm_chunk_size, 1))
del data
-in_data = in_data + noise
-del noise
-in_data = np.clip(in_data, 0, 255)
-features = np.reshape(features, (nb_frames*feature_chunk_size, nb_features))
+print("ulaw std = ", np.std(out_exc))-# Note: the LPC predictor output is now calculated by the loop below, this code was
-# for an ealier version that implemented the prediction filter in C
-
-upred = np.zeros((nb_frames*pcm_chunk_size,), dtype='float32')
-
-# Use 16th order LPC to generate LPC prediction output upred[] and (in
-# mu-law form) pred[]
-
-pred_in = ulaw2lin(in_data)
-for i in range(2, nb_frames*feature_chunk_size):
- upred[i*frame_size:(i+1)*frame_size] = 0
- for k in range(16):
- upred[i*frame_size:(i+1)*frame_size] = upred[i*frame_size:(i+1)*frame_size] - \
- pred_in[i*frame_size-k:(i+1)*frame_size-k]*features[i, nb_features-16+k]
-del pred_in
-
-pred = lin2ulaw(upred)
-
-in_data = np.reshape(in_data, (nb_frames, pcm_chunk_size, 1))
-in_data = in_data.astype('uint8')-
-# LPC residual, which is the difference between the input speech and
-# the predictor output, with a slight time shift this is also the
-# ideal excitation in_exc
-
-out_data = lin2ulaw(udata-upred)
-del upred
-del udata
-in_exc = np.concatenate([out_data[0:1], out_data[:-1]]);
-
-out_data = np.reshape(out_data, (nb_frames, pcm_chunk_size, 1))
-out_data = out_data.astype('uint8')-
-in_exc = np.reshape(in_exc, (nb_frames, pcm_chunk_size, 1))
-in_exc = in_exc.astype('uint8')-
-
features = np.reshape(features, (nb_frames, feature_chunk_size, nb_features))
features = features[:, :, :nb_used_features]
features[:,:,18:36] = 0
-pred = np.reshape(pred, (nb_frames, pcm_chunk_size, 1))
-pred = pred.astype('uint8') periods = (.1 + 50*features[:,:,36:37]+100).astype('int16')-in_data = np.concatenate([in_data, pred], axis=-1)
+in_data = np.concatenate([sig, pred], axis=-1)
+del sig
del pred
# dump models to disk as we go
-checkpoint = ModelCheckpoint('lpcnet15_384_10_G16_{epoch:02d}.h5')+checkpoint = ModelCheckpoint('lpcnet18_384_10_G16_{epoch:02d}.h5')-#model.load_weights('lpcnet9b_384_10_G16_01.h5')+model.load_weights('lpcnet9b_384_10_G16_01.h5')model.compile(optimizer=Adam(0.001, amsgrad=True, decay=5e-5), loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])
-model.fit([in_data, in_exc, features, periods], out_data, batch_size=batch_size, epochs=nb_epochs, validation_split=0.0, callbacks=[checkpoint, lpcnet.Sparsify(2000, 40000, 400, (0.1, 0.1, 0.1))])
+model.fit([in_data, in_exc, features, periods], out_exc, batch_size=batch_size, epochs=nb_epochs, validation_split=0.0, callbacks=[checkpoint, lpcnet.Sparsify(2000, 40000, 400, (0.1, 0.1, 0.1))])
--
⑨