FreeDV
/
LPCNet
mirror of https://github.com/drowe67/LPCNet.git
1
0
Fork 0
Code Issues Releases Wiki Activity

codec 2 pitch estimator integrated with dump_data and working OK. Updated process.sh script to plot before and after.

pull/8/head
David 2019-02-01 17:17:20 +10:30
parent 072d630041
commit 2ba56cc723
6 changed files with 250 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
process.sh
View File

@ -5,7 +5,7 @@
# 1. Process an input set of wave files using LPCNet under a variety of conditions.
# 2. Name output files to make them convenient to listen to in a file manager.
# 3. Generate a HTML table of samples for convenient replay on the web.
# 4. Generate a HTML table of distortion metrics.
# 4. Generate a bunch of other HTML files and PNGs.
# set these paths to suit your system
CODEC2_PATH=$HOME/codec2-dev/build_linux/src
@ -53,7 +53,7 @@ done
# Unquantised, baseline analysis-synthesis model, 10ms updates
for f in $WAV_FILES
do
sox $WAVIN_PATH/$f.wav -t raw - | ./dump_data -test - - | \
sox $WAVIN_PATH/$f.wav -t raw - | ./dump_data --test - - | \
./test_lpcnet - - | sox -r 16000 -t .s16 -c 1 - $WAVOUT_PATH/$f'_1_uq'.wav
done
@ -61,7 +61,7 @@ done
for f in $WAV_FILES
do
label=$(printf "3dB %-10s" "$f")
sox $WAVIN_PATH/$f.wav -t raw - | ./dump_data -test - - | \
sox $WAVIN_PATH/$f.wav -t raw - | ./dump_data --test - - | \
./quant_feat -l "$label" -d 1 --uniform 3 2>>$STATS | ./test_lpcnet - - | \
sox -r 16000 -t .s16 -c 1 - $WAVOUT_PATH/$f'_2_3dB'.wav
@ -70,7 +70,7 @@ done
# decimate features to 20ms updates, then lineary interpolate back up to 10ms updates
for f in $WAV_FILES
do
sox $WAVIN_PATH/$f.wav -t raw - | ./dump_data -test - - | \
sox $WAVIN_PATH/$f.wav -t raw - | ./dump_data --test - - | \
./quant_feat -d 2 | ./test_lpcnet - - | sox -r 16000 -t .s16 -c 1 - $WAVOUT_PATH/$f'_3_20ms'.wav
done
@ -79,7 +79,7 @@ done
for f in $WAV_FILES
do
label=$(printf "33bit_20ms %-10s" "$f")
sox $WAVIN_PATH/$f.wav -t raw - | ./dump_data -test - - | \
sox $WAVIN_PATH/$f.wav -t raw - | ./dump_data --test - - | \
./quant_feat -l "$label" -d 2 --mbest 5 -q pred2_stage1.f32,pred2_stage2.f32,pred2_stage3.f32 -s $SV_PATH/$f'_4_33bit_20ms'.txt 2>>$STATS | \
./test_lpcnet - - | sox -r 16000 -t .s16 -c 1 - $WAVOUT_PATH/$f'_4_33bit_20ms'.wav
done
@ -88,7 +88,7 @@ done
for f in $WAV_FILES
do
label=$(printf "33bit_30ms %-10s" "$f")
sox $WAVIN_PATH/$f.wav -t raw - | ./dump_data -test - - | \
sox $WAVIN_PATH/$f.wav -t raw - | ./dump_data --test - - | \
./quant_feat -l "$label" -d 3 --mbest 5 -q pred2_stage1.f32,pred2_stage2.f32,pred2_stage3.f32 -s $SV_PATH/$f'_5_33bit_30ms'.txt 2>>$STATS | \
./test_lpcnet - - | sox -r 16000 -t .s16 -c 1 - $WAVOUT_PATH/$f'_5_33bit_30ms'.wav
done
@ -97,7 +97,7 @@ done
for f in $WAV_FILES
do
label=$(printf "44bit_30ms %-10s" "$f")
sox $WAVIN_PATH/$f.wav -t raw - | ./dump_data -test - - | \
sox $WAVIN_PATH/$f.wav -t raw - | ./dump_data --test - - | \
./quant_feat -l "$label" -d 3 --mbest 5 -q pred2_stage1.f32,pred2_stage2.f32,pred2_stage3.f32,pred2_stage4.f32 -s $SV_PATH/$f'_6_44bit_30ms'.txt 2>>$STATS | \
./test_lpcnet - - | sox -r 16000 -t .s16 -c 1 - $WAVOUT_PATH/$f'_6_44bit_30ms'.wav
done
@ -148,13 +148,13 @@ function heading_row {
printf "</tr>\n" >> $HTML
}
heading_row
# for each wave file, create a row
printf "<table>\n" >> $HTML
printf "<caption>Samples</caption>\n" >> $HTML
heading_row
for f in $WAV_FILES
do
files=$(ls $WAVOUT_PATH/$f*);
@ -254,8 +254,9 @@ printf "<tr>\n" >> $HTML
for f in $WAV_FILES
do
sox $WAVIN_PATH/$f.wav -t raw $F32_PATH/$f.raw
./dump_data -test $F32_PATH/$f.raw $F32_PATH/$f.f32
octave --no-gui -p src -qf src/plot_speech_pitch.m $F32_PATH/$f.raw $F32_PATH/$f.f32 $PNG_PATH/$f'_pitch.png' ${mx[count]}
./dump_data --test $F32_PATH/$f.raw $F32_PATH/$f.f32
./dump_data --test --c2pitch $F32_PATH/$f.raw $F32_PATH/$f'_c2'.f32
octave --no-gui -p src -qf src/plot_speech_pitch.m $F32_PATH/$f.raw $F32_PATH/$f.f32 $F32_PATH/$f'_c2'.f32 $PNG_PATH/$f'_pitch.png' ${mx[count]}
count=$(( $count + 1 ))
b=$f'_pitch.png'
printf " <td align="center"><a href=\"png/%s\"><img width=100 height=100 src=\"png/%s\" /></a></td>\n" $b $b >> $HTML

124
src/codec2_kiss_fft.h 100644
View File

@ -0,0 +1,124 @@
#ifndef KISS_FFT_H
#define KISS_FFT_H
#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include <string.h>
#ifdef __cplusplus
extern "C" {
#endif
/*
ATTENTION!
If you would like a :
-- a utility that will handle the caching of fft objects
-- real-only (no imaginary time component ) FFT
-- a multi-dimensional FFT
-- a command-line utility to perform ffts
-- a command-line utility to perform fast-convolution filtering
Then see kfc.h kiss_fftr.h kiss_fftnd.h fftutil.c kiss_fastfir.c
in the tools/ directory.
*/
#ifdef USE_SIMD
# include <xmmintrin.h>
# define kiss_fft_scalar __m128
#define KISS_FFT_MALLOC(nbytes) _mm_malloc(nbytes,16)
#define KISS_FFT_FREE _mm_free
#else
#define KISS_FFT_MALLOC malloc
#define KISS_FFT_FREE free
#endif
#ifdef FIXED_POINT
#include <sys/types.h>
# if (FIXED_POINT == 32)
# define kiss_fft_scalar int32_t
# else
# define kiss_fft_scalar int16_t
# endif
#else
# ifndef kiss_fft_scalar
/* default is float */
# define kiss_fft_scalar float
# endif
#endif
typedef struct {
kiss_fft_scalar r;
kiss_fft_scalar i;
}kiss_fft_cpx;
typedef struct kiss_fft_state* kiss_fft_cfg;
/*
* kiss_fft_alloc
*
* Initialize a FFT (or IFFT) algorithm's cfg/state buffer.
*
* typical usage: kiss_fft_cfg mycfg=kiss_fft_alloc(1024,0,NULL,NULL);
*
* The return value from fft_alloc is a cfg buffer used internally
* by the fft routine or NULL.
*
* If lenmem is NULL, then kiss_fft_alloc will allocate a cfg buffer using malloc.
* The returned value should be free()d when done to avoid memory leaks.
*
* The state can be placed in a user supplied buffer 'mem':
* If lenmem is not NULL and mem is not NULL and *lenmem is large enough,
* then the function places the cfg in mem and the size used in *lenmem
* and returns mem.
*
* If lenmem is not NULL and ( mem is NULL or *lenmem is not large enough),
* then the function returns NULL and places the minimum cfg
* buffer size in *lenmem.
* */
kiss_fft_cfg kiss_fft_alloc(int nfft,int inverse_fft,void * mem,size_t * lenmem);
/*
* kiss_fft(cfg,in_out_buf)
*
* Perform an FFT on a complex input buffer.
* for a forward FFT,
* fin should be f[0] , f[1] , ... ,f[nfft-1]
* fout will be F[0] , F[1] , ... ,F[nfft-1]
* Note that each element is complex and can be accessed like
f[k].r and f[k].i
* */
void kiss_fft(kiss_fft_cfg cfg,const kiss_fft_cpx *fin,kiss_fft_cpx *fout);
/*
A more generic version of the above function. It reads its input from every Nth sample.
* */
void kiss_fft_stride(kiss_fft_cfg cfg,const kiss_fft_cpx *fin,kiss_fft_cpx *fout,int fin_stride);
/* If kiss_fft_alloc allocated a buffer, it is one contiguous
buffer and can be simply free()d when no longer needed*/
#define kiss_fft_free free
/*
Cleans up some memory that gets managed internally. Not necessary to call, but it might clean up
your compiler output to call this before you exit.
*/
void kiss_fft_cleanup(void);
/*
* Returns the smallest integer k, such that k>=n and k has only "fast" factors (2,3,5)
*/
int kiss_fft_next_fast_size(int n);
/* for real ffts, we need an even size */
#define kiss_fftr_next_fast_size_real(n) \
(kiss_fft_next_fast_size( ((n)+1)>>1)<<1)
#ifdef __cplusplus
}
#endif
#endif

19
src/codec2_pitch.h 100644
View File

@ -0,0 +1,19 @@
/*---------------------------------------------------------------------------*\
FILE........: codec2_pitch.h
AUTHOR......: David Rowe
DATE CREATED: 23/3/93 (!) Modified for LPCNet 2019
Codec 2 'NLP" Pitch estimator module for LPCNet.
\*---------------------------------------------------------------------------*/
#ifndef __CODEC2_PITCH__
#define __CODEC2_PITCH__
typedef struct CODEC2_PITCH_S CODEC2_PITCH;
CODEC2_PITCH *codec2_pitch_create(int *Sn_size, int *new_samples_each_call);
int codec2_pitch_est(CODEC2_PITCH *pitch, float Sn[], float *f0);
void codec2_pitch_destroy(CODEC2_PITCH *pitch);
#endif

45
src/dump_data.c
View File

@ -108,7 +108,7 @@ static void frame_analysis(DenoiseState *st, kiss_fft_cpx *X, float *Ex, const f
compute_band_energy(Ex, X);
}
static void compute_frame_features(DenoiseState *st, kiss_fft_cpx *X, kiss_fft_cpx *P, CODEC2_PITCH *c2pitch_st,
static void compute_frame_features(DenoiseState *st, kiss_fft_cpx *X, kiss_fft_cpx *P,
float *Ex, float *Ep, float *Exp, float *features, const float *in) {
int i;
float E = 0;
@ -124,13 +124,6 @@ static void compute_frame_features(DenoiseState *st, kiss_fft_cpx *X, kiss_fft_c
RNN_MOVE(st->pitch_buf, &st->pitch_buf[FRAME_SIZE], PITCH_BUF_SIZE-FRAME_SIZE);
RNN_COPY(&st->pitch_buf[PITCH_BUF_SIZE-FRAME_SIZE], in, FRAME_SIZE);
RNN_COPY(pitch_buf, &st->pitch_buf[0], PITCH_BUF_SIZE);
// run Codec 2 pitch est here before buf overwritten
int c2pitch_index;
float c2_f0;
if (c2pitch_st != NULL)
c2pitch_index = codec2_pitch_est(c2pitch_st, pitch_buf, &c2_f0);
pitch_downsample(pitch_buf, PITCH_BUF_SIZE);
pitch_search(pitch_buf+PITCH_MAX_PERIOD, pitch_buf, PITCH_FRAME_SIZE<<1,
(PITCH_MAX_PERIOD-3*PITCH_MIN_PERIOD)<<1, &pitch_index);
@ -167,8 +160,6 @@ static void compute_frame_features(DenoiseState *st, kiss_fft_cpx *X, kiss_fft_c
printf("\n");
#endif
fprintf(stderr, "%f %d %d\n", c2_f0, c2pitch_index, pitch_index);
features[2*NB_BANDS] = .01*(pitch_index-200);
features[2*NB_BANDS+1] = gain;
features[2*NB_BANDS+2] = log10(g);
@ -267,7 +258,7 @@ int main(int argc, char **argv) {
DenoiseState *st;
float noise_std=0;
int training = -1;
int c2pitch = 0;
int c2pitch_en = 0;
st = rnnoise_create();
@ -292,7 +283,7 @@ int main(int argc, char **argv) {
training = 0;
break;
case 'c':
c2pitch = 1;
c2pitch_en = 1;
break;
case 'h':
case '?':
@ -343,9 +334,16 @@ int main(int argc, char **argv) {
}
}
CODEC2_PITCH *c2pitch_st = NULL;
if (c2pitch)
c2pitch_st = codec2_pitch_create();
/* optionally fire up Codec 2 pitch estimator */
CODEC2_PITCH *c2pitch = NULL;
int c2_Sn_size, c2_frame_size;
float *c2_Sn = NULL;
if (c2pitch_en) {
c2pitch = codec2_pitch_create(&c2_Sn_size, &c2_frame_size);
assert(FRAME_SIZE == c2_frame_size);
c2_Sn = (float*)malloc(sizeof(float)*c2_Sn_size); assert(c2_Sn != NULL);
for(i=0; i<c2_Sn_size; i++) c2_Sn[i] = 0.0;
}
while (1) {
kiss_fft_cpx X[FREQ_SIZE], P[WINDOW_SIZE];
@ -401,7 +399,20 @@ int main(int argc, char **argv) {
x[i] *= g;
}
for (i=0;i<FRAME_SIZE;i++) x[i] += rand()/(float)RAND_MAX - .5;
compute_frame_features(st, X, P, c2pitch_st, Ex, Ep, Exp, features, x);
compute_frame_features(st, X, P, Ex, Ep, Exp, features, x);
if (c2pitch_en) {
for(i=0; i<c2_Sn_size-c2_frame_size; i++)
c2_Sn[i] = c2_Sn[i+c2_frame_size];
for(i=0; i<c2_frame_size; i++)
c2_Sn[i+c2_Sn_size-c2_frame_size] = x[i];
float f0; int pitch_index;
pitch_index = codec2_pitch_est(c2pitch, c2_Sn, &f0);
features[2*NB_BANDS] = 0.01*(pitch_index-200);
//int pitch_index_lpcnet = 100*features[2*NB_BANDS] + 200;
//fprintf(stderr, "%f %d %d\n", f0, pitch_index, pitch_index_lpcnet);
}
fwrite(features, sizeof(float), NB_FEATURES, ffeat);
/* PCM is delayed by 1/2 frame to make the features centered on the frames. */
for (i=0;i<FRAME_SIZE-TRAINING_OFFSET;i++) pcm[i+TRAINING_OFFSET] = float2short(x[i]);
@ -414,7 +425,7 @@ int main(int argc, char **argv) {
fclose(f1);
fclose(ffeat);
if (fpcm) fclose(fpcm);
if (c2pitch) codec2_pitch_destroy(c2pitch_st);
if (c2pitch_en) { free(c2_Sn); codec2_pitch_destroy(c2pitch); }
rnnoise_destroy(st);
return 0;
}

56
src/plot_pitch_test.m 100644
View File

@ -0,0 +1,56 @@
% plot_pitch_test_ext.m
% David Rowe Feb 2019
%
% Octave script to test Codec 2 pitch track from tnlp and est integrated into dump_data
Fs = 16000;
Fsp = 100;
graphics_toolkit ("gnuplot")
fn_raw = "~/Desktop/deep/quant/birch.wav";
tnlp = load("../birch_f0_pp.txt");
tcodec2_pitch = load("../birch_f0_pp2.txt");
dump_data_pitch = load("../birch_dd_pp.txt");
lpcnet_f0 = 2*Fs./dump_data_pitch(:,3);
fs=fopen(fn_raw,"rb");
s = fread(fs,Inf,"short");
fclose(fs);
figure(1); clf;
st_sec=0; en_sec=en_sec=length(s)/Fs;
st = Fs*st_sec; en = Fs*en_sec;
t = st_sec:1/Fs:en_sec-1/Fs;
subplot(211,"position",[0.1 0.8 0.8 0.15]);
plot(t,s(st+1:en));
s_max = max(abs(s(st+1:en)));
axis([st_sec en_sec -s_max s_max])
subplot(212,"position",[0.1 0.05 0.8 0.7]);
st = Fsp*st_sec; en = Fsp*en_sec;
t = st_sec:1/Fsp:en_sec-1/Fsp;
plot(t,tnlp(st+1:en,1),'b;F0 tnlp;');
hold on;
%plot(t,tcodec2_pitch(st+1:en,1),'g;F0 tcodec2_pitch;');
plot(t,dump_data_pitch(st+1:en,1),'r;F0 dump_data_pitch;');
plot(t,lpcnet_f0,'m;F0 lpcnet_pitch;');
hold off;
axis([st_sec en_sec 0 400])
figure(2); clf;
subplot(211,"position",[0.1 0.8 0.8 0.15]);
st = Fs*st_sec; en = Fs*en_sec;
t = st_sec:1/Fs:en_sec-1/Fs;
plot(t,s(st+1:en));
axis([st_sec en_sec -s_max s_max])
subplot(212,"position",[0.1 0.05 0.8 0.7]);
st = Fsp*st_sec; en = Fsp*en_sec;
t = st_sec:1/Fsp:en_sec-1/Fsp;
plot(t,dump_data_pitch(st+1:en,2),'b;pitch codec 2;');
hold on;
plot(t,dump_data_pitch(st+1:en,3),'r;pitch lpcnet;');
hold off;

14
src/plot_speech_pitch.m
View File

@ -15,17 +15,22 @@ graphics_toolkit ("gnuplot")
arg_list = argv ();
if nargin == 0
printf("\nusage: %s rawSpeechFile featureFile PNGname f0AxisMax\n\n", program_name());
printf("\nusage: %s rawSpeechFile featureFile featureFile_c2 PNGname f0AxisMax\n\n", program_name());
exit(0);
end
fn_raw = arg_list{1};
fn_feat = arg_list{2};
fn_feat_c2 = arg_list{3};
feat=load_f32(fn_feat, nb_lpcnet_features);
pitch_index_lpcnet = 100*feat(:,2*nb_lpcnet_bands+1) + 200;
f0 = 2*Fs ./ pitch_index_lpcnet;
feat_c2=load_f32(fn_feat_c2, nb_lpcnet_features);
pitch_index_c2 = 100*feat_c2(:,2*nb_lpcnet_bands+1) + 200;
f0_c2 = 2*Fs ./ pitch_index_c2;
fs=fopen(fn_raw,"rb");
s = fread(fs,Inf,"short");
fclose(fs);
@ -44,12 +49,15 @@ subplot(212,"position",[0.1 0.05 0.8 0.7]);
st = Fsp*st_sec; en = Fsp*en_sec;
t = st_sec:1/Fsp:en_sec-1/Fsp;
plot(t,f0(st+1:en),'b;F0 Hz;');
hold on;
plot(t,f0_c2(st+1:en),'g;F0 Hz CODEC 2;');
hold off;
ylabel('F0 Hz')
% adjust scale to make plot clearer
mx = str2num(arg_list{4})
mx = str2num(arg_list{5})
axis([st_sec en_sec 50 mx])
str=sprintf("-S%d,700",floor(1200*length(s)/(2*Fs)))
print(arg_list{3},'-dpng',str)
print(arg_list{4},'-dpng',str)