diff --git a/examples/mini_librispeech/cmd.sh b/examples/mini_librispeech/cmd.sh index 1dc96d4..d014014 100644 --- a/examples/mini_librispeech/cmd.sh +++ b/examples/mini_librispeech/cmd.sh @@ -10,12 +10,12 @@ # conf/queue.conf in http://kaldi-asr.org/doc/queue.html for more information, # or search for the string 'default_config' in utils/queue.pl or utils/slurm.pl. -#export train_cmd="run.pl --mem 4G" -#export cuda_cmd="run.pl --mem 4G --gpu 1" -#export decode_cmd="run.pl --mem 4G" +export train_cmd="run.pl --mem 4G" +export cuda_cmd="run.pl --mem 4G --gpu 1" +export decode_cmd="run.pl --mem 4G" # JHU setup -export train_cmd="queue.pl --mem 4G" +#export train_cmd="queue.pl --mem 4G" #export cuda_cmd="queue.pl --mem 4G --gpu 1 --config conf/gpu.conf" -export cuda_cmd="queue-freegpu.pl --mem 4G --gpu 1 --config conf/gpu.conf" -export decode_cmd="queue.pl --mem 4G" +#export cuda_cmd="queue-freegpu.pl --mem 4G --gpu 1 --config conf/gpu.conf" +#export decode_cmd="queue.pl --mem 4G" diff --git a/examples/mini_librispeech/prepare_graph.sh b/examples/mini_librispeech/prepare_graph.sh index 99ef598..d79b0b9 100755 --- a/examples/mini_librispeech/prepare_graph.sh +++ b/examples/mini_librispeech/prepare_graph.sh @@ -9,6 +9,7 @@ set -e -o pipefail stage=0 train_set=train_clean_5 valid_set=dev_clean_2 +combine_set=all rootdir=data langdir=data/lang @@ -22,13 +23,18 @@ nj=10 . ./cmd.sh . ./utils/parse_options.sh -lang=$langdir/lang_${type}${unit}_e2e +lang=${langdir}_${type}${unit}_e2e graph=$graphdir/${type}${unit} if [ $stage -le 0 ]; then echo "$0: Stage 0: Phone LM estimating" - rm -rf $lang - cp -r $langdir/lang_nosp $lang + + if [ -f $lang ];then + rm -rf $lang + fi + + utils/combine_data.sh $rootdir/$combine_set $rootdir/train_clean_5 $rootdir/dev_clean_2 + cp -r $langdir $lang silphonelist=$(cat $lang/phones/silence.csl) || exit 1; nonsilphonelist=$(cat $lang/phones/nonsilence.csl) || exit 1; # Use our special topology... note that later on may have to tune this @@ -38,10 +44,10 @@ if [ $stage -le 0 ]; then echo "Estimating a phone language model for the denominator graph..." mkdir -p $graph/log $train_cmd $graph/log/make_phone_lm.log \ - cat $rootdir/$train_set/text \| \ + cat $rootdir/$combine_set/text \| \ steps/nnet3/chain/e2e/text_to_phones.py --between-silprob 0.1 \ - $langdir/lang_nosp \| \ - utils/sym2int.pl -f 2- $langdir/lang_nosp/phones.txt \| \ + ${langdir} \| \ + utils/sym2int.pl -f 2- $langdir/phones.txt \| \ chain-est-phone-lm --num-extra-lm-states=2000 \ ark:- $graph/phone_lm.fst fi @@ -56,7 +62,7 @@ if [ $stage -le 1 ]; then prepare_e2e.sh --nj $nj --cmd "$train_cmd" \ --type $type_arg \ --shared-phones true \ - $rootdir/$train_set $lang $graph + $rootdir/$combine_set $lang $graph echo "Making denominator graph..." $train_cmd $graph/log/make_den_fst.log \ chain-make-den-fst $graph/tree $graph/0.trans_mdl \ @@ -87,8 +93,8 @@ fi if [ $stage -le 3 ]; then echo "Making HCLG full graph..." utils/lang/check_phones_compatible.sh \ - $langdir/lang_nosp_test_tgsmall/phones.txt $lang/phones.txt + ${langdir}_nosp_test_tgsmall/phones.txt $lang/phones.txt utils/mkgraph.sh \ - --self-loop-scale 1.0 $langdir/lang_nosp_test_tgsmall \ + --self-loop-scale 1.0 data/lang_nosp_test_tgsmall \ $graph $graph/graph_tgsmall || exit 1; fi diff --git a/examples/mini_librispeech/run.sh b/examples/mini_librispeech/run.sh index 660a315..77e1c88 100755 --- a/examples/mini_librispeech/run.sh +++ b/examples/mini_librispeech/run.sh @@ -32,7 +32,7 @@ unit=phone # phone/char type=mono # mono/bi affix= -stage=0 +stage=-1 . ./path.sh . ./utils/parse_options.sh @@ -89,9 +89,9 @@ if [ $stage -le 3 ]; then --valid_set $valid_set \ --rootdir $rootdir \ --graphdir $graphdir \ - --langdir $langdir \ + --langdir ${langdir}_nosp \ --type $type \ - --unit $unit + --unit $unit fi if [ ${stage} -le 4 ]; then @@ -134,7 +134,7 @@ if [ ${stage} -le 5 ]; then opts="" mkdir -p $dir/logs log_file=$dir/logs/train.log - python3 train.py + python3 $MAIN_ROOT/train.py \ --train data/train_${type}${unit}.json \ --valid data/valid_${type}${unit}.json \ --den-fst $graph/normalization.fst \ @@ -157,7 +157,7 @@ if [ ${stage} -le 6 ]; then log_file=$dir/logs/dump_$test_set.log result_file=$test_set/posteriors.ark mkdir -p $dir/$test_set - python3 test.py \ + python3 $MAIN_ROOT/test.py \ --test data/valid_${type}${unit}.json \ --model model_best.pth.tar \ --results $result_file \ @@ -179,8 +179,8 @@ fi if [ $stage -le 8 ]; then echo "Stage 8: Forthgram LM rescoring" - oldlang=$langdir/lang_nosp_test_tgsmall - newlang=$langdir/lang_nosp_test_tglarge + oldlang=${langdir}_nosp_test_tgsmall + newlang=${langdir}_nosp_test_tglarge oldlm=$oldlang/G.fst newlm=$newlang/G.carpa oldlmcommand="fstproject --project_output=true $oldlm |" diff --git a/examples/wsj/prepare_graph.sh b/examples/wsj/prepare_graph.sh index ead8b46..2536fa0 100755 --- a/examples/wsj/prepare_graph.sh +++ b/examples/wsj/prepare_graph.sh @@ -9,6 +9,7 @@ set -e -o pipefail stage=0 train_set=train_si284 valid_set=test_dev935 +combine_set=all rootdir=data langdir=data/lang @@ -28,6 +29,7 @@ graph=$graphdir/${type}${unit} if [ $stage -le 0 ]; then echo "$0: Stage 0: Phone LM estimating" rm -rf $lang + ./utils/combine_data.sh $rootdir/$combine_set $rootdir/$train_set $rootdir/$valid_set cp -r $langdir/$unit/lang_${unit} $lang silphonelist=$(cat $lang/phones/silence.csl) || exit 1; nonsilphonelist=$(cat $lang/phones/nonsilence.csl) || exit 1; @@ -38,7 +40,7 @@ if [ $stage -le 0 ]; then echo "Estimating a phone language model for the denominator graph..." mkdir -p $graph/log $train_cmd $graph/log/make_phone_lm.log \ - cat $rootdir/$train_set/text \| \ + cat $rootdir/$combine_set/text \| \ steps/nnet3/chain/e2e/text_to_phones.py --between-silprob 0.1 \ $langdir/$unit/lang_$unit \| \ utils/sym2int.pl -f 2- $langdir/$unit/lang_$unit/phones.txt \| \ @@ -56,7 +58,7 @@ if [ $stage -le 1 ]; then prepare_e2e.sh --nj $nj --cmd "$train_cmd" \ --type $type_arg \ --shared-phones true \ - $rootdir/$train_set $lang $graph + $rootdir/$combine_set $lang $graph echo "Making denominator graph..." $train_cmd $graph/log/make_den_fst.log \ chain-make-den-fst $graph/tree $graph/0.trans_mdl \ diff --git a/examples/wsj/run.sh b/examples/wsj/run.sh index 17afd62..64b3476 100755 --- a/examples/wsj/run.sh +++ b/examples/wsj/run.sh @@ -137,7 +137,7 @@ if [ ${stage} -le 5 ]; then opts="" mkdir -p $dir/logs log_file=$dir/logs/train.log - python3 train.py \ + python3 $MAIN_ROOT/train.py \ --train data/train_${type}${unit}.json \ --valid data/valid_${type}${unit}.json \ --den-fst $graph/normalization.fst \ @@ -160,7 +160,7 @@ if [ ${stage} -le 6 ]; then log_file=$dir/logs/dump_$test_set.log result_file=$test_set/posteriors.ark mkdir -p $dir/$test_set - python3 test.py \ + python3 $MAIN_ROOT/test.py \ --test data/test_${type}${unit}.json \ --model model_best.pth.tar \ --results $result_file \