Skip to content

bug fix: valid set graph creating for training model #9

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
indra622 wants to merge 2 commits into
base: master
Choose a base branch
from
Open

bug fix: valid set graph creating for training model #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
02fe3f7
bug fix: valid set graph creating for training model
indra622 Jan 8, 2021
a92efb1
bug fix for wsj
indra622 Jan 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions examples/mini_librispeech/cmd.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,12 @@
# conf/queue.conf in http://kaldi-asr.org/doc/queue.html for more information,
# or search for the string 'default_config' in utils/queue.pl or utils/slurm.pl.

#export train_cmd="run.pl --mem 4G"
#export cuda_cmd="run.pl --mem 4G --gpu 1"
#export decode_cmd="run.pl --mem 4G"
export train_cmd="run.pl --mem 4G"
export cuda_cmd="run.pl --mem 4G --gpu 1"
export decode_cmd="run.pl --mem 4G"

# JHU setup
export train_cmd="queue.pl --mem 4G"
#export train_cmd="queue.pl --mem 4G"
#export cuda_cmd="queue.pl --mem 4G --gpu 1 --config conf/gpu.conf"
export cuda_cmd="queue-freegpu.pl --mem 4G --gpu 1 --config conf/gpu.conf"
export decode_cmd="queue.pl --mem 4G"
#export cuda_cmd="queue-freegpu.pl --mem 4G --gpu 1 --config conf/gpu.conf"
#export decode_cmd="queue.pl --mem 4G"
24 changes: 15 additions & 9 deletions examples/mini_librispeech/prepare_graph.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ set -e -o pipefail
stage=0
train_set=train_clean_5
valid_set=dev_clean_2
combine_set=all
rootdir=data

langdir=data/lang
Expand All @@ -22,13 +23,18 @@ nj=10
. ./cmd.sh
. ./utils/parse_options.sh

lang=$langdir/lang_${type}${unit}_e2e
lang=${langdir}_${type}${unit}_e2e
graph=$graphdir/${type}${unit}

if [ $stage -le 0 ]; then
echo "$0: Stage 0: Phone LM estimating"
rm -rf $lang
cp -r $langdir/lang_nosp $lang

if [ -f $lang ];then
rm -rf $lang
fi

utils/combine_data.sh $rootdir/$combine_set $rootdir/train_clean_5 $rootdir/dev_clean_2
cp -r $langdir $lang
silphonelist=$(cat $lang/phones/silence.csl) || exit 1;
nonsilphonelist=$(cat $lang/phones/nonsilence.csl) || exit 1;
# Use our special topology... note that later on may have to tune this
Expand All @@ -38,10 +44,10 @@ if [ $stage -le 0 ]; then
echo "Estimating a phone language model for the denominator graph..."
mkdir -p $graph/log
$train_cmd $graph/log/make_phone_lm.log \
cat $rootdir/$train_set/text \| \
cat $rootdir/$combine_set/text \| \
steps/nnet3/chain/e2e/text_to_phones.py --between-silprob 0.1 \
$langdir/lang_nosp \| \
utils/sym2int.pl -f 2- $langdir/lang_nosp/phones.txt \| \
${langdir} \| \
utils/sym2int.pl -f 2- $langdir/phones.txt \| \
chain-est-phone-lm --num-extra-lm-states=2000 \
ark:- $graph/phone_lm.fst
fi
Expand All @@ -56,7 +62,7 @@ if [ $stage -le 1 ]; then
prepare_e2e.sh --nj $nj --cmd "$train_cmd" \
--type $type_arg \
--shared-phones true \
$rootdir/$train_set $lang $graph
$rootdir/$combine_set $lang $graph
echo "Making denominator graph..."
$train_cmd $graph/log/make_den_fst.log \
chain-make-den-fst $graph/tree $graph/0.trans_mdl \
Expand Down Expand Up @@ -87,8 +93,8 @@ fi
if [ $stage -le 3 ]; then
echo "Making HCLG full graph..."
utils/lang/check_phones_compatible.sh \
$langdir/lang_nosp_test_tgsmall/phones.txt $lang/phones.txt
${langdir}_nosp_test_tgsmall/phones.txt $lang/phones.txt
utils/mkgraph.sh \
--self-loop-scale 1.0 $langdir/lang_nosp_test_tgsmall \
--self-loop-scale 1.0 data/lang_nosp_test_tgsmall \
$graph $graph/graph_tgsmall || exit 1;
fi
14 changes: 7 additions & 7 deletions examples/mini_librispeech/run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ unit=phone # phone/char
type=mono # mono/bi

affix=
stage=0
stage=-1
. ./path.sh
. ./utils/parse_options.sh

Expand Down Expand Up @@ -89,9 +89,9 @@ if [ $stage -le 3 ]; then
--valid_set $valid_set \
--rootdir $rootdir \
--graphdir $graphdir \
--langdir $langdir \
--langdir ${langdir}_nosp \
--type $type \
--unit $unit
--unit $unit
fi

if [ ${stage} -le 4 ]; then
Expand Down Expand Up @@ -134,7 +134,7 @@ if [ ${stage} -le 5 ]; then
opts=""
mkdir -p $dir/logs
log_file=$dir/logs/train.log
python3 train.py
python3 $MAIN_ROOT/train.py \
--train data/train_${type}${unit}.json \
--valid data/valid_${type}${unit}.json \
--den-fst $graph/normalization.fst \
Expand All @@ -157,7 +157,7 @@ if [ ${stage} -le 6 ]; then
log_file=$dir/logs/dump_$test_set.log
result_file=$test_set/posteriors.ark
mkdir -p $dir/$test_set
python3 test.py \
python3 $MAIN_ROOT/test.py \
--test data/valid_${type}${unit}.json \
--model model_best.pth.tar \
--results $result_file \
Expand All @@ -179,8 +179,8 @@ fi

if [ $stage -le 8 ]; then
echo "Stage 8: Forthgram LM rescoring"
oldlang=$langdir/lang_nosp_test_tgsmall
newlang=$langdir/lang_nosp_test_tglarge
oldlang=${langdir}_nosp_test_tgsmall
newlang=${langdir}_nosp_test_tglarge
oldlm=$oldlang/G.fst
newlm=$newlang/G.carpa
oldlmcommand="fstproject --project_output=true $oldlm |"
Expand Down
6 changes: 4 additions & 2 deletions examples/wsj/prepare_graph.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ set -e -o pipefail
stage=0
train_set=train_si284
valid_set=test_dev935
combine_set=all
rootdir=data

langdir=data/lang
Expand All @@ -28,6 +29,7 @@ graph=$graphdir/${type}${unit}
if [ $stage -le 0 ]; then
echo "$0: Stage 0: Phone LM estimating"
rm -rf $lang
./utils/combine_data.sh $rootdir/$combine_set $rootdir/$train_set $rootdir/$valid_set
cp -r $langdir/$unit/lang_${unit} $lang
silphonelist=$(cat $lang/phones/silence.csl) || exit 1;
nonsilphonelist=$(cat $lang/phones/nonsilence.csl) || exit 1;
Expand All @@ -38,7 +40,7 @@ if [ $stage -le 0 ]; then
echo "Estimating a phone language model for the denominator graph..."
mkdir -p $graph/log
$train_cmd $graph/log/make_phone_lm.log \
cat $rootdir/$train_set/text \| \
cat $rootdir/$combine_set/text \| \
steps/nnet3/chain/e2e/text_to_phones.py --between-silprob 0.1 \
$langdir/$unit/lang_$unit \| \
utils/sym2int.pl -f 2- $langdir/$unit/lang_$unit/phones.txt \| \
Expand All @@ -56,7 +58,7 @@ if [ $stage -le 1 ]; then
prepare_e2e.sh --nj $nj --cmd "$train_cmd" \
--type $type_arg \
--shared-phones true \
$rootdir/$train_set $lang $graph
$rootdir/$combine_set $lang $graph
echo "Making denominator graph..."
$train_cmd $graph/log/make_den_fst.log \
chain-make-den-fst $graph/tree $graph/0.trans_mdl \
Expand Down
4 changes: 2 additions & 2 deletions examples/wsj/run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ if [ ${stage} -le 5 ]; then
opts=""
mkdir -p $dir/logs
log_file=$dir/logs/train.log
python3 train.py \
python3 $MAIN_ROOT/train.py \
--train data/train_${type}${unit}.json \
--valid data/valid_${type}${unit}.json \
--den-fst $graph/normalization.fst \
Expand All @@ -160,7 +160,7 @@ if [ ${stage} -le 6 ]; then
log_file=$dir/logs/dump_$test_set.log
result_file=$test_set/posteriors.ark
mkdir -p $dir/$test_set
python3 test.py \
python3 $MAIN_ROOT/test.py \
--test data/test_${type}${unit}.json \
--model model_best.pth.tar \
--results $result_file \
Expand Down