Semi-supervised training using chain models

egs/ami/s5b/local/nnet3/multi_condition/run_ivector_common.sh

@@ -10,30 +10,25 @@ set -e -o pipefail
 stage=1
 mic=ihm
 nj=30
-min_seg_len=1.55  # min length in seconds... we do this because chain training
-                  # will discard segments shorter than 1.5 seconds.  Must remain in sync with
-                  # the same option given to prepare_lores_feats.sh.
 train_set=train_cleaned   # you might set this to e.g. train_cleaned.
-gmm=tri3_cleaned  # This specifies a GMM-dir from the features of the type you're training the system on;
-                  # it should contain alignments for 'train_set'.
-
+norvb_datadir=data/ihm/train_cleaned_sp
 
 num_threads_ubm=32
 rvb_affix=_rvb
 nnet3_affix=_cleaned     # affix for exp/$mic/nnet3 directory to put iVector stuff in, so it
                          # becomes exp/$mic/nnet3_cleaned or whatever.
 num_data_reps=1
+sample_rate=16000
+
+max_jobs_run=10
 
 . ./cmd.sh
 . ./path.sh
 . ./utils/parse_options.sh
 
 nnet3_affix=${nnet3_affix}$rvb_affix
 
-gmmdir=exp/${mic}/${gmm}
-
-
-for f in data/${mic}/${train_set}/feats.scp ${gmmdir}/final.mdl; do
+for f in data/${mic}/${train_set}/feats.scp; do
   if [ ! -f $f ]; then
     echo "$0: expected file $f to exist"
     exit 1
@@ -73,35 +68,22 @@ if [ $stage -le 1 ]; then
 
   for datadir in ${train_set}_sp dev eval; do
     steps/make_mfcc.sh --nj $nj --mfcc-config conf/mfcc_hires.conf \
-      --cmd "$train_cmd" data/$mic/${datadir}_hires
+      --cmd "$train_cmd --max-jobs-run $max_jobs_run" data/$mic/${datadir}_hires
     steps/compute_cmvn_stats.sh data/$mic/${datadir}_hires
     utils/fix_data_dir.sh data/$mic/${datadir}_hires
   done
 fi
 
-if [ $stage -le 2 ]; then
-  echo "$0: combining short segments of speed-perturbed high-resolution MFCC training data"
-  # we have to combine short segments or we won't be able to train chain models
-  # on those segments.
-  utils/data/combine_short_segments.sh \
-     data/${mic}/${train_set}_sp_hires $min_seg_len data/${mic}/${train_set}_sp_hires_comb
-
-  # just copy over the CMVN to avoid having to recompute it.
-  cp data/${mic}/${train_set}_sp_hires/cmvn.scp data/${mic}/${train_set}_sp_hires_comb/
-  utils/fix_data_dir.sh data/${mic}/${train_set}_sp_hires_comb/
-fi
 
 if [ $stage -le 3 ]; then
   echo "$0: creating reverberated MFCC features"
 
-  datadir=data/ihm/train_cleaned_sp
-
-  mfccdir=${datadir}_rvb${num_data_reps}_hires/data
+  mfccdir=${norvb_datadir}_rvb${num_data_reps}_hires/data
   if [[ $(hostname -f) == *.clsp.jhu.edu ]] && [ ! -d $mfccdir/storage ]; then
     utils/create_split_dir.pl /export/b0{5,6,7,8}/$USER/kaldi-data/egs/ami-$mic-$(date +'%m_%d_%H_%M')/s5/$mfccdir/storage $mfccdir/storage
   fi
 
-  if [ ! -f ${datadir}_rvb${num_data_reps}_hires/feats.scp ]; then
+  if [ ! -f ${norvb_datadir}_rvb${num_data_reps}_hires/feats.scp ]; then
     if [ ! -d "RIRS_NOISES" ]; then
       # Download the package that includes the real RIRs, simulated RIRs, isotropic noises and point-source noises
       wget --no-check-certificate http://www.openslr.org/resources/28/rirs_noises.zip
@@ -123,60 +105,27 @@ if [ $stage -le 3 ]; then
       --isotropic-noise-addition-probability 1 \
       --num-replications ${num_data_reps} \
       --max-noises-per-minute 1 \
-      --source-sampling-rate 16000 \
-      ${datadir} ${datadir}_rvb${num_data_reps}
+      --source-sampling-rate $sample_rate \
+      ${norvb_datadir} ${norvb_datadir}_rvb${num_data_reps}
 
-    utils/copy_data_dir.sh ${datadir}_rvb${num_data_reps} ${datadir}_rvb${num_data_reps}_hires
-    utils/data/perturb_data_dir_volume.sh ${datadir}_rvb${num_data_reps}_hires
+    utils/copy_data_dir.sh ${norvb_datadir}_rvb${num_data_reps} ${norvb_datadir}_rvb${num_data_reps}_hires
+    utils/data/perturb_data_dir_volume.sh ${norvb_datadir}_rvb${num_data_reps}_hires
 
     steps/make_mfcc.sh --nj $nj --mfcc-config conf/mfcc_hires.conf \
-      --cmd "$train_cmd" ${datadir}_rvb${num_data_reps}_hires
-    steps/compute_cmvn_stats.sh ${datadir}_rvb${num_data_reps}_hires
-    utils/fix_data_dir.sh ${datadir}_rvb${num_data_reps}_hires  
-
-    utils/data/combine_short_segments.sh \
-      ${datadir}_rvb${num_data_reps}_hires $min_seg_len ${datadir}_rvb${num_data_reps}_hires_comb
-
-    # just copy over the CMVN to avoid having to recompute it.
-    cp ${datadir}_rvb${num_data_reps}_hires/cmvn.scp ${datadir}_rvb${num_data_reps}_hires_comb/
-    utils/fix_data_dir.sh ${datadir}_rvb${num_data_reps}_hires_comb/
+      --cmd "$train_cmd --max-jobs-run $max_jobs_run" ${norvb_datadir}_rvb${num_data_reps}_hires
+    steps/compute_cmvn_stats.sh ${norvb_datadir}_rvb${num_data_reps}_hires
+    utils/fix_data_dir.sh ${norvb_datadir}_rvb${num_data_reps}_hires
   fi
 
-  utils/combine_data.sh data/${mic}/${train_set}_sp_rvb_hires data/${mic}/${train_set}_sp_hires ${datadir}_rvb${num_data_reps}_hires
-  utils/combine_data.sh data/${mic}/${train_set}_sp_rvb_hires_comb data/${mic}/${train_set}_sp_hires_comb ${datadir}_rvb${num_data_reps}_hires_comb
+  utils/combine_data.sh data/${mic}/${train_set}_sp_rvb_hires data/${mic}/${train_set}_sp_hires ${norvb_datadir}_rvb${num_data_reps}_hires
 fi
 
-
 if [ $stage -le 4 ]; then
-  echo "$0: selecting segments of hires training data that were also present in the"
-  echo " ... original training data."
-
-  # note, these data-dirs are temporary; we put them in a sub-directory
-  # of the place where we'll make the alignments.
-  temp_data_root=exp/$mic/nnet3${nnet3_affix}/tri5
-  mkdir -p $temp_data_root
-
-  utils/data/subset_data_dir.sh --utt-list data/${mic}/${train_set}/feats.scp \
-          data/${mic}/${train_set}_sp_hires $temp_data_root/${train_set}_hires
-
-  # note: essentially all the original segments should be in the hires data.
-  n1=$(wc -l <data/${mic}/${train_set}/feats.scp)
-  n2=$(wc -l <$temp_data_root/${train_set}_hires/feats.scp)
-  if [ $n1 != $n1 ]; then
-    echo "$0: warning: number of feats $n1 != $n2, if these are very different it could be bad."
-  fi
-
-  echo "$0: training a system on the hires data for its LDA+MLLT transform, in order to produce the diagonal GMM."
-  if [ -e exp/$mic/nnet3${nnet3_affix}/tri5/final.mdl ]; then
-    # we don't want to overwrite old stuff, ask the user to delete it.
-    echo "$0: exp/$mic/nnet3${nnet3_affix}/tri5/final.mdl already exists: "
-    echo " ... please delete and then rerun, or use a later --stage option."
-    exit 1;
-  fi
-  steps/train_lda_mllt.sh --cmd "$train_cmd" --num-iters 7 --mllt-iters "2 4 6" \
-     --splice-opts "--left-context=3 --right-context=3" \
-     3000 10000 $temp_data_root/${train_set}_hires data/lang \
-      $gmmdir exp/$mic/nnet3${nnet3_affix}/tri5
+  steps/online/nnet2/get_pca_transform.sh --cmd "$train_cmd" \
+    --splice-opts "--left-context=3 --right-context=3" \
+    --max-utts 30000 --subsample 2 \
+    data/${mic}/${train_set}_sp_rvb_hires \
+    exp/$mic/nnet3${nnet3_affix}/pca_transform
 fi
 
 
@@ -186,9 +135,8 @@ if [ $stage -le 5 ]; then
   mkdir -p exp/$mic/nnet3${nnet3_affix}/diag_ubm
   temp_data_root=exp/$mic/nnet3${nnet3_affix}/diag_ubm
 
-  # train a diagonal UBM using a subset of about a quarter of the data
-  # we don't use the _comb data for this as there is no need for compatibility with
-  # the alignments, and using the non-combined data is more efficient for I/O
+  # train a diagonal UBM using a subset of about a quarter of the data,
+  # and using the non-combined data is more efficient for I/O
   # (no messing about with piped commands).
   num_utts_total=$(wc -l <data/$mic/${train_set}_sp_rvb_hires/utt2spk)
   num_utts=$[$num_utts_total/4]
@@ -201,7 +149,7 @@ if [ $stage -le 5 ]; then
     --num-frames 700000 \
     --num-threads $num_threads_ubm \
     ${temp_data_root}/${train_set}_sp_rvb_hires_subset 512 \
-    exp/$mic/nnet3${nnet3_affix}/tri5 exp/$mic/nnet3${nnet3_affix}/diag_ubm
+    exp/$mic/nnet3${nnet3_affix}/pca_transform exp/$mic/nnet3${nnet3_affix}/diag_ubm
 fi
 
 if [ $stage -le 6 ]; then
@@ -217,7 +165,7 @@ if [ $stage -le 7 ]; then
   # note, we don't encode the 'max2' in the name of the ivectordir even though
   # that's the data we extract the ivectors from, as it's still going to be
   # valid for the non-'max2' data, the utterance list is the same.
-  ivectordir=exp/$mic/nnet3${nnet3_affix}/ivectors_${train_set}_sp_rvb_hires_comb
+  ivectordir=exp/$mic/nnet3${nnet3_affix}/ivectors_${train_set}_sp_rvb_hires
   if [[ $(hostname -f) == *.clsp.jhu.edu ]] && [ ! -d $ivectordir/storage ]; then
     utils/create_split_dir.pl /export/b0{5,6,7,8}/$USER/kaldi-data/egs/ami-$mic-$(date +'%m_%d_%H_%M')/s5/$ivectordir/storage $ivectordir/storage
   fi
@@ -231,10 +179,10 @@ if [ $stage -le 7 ]; then
   # handle per-utterance decoding well (iVector starts at zero).
   temp_data_root=${ivectordir}
   utils/data/modify_speaker_info.sh --utts-per-spk-max 2 \
-    data/${mic}/${train_set}_sp_rvb_hires_comb ${temp_data_root}/${train_set}_sp_rvb_hires_comb_max2
+    data/${mic}/${train_set}_sp_rvb_hires ${temp_data_root}/${train_set}_sp_rvb_hires_max2
 
   steps/online/nnet2/extract_ivectors_online.sh --cmd "$train_cmd" --nj $nj \
-    ${temp_data_root}/${train_set}_sp_rvb_hires_comb_max2 \
+    ${temp_data_root}/${train_set}_sp_rvb_hires_max2 \
     exp/$mic/nnet3${nnet3_affix}/extractor $ivectordir
 
   # Also extract iVectors for the test data, but in this case we don't need the speed

egs/ami/s5b/local/nnet3/prepare_lores_feats.sh

@@ -14,9 +14,6 @@ set -e -o pipefail
 stage=0
 mic=ihm
 nj=30
-min_seg_len=1.55  # min length in seconds... we do this because chain training
-                  # will discard segments shorter than 1.5 seconds.  Must remain in
-                  # sync with the same option given to run_ivector_common.sh.
 use_ihm_ali=false # If true, we use alignments from the IHM data (which is better..
                   # don't set this to true if $mic is set to ihm.)
 train_set=train   # you might set this to e.g. train_cleaned.
@@ -69,16 +66,4 @@ if [ $stage -le 9 ]; then
   utils/fix_data_dir.sh data/${mic}/${train_set}${ihm_suffix}_sp
 fi
 
-if [ $stage -le 10 ]; then
-  echo "$0: combining short segments of 13-dimensional speed-perturbed ${maybe_ihm}MFCC data"
-  src=data/${mic}/${train_set}${ihm_suffix}_sp
-  dest=data/${mic}/${train_set}${ihm_suffix}_sp_comb
-  utils/data/combine_short_segments.sh $src $min_seg_len $dest
-  # re-use the CMVN stats from the source directory, since it seems to be slow to
-  # re-compute them after concatenating short segments.
-  cp $src/cmvn.scp $dest/
-  utils/fix_data_dir.sh $dest
-fi
-
-
 exit 0;

egs/ami/s5b/local/prepare_parallel_train_data.sh

@@ -60,6 +60,6 @@ utils/apply_map.pl -f 1 $tmpdir/ihmutt2utt <data/ihm/train/segments >data/$mic/t
 
 utils/fix_data_dir.sh data/$mic/train_ihmdata
 
-rm $tmpdir/ihmutt2utt
+#rm $tmpdir/ihmutt2utt
 
 exit 0;

egs/ami/s5b/path.sh

@@ -11,3 +11,5 @@ BEAMFORMIT=$KALDI_ROOT/tools/BeamformIt
 
 export PATH=$PATH:$LMBIN:$BEAMFORMIT:$SRILM
 
+. /etc/profile.d/modules.sh
+module load shared cuda80/toolkit

egs/ami/s5b/run.sh

@@ -31,6 +31,7 @@ case $(hostname -d) in
   fit.vutbr.cz) AMI_DIR=/mnt/matylda5/iveselyk/KALDI_AMI_WAV ;; # BUT,
   clsp.jhu.edu) AMI_DIR=/export/corpora4/ami/amicorpus ;; # JHU,
   cstr.ed.ac.uk) AMI_DIR= ;; # Edinburgh,
+  cm.gemini) AMI_DIR=/export/common/data/corpora/amicorpus;; # COE
 esac
 
 [ ! -r data/local/lm/final_lm ] && echo "Please, run 'run_prepare_shared.sh' first!" && exit 1
@@ -163,6 +164,8 @@ if [ $stage -le 10 ]; then
   local/run_cleanup_segmentation.sh --mic $mic
 fi
 
+exit 0
+
 if [ $stage -le 11 ]; then
   ali_opt=
   [ "$mic" != "ihm" ] && ali_opt="--use-ihm-ali true"

egs/ami/s5b/run_prepare_shared.sh

@@ -8,6 +8,7 @@ case $(hostname -d) in
   fit.vutbr.cz) FISHER_TRANS=/mnt/matylda2/data/FISHER/fe_03_p1_tran ;; # BUT,
   clsp.jhu.edu) FISHER_TRANS=/export/corpora4/ami/fisher_trans/part1 ;; # JHU,
   cstr.ed.ac.uk) FISHER_TRANS=`pwd`/eddie_data/lm/data/fisher/part1 ;; # Edinburgh,
+  cm.gemini) FISHER_TRANS=/export/common/data/corpora/LDC/LDC2004T19_CLSP_format/fe_03_p1_tran/;; # COE
   *) echo "Please modify the script to add your loaction of the Fisher transcripts, or modify this script."; exit 1;;
 esac
 # Or select manually,

egs/aspire/s5/conf/mfcc_hires.conf

@@ -8,3 +8,4 @@
 --num-ceps=40     # there is no dimensionality reduction.
 --low-freq=40    # low cutoff frequency for mel bins
 --high-freq=-200 # high cutoff frequently, relative to Nyquist of 4000 (=3800)
+--allow-downsample=true

egs/aspire/s5/local/chain/tuning/run_tdnn_lstm_1a.sh

@@ -20,6 +20,7 @@ test_stage=1
 nj=70
 
 tdnn_affix=_1a
+chain_affix=
 
 hidden_dim=1024
 cell_dim=1024
@@ -62,9 +63,9 @@ fi
 train_set=train_rvb
 
 gmm_dir=exp/tri5a   # used to get training lattices (for chain supervision)
-treedir=exp/chain/tree_bi_a
-lat_dir=exp/chain/tri5a_${train_set}_lats  # training lattices directory
-dir=exp/chain/tdnn_lstm${tdnn_affix}
+treedir=exp/chain${chain_affix}/tree_bi_a
+lat_dir=exp/chain${chain_affix}/tri5a_${train_set}_lats  # training lattices directory
+dir=exp/chain${chain_affix}/tdnn_lstm${tdnn_affix}
 train_data_dir=data/${train_set}_hires
 train_ivector_dir=exp/nnet3/ivectors_${train_set}
 lang=data/lang_chain
@@ -77,7 +78,7 @@ local/nnet3/run_ivector_common.sh --stage $stage --num-data-reps 3 || exit 1
 
 mkdir -p $dir
 
-norvb_lat_dir=exp/chain/tri5a_train_lats
+norvb_lat_dir=exp/chain${chain_affix}/tri5a_train_lats
 
 if [ $stage -le 7 ]; then
   # Get the alignments as lattices (gives the chain training more freedom).
@@ -257,10 +258,10 @@ if [ $stage -le 15 ]; then
 
   for d in dev_rvb test_rvb; do
     (
-      if [ ! -f exp/nnet3/ivectors_${d}/ivector_online.scp ]; then
+      if [ ! -f exp/nnet3${nnet3_affix}/ivectors_${d}/ivector_online.scp ]; then
         steps/online/nnet2/extract_ivectors_online.sh --cmd "$train_cmd" --nj 30 \
-          data/${d}_hires exp/nnet3/extractor \
-          exp/nnet3/ivectors_${d} || { echo "Failed i-vector extraction for data/${d}_hires"; touch $dir/.error; }
+          data/${d}_hires exp/nnet3${nnet3_affix}/extractor \
+          exp/nnet3${nnet3_affix}/ivectors_${d} || { echo "Failed i-vector extraction for data/${d}_hires"; touch $dir/.error; }
       fi
 
       decode_dir=$dir/decode_${d}_pp
@@ -270,7 +271,7 @@ if [ $stage -le 15 ]; then
         --extra-right-context $extra_right_context \
         --extra-left-context-initial 0 --extra-right-context-final 0 \
         --frames-per-chunk 160 \
-        --online-ivector-dir exp/nnet3/ivectors_${d} \
+        --online-ivector-dir exp/nnet3${nnet3_affix}/ivectors_${d} \
         $graph_dir data/${d}_hires $decode_dir || { echo "Failed decoding in $decode_dir"; touch $dir/.error; }
     ) &
   done
@@ -292,7 +293,7 @@ if [ $stage -le 16 ]; then
    --extra-left-context-initial 0 --extra-right-context-final 0 \
    --sub-speaker-frames 6000 --max-count 75 --ivector-scale 0.75 \
    --pass2-decode-opts "--min-active 1000" \
-   dev_aspire data/lang $dir/graph_pp $dir
+   dev_aspire_ldc data/lang $dir/graph_pp $dir
 fi
 
 if [ $stage -le 17 ]; then
@@ -305,7 +306,7 @@ if [ $stage -le 17 ]; then
    --extra-left-context-initial 0 \
    --max-count 75 \
    --pass2-decode-opts "--min-active 1000" \
-   dev_aspire data/lang $dir/graph_pp $dir
+   dev_aspire_ldc data/lang $dir/graph_pp $dir
 fi
 
 exit 0;