[scripts,build] Fix the MKL LD_PRELOAD issue for kaldi pybind test; rename a file. (#3817)

Author: csukuangfj

egs/aishell/s10/chain/inference.py

@@ -43,7 +43,7 @@ def main():
     model.eval()
 
     specifier = 'ark,scp:{filename}.ark,{filename}.scp'.format(
-        filename=os.path.join(args.dir, 'confidence'))
+        filename=os.path.join(args.dir, 'nnet_output'))
 
     if args.save_as_compressed:
         Writer = kaldi.CompressedMatrixWriter
@@ -83,8 +83,8 @@ def main():
                 float(batch_idx) / len(dataloader) * 100))
 
     writer.Close()
-    logging.info('confidence is saved to {}'.format(
-        os.path.join(args.dir, 'confidence.scp')))
+    logging.info('pseudo-log-likelihood is saved to {}'.format(
+        os.path.join(args.dir, 'nnet_output.scp')))
 
 
 if __name__ == '__main__':

egs/aishell/s10/local/decode.sh

@@ -20,13 +20,13 @@ post_decode_acwt=10  # can be used in 'chain' systems to scale acoustics by 10
 . parse_options.sh || exit 1
 
 if [ $# != 4 ]; then
-    echo "Usage: $0 [options] <graph-dir> <trans_model> <confidence_scp> <decode-dir>"
+    echo "Usage: $0 [options] <graph-dir> <trans_model> <nnet_output_scp> <decode-dir>"
     exit 1
 fi
 
 graphdir=$1
 trans_model=$2
-confidence_scp=$3
+nnet_output_scp=$3
 dir=$4
 
 if [[ ! -d $graphdir ]]; then
@@ -39,15 +39,15 @@ if [[ ! -f $trans_model ]]; then
   exit 1
 fi
 
-if [[ ! -f $confidence_scp ]]; then
-  echo "confidence scp $confidence_scp does not exist"
+if [[ ! -f $nnet_output_scp ]]; then
+  echo "nnet_output_scp $nnet_output_scp does not exist"
   exit 1
 fi
 
 mkdir -p $dir
 
 for i in $(seq $nj); do
-  utils/split_scp.pl -j $nj $[$i - 1] $confidence_scp $dir/confidence.$i.scp
+  utils/split_scp.pl -j $nj $[$i - 1] $nnet_output_scp $dir/nnet_output.$i.scp
 done
 
 lat_wspecifier="ark:|lattice-scale --acoustic-scale=$post_decode_acwt ark:- ark:- | gzip -c >$dir/lat.JOB.gz"
@@ -66,4 +66,4 @@ $cmd --num-threads $num_threads JOB=1:$nj $dir/log/decode.JOB.log \
     --min-active=$min_active \
     --num-threads=$num_threads \
     --word-symbol-table=$graphdir/words.txt \
-    $trans_model $graphdir/HCLG.fst scp:$dir/confidence.JOB.scp "$lat_wspecifier" || exit 1
+    $trans_model $graphdir/HCLG.fst scp:$dir/nnet_output.JOB.scp "$lat_wspecifier" || exit 1

egs/aishell/s10/local/run_chain.sh

@@ -174,8 +174,8 @@ if [[ $stage -le 9 ]]; then
   echo "inference: computing likelihood"
   for x in test dev; do
     mkdir -p exp/chain/inference/$x
-    if [[ -f exp/chain/inference/$x/confidence.scp ]]; then
-      echo "exp/chain/inference/$x/confidence.scp already exists! Skip"
+    if [[ -f exp/chain/inference/$x/nnet_output.scp ]]; then
+      echo "exp/chain/inference/$x/nnet_output.scp already exists! Skip"
     else
       best_epoch=$(cat exp/chain/train/best-epoch-info | grep 'best epoch' | awk '{print $NF}')
       inference_checkpoint=exp/chain/train/epoch-${best_epoch}.pt
@@ -201,8 +201,8 @@ fi
 if [[ $stage -le 10 ]]; then
   echo "decoding"
   for x in test dev; do
-    if [[ ! -f exp/chain/inference/$x/confidence.scp ]]; then
-      echo "exp/chain/inference/$x/confidence.scp does not exist!"
+    if [[ ! -f exp/chain/inference/$x/nnet_output.scp ]]; then
+      echo "exp/chain/inference/$x/nnet_output.scp does not exist!"
       echo "Please run inference.py first"
       exit 1
     fi
@@ -212,7 +212,7 @@ if [[ $stage -le 10 ]]; then
       --nj $nj \
       exp/chain/graph \
       exp/chain/0.trans_mdl \
-      exp/chain/inference/$x/confidence.scp \
+      exp/chain/inference/$x/nnet_output.scp \
       exp/chain/decode_res/$x
   done
 fi

src/configure

@@ -1244,6 +1244,15 @@ elif [ "`uname`" == "Linux" ]; then
       echo MKLLIB = $MKLLIBDIR >> kaldi.mk
     fi
     echo >> kaldi.mk
+
+    echo "MKL_LD_PRELOAD := $MKLROOT/lib/intel64/libmkl_def.so \\
+                  $MKLROOT/lib/intel64/libmkl_avx2.so \\
+                  $MKLROOT/lib/intel64/libmkl_core.so \\
+                  $MKLROOT/lib/intel64/libmkl_intel_lp64.so \\
+                  $MKLROOT/lib/intel64/libmkl_intel_thread.so" >> kaldi.mk
+
+    echo >> kaldi.mk
+
     check_exists makefiles/linux_x86_64_mkl.mk
     cat makefiles/linux_x86_64_mkl.mk >> kaldi.mk
     echo "MKLFLAGS = ${MKL_LINK_LINE} ${THREADING_LINE} $EXTRA_LIBS " >> kaldi.mk

src/pybind/Makefile

@@ -97,6 +97,10 @@ LDFLAGS += -Wl,-rpath=$(CURDIR)/../../tools/openfst/lib
 
 LIBFILE=$(LIBNAME)$(LIBFILE_EXTENSION)
 
+ifdef MKL_LD_PRELOAD
+LD_PRELOAD += $(MKL_LD_PRELOAD)
+export LD_PRELOAD
+endif
 
 .PHONY: all clean test
 
@@ -138,6 +142,6 @@ depend:
 	for f in $(CCFILES); do \
            $(CXX) -M -MT "$$(dirname $$f)/$$(basename -s .cc $$f).o" \
            $(CXXFLAGS) $$f >> .depend.mk; \
-        done
+      done
 
 -include .depend.mk

tools/extras/install_mkl.sh

@@ -254,13 +254,15 @@ ConfigLdSo() {
     ldconfig )
 }
 
-# Load MKL libraris, which is need for pybind tests to pass
+# Load MKL libraries, which is needed for pybind tests to pass
 (
-  echo "export LD_PRELOAD:=/opt/intel/mkl/lib/intel64/libmkl_def.so:\
+  # WARNING: we assume that you installed MKL in the default path: /opt/intel
+  # if not, please refer to `MKL_LD_PRELOAD` in `src/kaldi.mk`
+  echo "export LD_PRELOAD=/opt/intel/mkl/lib/intel64/libmkl_def.so:\
 /opt/intel/mkl/lib/intel64/libmkl_avx2.so:\
 /opt/intel/mkl/lib/intel64/libmkl_core.so:\
 /opt/intel/mkl/lib/intel64/libmkl_intel_lp64.so:\
-/opt/intel/mkl/lib/intel64/libmkl_intel_thread.so"
+/opt/intel/mkl/lib/intel64/libmkl_intel_thread.so:${LD_PRELOAD}"
 ) >> env.sh
 
 # Invoke installation.