ROCBLAS_LAYER_debug branch

ROCm · Nov 20, 2023 · 71009c8 · 71009c8
1 parent 02dd2d9
commit 71009c8
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 24 deletions.
diff --git a/onnxruntime/python/tools/transformers/models/llama2/llama2_config.sh b/onnxruntime/python/tools/transformers/models/llama2/llama2_config.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-N_GPU=1
+N_GPU=8
 
 bash sample_run.sh $N_GPU --export --optimize --merge
 
@@ -11,25 +11,3 @@ bash sample_run.sh $N_GPU --optimize --merge --custom-gen --benchmark --ort 2>&1
 sed -i -E 's/\[[0-9]*,[0-9]*\]<std(out|err)>://g' ${MODEL_NAME}_rocblas_configs.log
 grep -o "rocblas-bench.*" ${MODEL_NAME}_rocblas_configs.log | sort -u &> unique_rocblas_configs_$MODEL_NAME.log
 unset ROCBLAS_LAYER
-
-export MIOPEN_ENABLE_LOGGING_CMD=1
-bash sample_run.sh $N_GPU --optimize --merge --custom-gen --benchmark --ort 2>&1 | tee ${MODEL_NAME}_miopen_configs.log
-sed -i -E 's/\[[0-9]*,[0-9]*\]<std(out|err)>://g' ${MODEL_NAME}_miopen_configs.log
-grep "MIOpenDriver " ${MODEL_NAME}_miopen_configs.log | sed -e 's/.*]//' | sort -u &> unique_miopen_configs_$MODEL_NAME.log
-unset MIOPEN_ENABLE_LOGGING_CMD
-
-export TENSILE_DB=0x8000  # dump Tensile kernel names
-bash sample_run.sh $N_GPU --optimize --merge --custom-gen --benchmark --ort 2>&1 | tee ${MODEL_NAME}_tensile_configs.log
-sed -i -E 's/\[[0-9]*,[0-9]*\]<std(out|err)>://g' ${MODEL_NAME}_tensile_configs.log
-grep "Running kernel: " ${MODEL_NAME}_tensile_configs.log | sort -u &> unique_kernel_names_$MODEL_NAME.log
-unset TENSILE_DB
-
-# export HIPBLASLT_LOG_LEVEL=2
-# unset HIPBLASLT_LOG_LEVEL
-
-echo "========================Math lib profiling done"
-
-export RCCL_MSCCL_ENABLE=0
-NCCL_LOGFILE=$MODEL_NAME-NCCL.log
-HSA_FORCE_FINE_GRAIN_PCIE=1 NCCL_DEBUG=INFO NCCL_DEBUG_SUBSYS=INIT,COLL ./sample_run.sh $N_GPU --optimize --merge --custom-gen --benchmark --ort 2>&1 | tee $NCCL_LOGFILE
-python /workspace/nccl-rccl-parser/rccl_nccl_parser.py --nccl-debug-log $NCCL_LOGFILE --output-script-name unique_nccl_configs_$MODEL_NAME.log --unique
diff --git a/onnxruntime/python/tools/transformers/models/llama2/sample_run.sh b/onnxruntime/python/tools/transformers/models/llama2/sample_run.sh
@@ -5,7 +5,7 @@ NUM_GPUS=${1:-16}
 
 MPI="mpirun --allow-run-as-root
     -mca btl_openib_warn_no_device_params_found 0 -mca pml ob1 -mca btl ^openib -mca btl_tcp_if_include eth0
-    --tag-output --npernode $NUM_GPUS --bind-to numa
+    --tag-output --npernode $NUM_GPUS --bind-to numa --output-filename outputs
     -x MIOPEN_FIND_MODE=1"
 
 # MPI+=" -x TRANSFORMERS_OFFLINE=1"