From 7eb510774e8880a6deda9e690a5f96af24aecc31 Mon Sep 17 00:00:00 2001 From: asyatrhl <123384000+asyatrhl@users.noreply.github.com> Date: Wed, 18 Oct 2023 20:50:34 +0300 Subject: [PATCH] Add evaluation test to regression testing (#255) --- .github/workflows/test.yml | 46 ++++++++++++++++++-- notebooks/KWS_Audio_Evaluation.ipynb | 1 + regression/create_eval_script.py | 65 ++++++++++++++++++++++++++++ regression/create_onnx_script.py | 1 - regression/create_test_script.py | 18 ++++++-- regression/eval_pass_fail.py | 46 ++++++++++++++++++++ regression/last_dev.py | 17 ++++++-- regression/log_comparison.py | 2 +- regression/paths.yaml | 3 +- regression/test_config.yaml | 7 ++- 10 files changed, 191 insertions(+), 15 deletions(-) create mode 100644 regression/create_eval_script.py create mode 100644 regression/eval_pass_fail.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3974208fc..8ed3217e8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,8 +5,48 @@ on: - develop jobs: + eval: + runs-on: self-hosted + timeout-minutes: 345600 + steps: + - name: Checkout last-dev + uses: actions/checkout@v2 + with: + repository: MaximIntegratedAI/ai8x-training + ref: develop + submodules: recursive + - name: Checkout synthesis + uses: actions/checkout@v2 + with: + repository: MaximIntegratedAI/ai8x-synthesis + ref: develop + path: ai8x-synthesis + - name: Setup Pyenv and Install Dependencies + uses: gabrielfalcao/pyenv-action@v13 + with: + default: 3.8.11 + - name: Create Venv + run: | + pyenv local 3.8.11 + python -m venv venv --prompt ai8x-training + - name: Activate Venv + run: source venv/bin/activate + - name: Install Dependencies + run: | + pip3 install -U pip wheel setuptools + pip3 install -r requirements-cu11.txt + - name: Create Evaluation Scripts + run: python ./regression/create_eval_script.py --testconf ./regression/test_config.yaml --testpaths ./regression/paths.yaml + - name: Run Evaluation Scripts + run: bash ./scripts/evaluation_file.sh + - name: Save Evaluation Log Files + run: cp -r /home/test/actions-runner/_work/ai8x-training/ai8x-training/logs/ /home/test/max7800x/evaluation_logs/ + - name: Evaluation Results + run: python ./regression/eval_pass_fail.py --testpaths ./regression/paths.yaml + build: runs-on: self-hosted + needs: [eval] timeout-minutes: 345600 steps: - name: Checkout last-dev @@ -22,7 +62,7 @@ jobs: - name: Create Venv run: | pyenv local 3.8.11 - python3 -m venv venv --prompt ai8x-training + python -m venv venv --prompt ai8x-training - name: Activate Venv run: source venv/bin/activate - name: Install Dependencies @@ -49,7 +89,7 @@ jobs: - name: Create Venv run: | pyenv local 3.8.11 - python3 -m venv venv --prompt ai8x-training + python -m venv venv --prompt ai8x-training - name: Activate Venv run: source venv/bin/activate - name: Install Dependencies @@ -85,7 +125,7 @@ jobs: - name: Create Venv run: | pyenv local 3.8.11 - python3 -m venv venv --prompt ai8x-training + python -m venv venv --prompt ai8x-training - name: Activate Venv run: source venv/bin/activate - name: Install Dependencies diff --git a/notebooks/KWS_Audio_Evaluation.ipynb b/notebooks/KWS_Audio_Evaluation.ipynb index a48319e47..068f9e1f7 100644 --- a/notebooks/KWS_Audio_Evaluation.ipynb +++ b/notebooks/KWS_Audio_Evaluation.ipynb @@ -608,6 +608,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "303428e3", "metadata": {}, diff --git a/regression/create_eval_script.py b/regression/create_eval_script.py new file mode 100644 index 000000000..7ff2ebd2d --- /dev/null +++ b/regression/create_eval_script.py @@ -0,0 +1,65 @@ +################################################################################################### +# +# Copyright (C) 2023 Analog Devices, Inc. All Rights Reserved. +# This software is proprietary to Analog Devices, Inc. and its licensors. +# +################################################################################################### +""" +Create training bash scripts for test +""" +import argparse +import os + +import yaml + + +def joining(lst): + """ + Join list based on the ' ' delimiter + """ + join_str = ' '.join(lst) + return join_str + + +parser = argparse.ArgumentParser() +parser.add_argument('--testconf', help='Enter the config file for the test', required=True) +parser.add_argument('--testpaths', help='Enter the paths for the test', required=True) +args = parser.parse_args() +yaml_path = args.testconf +test_path = args.testpaths + +# Open the YAML file +with open(yaml_path, 'r', encoding='utf-8') as yaml_file: + # Load the YAML content into a Python dictionary + config = yaml.safe_load(yaml_file) + +with open(test_path, 'r', encoding='utf-8') as path_file: + # Load the YAML content into a Python dictionary + pathconfig = yaml.safe_load(path_file) + +# Folder containing the files to be concatenated +script_path = pathconfig["script_path"] + +# Output file name and path +output_file_path = pathconfig["output_file_path_evaluation"] + +# Loop through all files in the folder +with open(output_file_path, "w", encoding='utf-8') as evaluate_file: + for filename in os.listdir(script_path): + # Check if the file is a text file + if filename.startswith("evaluate"): + # Open the file and read its contents + with open(os.path.join(script_path, filename), encoding='utf-8') as input_file: + contents = input_file.read() + temp = contents.split() + temp.insert(1, "\n") + + i = temp.index("--exp-load-weights-from") + temp[i+1] = temp[i+1][1:] + + temp.insert(-1, "--name " + filename[9:-3]) + temp.insert(-1, "--data /data_ssd") + + temp.append(" \n") + contents = joining(temp) + evaluate_file.write(contents) diff --git a/regression/create_onnx_script.py b/regression/create_onnx_script.py index d8564b2e1..5c64177c9 100644 --- a/regression/create_onnx_script.py +++ b/regression/create_onnx_script.py @@ -108,7 +108,6 @@ def time_stamp(): modelsearch = element[-4][3:] datasearch = element[-3].split('_')[0] if datasearch == dataset.split('_')[0] and modelsearch == model: - # model_paths.remove(tar) tar_path = tar timestamp = time_stamp() temp = ( diff --git a/regression/create_test_script.py b/regression/create_test_script.py index 48f7cb6be..b8a174aaf 100644 --- a/regression/create_test_script.py +++ b/regression/create_test_script.py @@ -83,18 +83,28 @@ def joining(lst): if log_data == "VGGFace2_FaceDetection": continue - temp[i+1] = str(config[log_data][log_model]["epoch"]) + try: + temp[i+1] = str(config[log_data][log_model]["epoch"]) + except KeyError: + print(f"\033[93m\u26A0\033[0m Warning: {temp[j+1]} model is" + + " missing information in test configuration files.") + continue if '--deterministic' not in temp: temp.insert(-1, '--deterministic') temp.insert(-1, '--name ' + log_name) - data_name = temp[k+1] - if data_name in config and "datapath" in config[data_name]: + try: path_data = config[log_data]["datapath"] - temp.insert(-1, '--data ' + path_data) + temp[i+1] = str(config[log_data][log_model]["epoch"]) + except KeyError: + print(f"\033[93m\u26A0\033[0m Warning: {temp[j+1]} model is" + + " missing information in test configuration files.") + continue + temp.insert(-1, '--data ' + path_data) temp.append("\n") + contents = joining(temp) output_file.write(contents) diff --git a/regression/eval_pass_fail.py b/regression/eval_pass_fail.py new file mode 100644 index 000000000..0de504ea7 --- /dev/null +++ b/regression/eval_pass_fail.py @@ -0,0 +1,46 @@ +################################################################################################### +# +# Copyright (C) 2023 Analog Devices, Inc. All Rights Reserved. +# This software is proprietary to Analog Devices, Inc. and its licensors. +# +################################################################################################### +""" +Check the test results +""" +import argparse +import os + +import yaml + +parser = argparse.ArgumentParser() +parser.add_argument('--testpaths', help='Enter the paths for the test', required=True) +args = parser.parse_args() +test_path = args.testpaths + +# Open the YAML file +with open(test_path, 'r', encoding='utf-8') as path_file: + # Load the YAML content into a Python dictionary + pathconfig = yaml.safe_load(path_file) + +eval_path = pathconfig["eval_path"] +eval_file = os.listdir(eval_path)[-1] +directory_path = os.path.join(eval_path, eval_file) +passed = [] +failed = [] + +for filename in sorted(os.listdir(directory_path)): + path = os.path.join(directory_path, filename) + file_path = os.path.join(path, os.listdir(path)[0]) + with open(file_path, 'r', encoding='utf-8') as file: + content = file.read() + if "Loss" in content: + pass_file = filename.split("___")[0] + passed.append(f"\033[32m\u2714\033[0m Evaluation test passed for {pass_file}.") + else: + fail_file = filename.split("___")[0] + failed.append(f"\033[31m\u2718\033[0m Evaluation test failed for {fail_file}.") + +for filename in failed: + print(filename) +for filename in passed: + print(filename) diff --git a/regression/last_dev.py b/regression/last_dev.py index 76b78e92d..767eda8f2 100644 --- a/regression/last_dev.py +++ b/regression/last_dev.py @@ -91,18 +91,27 @@ def dev_scripts(script_pth, output_file_pth): if log_data == "ai85tinierssdface": continue - temp[i+1] = str(config[log_data][log_model]["epoch"]) + try: + temp[i+1] = str(config[log_data][log_model]["epoch"]) + except KeyError: + print(f"\033[93m\u26A0\033[0m Warning: {temp[j+1]} model is" + + " missing information in test configuration files.") + continue if '--deterministic' not in temp: temp.insert(-1, '--deterministic') temp.insert(-1, '--name ' + log_name) - data_name = temp[k+1] - if data_name in config and "datapath" in config[data_name]: + try: path_data = config[log_data]["datapath"] - temp.insert(-1, '--data ' + path_data) + temp[i+1] = str(config[log_data][log_model]["epoch"]) + except KeyError: + print(f"\033[93m\u26A0\033[0m Warning: {temp[j+1]} model is" + + " missing information in test configuration files.") + continue + temp.insert(-1, '--data ' + path_data) temp.append("\n") contents = joining(temp) output_file.write(contents) diff --git a/regression/log_comparison.py b/regression/log_comparison.py index 41c4fcab9..23514376c 100644 --- a/regression/log_comparison.py +++ b/regression/log_comparison.py @@ -126,7 +126,7 @@ def compare_logs(old_log, new_log, output_name, output_pth): i = 0 for (map1, map2) in zip(mAP_list1, mAP_list2): if float(map1[1]) == 0: - print("Map value of " + output_name + " is 0.00.") + print(f"Map value of {output_name} is 0.00 at epoch {i}.") map1[1] = 0.000001 i = i+1 if '[mAP:' in map2: diff --git a/regression/paths.yaml b/regression/paths.yaml index 200934c1f..e73a8fe70 100644 --- a/regression/paths.yaml +++ b/regression/paths.yaml @@ -10,7 +10,7 @@ train_path: /home/test/max7800x/test_scripts/output_file.sh script_path: ./scripts output_file_path: ./scripts/output_file.sh - +output_file_path_evaluation: ./scripts/evaluation_file.sh # last_dev.py @@ -31,3 +31,4 @@ output_path: /home/test/max7800x/log_diff/ # pass_fail.py log_path: /home/test/max7800x/log_diff +eval_path: /home/test/max7800x/evaluation_logs/ diff --git a/regression/test_config.yaml b/regression/test_config.yaml index acbc62aae..856b584f3 100644 --- a/regression/test_config.yaml +++ b/regression/test_config.yaml @@ -54,7 +54,12 @@ ImageNet: datapath: "/data_ssd" ai87imageneteffnetv2: threshold: -5 - epoch: 13 + epoch: 15 +ImageNet_Bayer: + datapath: "/data_ssd" + bayer2rgbnet: + threshold: -5 + epoch: 15 KWS_20: datapath: "/data_ssd" ai85kws20net: