Add evaluation test to regression testing (#255)

.github/workflows/test.yml

@@ -5,8 +5,48 @@ on:
       - develop
 
 jobs:
+  eval:
+    runs-on: self-hosted
+    timeout-minutes: 345600
+    steps:
+      - name: Checkout last-dev
+        uses: actions/checkout@v2
+        with:
+          repository: MaximIntegratedAI/ai8x-training
+          ref: develop
+          submodules: recursive
+      - name: Checkout synthesis
+        uses: actions/checkout@v2
+        with:
+          repository: MaximIntegratedAI/ai8x-synthesis
+          ref: develop
+          path: ai8x-synthesis
+      - name: Setup Pyenv and Install Dependencies
+        uses: gabrielfalcao/pyenv-action@v13
+        with:
+          default: 3.8.11
+      - name: Create Venv
+        run: |
+          pyenv local 3.8.11
+          python -m venv venv --prompt ai8x-training
+      - name: Activate Venv
+        run: source venv/bin/activate
+      - name: Install Dependencies
+        run: |
+          pip3 install -U pip wheel setuptools
+          pip3 install -r requirements-cu11.txt
+      - name: Create Evaluation Scripts
+        run: python ./regression/create_eval_script.py --testconf ./regression/test_config.yaml --testpaths ./regression/paths.yaml
+      - name: Run Evaluation Scripts
+        run: bash ./scripts/evaluation_file.sh
+      - name: Save Evaluation Log Files
+        run: cp -r /home/test/actions-runner/_work/ai8x-training/ai8x-training/logs/ /home/test/max7800x/evaluation_logs/
+      - name: Evaluation Results
+        run: python ./regression/eval_pass_fail.py --testpaths ./regression/paths.yaml
+
   build:
     runs-on: self-hosted
+    needs: [eval]
     timeout-minutes: 345600
     steps:
       - name: Checkout last-dev
@@ -22,7 +62,7 @@ jobs:
       - name: Create Venv
         run: |
           pyenv local 3.8.11
-          python3 -m venv venv --prompt ai8x-training
+          python -m venv venv --prompt ai8x-training
       - name: Activate Venv
         run: source venv/bin/activate
       - name: Install Dependencies
@@ -49,7 +89,7 @@ jobs:
       - name: Create Venv
         run: |
           pyenv local 3.8.11
-          python3 -m venv venv --prompt ai8x-training
+          python -m venv venv --prompt ai8x-training
       - name: Activate Venv
         run: source venv/bin/activate
       - name: Install Dependencies
@@ -85,7 +125,7 @@ jobs:
       - name: Create Venv
         run: |
           pyenv local 3.8.11
-          python3 -m venv venv --prompt ai8x-training
+          python -m venv venv --prompt ai8x-training
       - name: Activate Venv
         run: source venv/bin/activate
       - name: Install Dependencies

notebooks/KWS_Audio_Evaluation.ipynb

@@ -608,6 +608,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "303428e3",
    "metadata": {},

regression/create_eval_script.py

@@ -0,0 +1,65 @@
+###################################################################################################
+#
+# Copyright (C) 2023 Analog Devices, Inc. All Rights Reserved.
+# This software is proprietary to Analog Devices, Inc. and its licensors.
+#
+###################################################################################################
+"""
+Create training bash scripts for test
+"""
+import argparse
+import os
+
+import yaml
+
+
+def joining(lst):
+    """
+    Join list based on the ' ' delimiter
+    """
+    join_str = ' '.join(lst)
+    return join_str
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--testconf', help='Enter the config file for the test', required=True)
+parser.add_argument('--testpaths', help='Enter the paths for the test', required=True)
+args = parser.parse_args()
+yaml_path = args.testconf
+test_path = args.testpaths
+
+# Open the YAML file
+with open(yaml_path, 'r', encoding='utf-8') as yaml_file:
+    # Load the YAML content into a Python dictionary
+    config = yaml.safe_load(yaml_file)
+
+with open(test_path, 'r', encoding='utf-8') as path_file:
+    # Load the YAML content into a Python dictionary
+    pathconfig = yaml.safe_load(path_file)
+
+# Folder containing the files to be concatenated
+script_path = pathconfig["script_path"]
+
+# Output file name and path
+output_file_path = pathconfig["output_file_path_evaluation"]
+
+# Loop through all files in the folder
+with open(output_file_path, "w", encoding='utf-8') as evaluate_file:
+    for filename in os.listdir(script_path):
+        # Check if the file is a text file
+        if filename.startswith("evaluate"):
+            # Open the file and read its contents
+            with open(os.path.join(script_path, filename), encoding='utf-8') as input_file:
+                contents = input_file.read()
+                temp = contents.split()
+                temp.insert(1, "\n")
+
+                i = temp.index("--exp-load-weights-from")
+                temp[i+1] = temp[i+1][1:]
+
+                temp.insert(-1, "--name " + filename[9:-3])
+                temp.insert(-1, "--data /data_ssd")
+
+                temp.append(" \n")
+                contents = joining(temp)
+                evaluate_file.write(contents)

regression/create_onnx_script.py

@@ -108,7 +108,6 @@ def time_stamp():
             modelsearch = element[-4][3:]
             datasearch = element[-3].split('_')[0]
             if datasearch == dataset.split('_')[0] and modelsearch == model:
-                # model_paths.remove(tar)
                 tar_path = tar
                 timestamp = time_stamp()
                 temp = (

regression/create_test_script.py

@@ -83,18 +83,28 @@ def joining(lst):
                 if log_data == "VGGFace2_FaceDetection":
                     continue
 
-                temp[i+1] = str(config[log_data][log_model]["epoch"])
+                try:
+                    temp[i+1] = str(config[log_data][log_model]["epoch"])
+                except KeyError:
+                    print(f"\033[93m\u26A0\033[0m Warning: {temp[j+1]} model is" +
+                          " missing information in test configuration files.")
+                    continue
 
                 if '--deterministic' not in temp:
                     temp.insert(-1, '--deterministic')
 
                 temp.insert(-1, '--name ' + log_name)
 
-                data_name = temp[k+1]
-                if data_name in config and "datapath" in config[data_name]:
+                try:
                     path_data = config[log_data]["datapath"]
-                    temp.insert(-1, '--data ' + path_data)
+                    temp[i+1] = str(config[log_data][log_model]["epoch"])
+                except KeyError:
+                    print(f"\033[93m\u26A0\033[0m Warning: {temp[j+1]} model is" +
+                          " missing information in test configuration files.")
+                    continue
 
+                temp.insert(-1, '--data ' + path_data)
                 temp.append("\n")
+
                 contents = joining(temp)
                 output_file.write(contents)

regression/eval_pass_fail.py

@@ -0,0 +1,46 @@
+###################################################################################################
+#
+# Copyright (C) 2023 Analog Devices, Inc. All Rights Reserved.
+# This software is proprietary to Analog Devices, Inc. and its licensors.
+#
+###################################################################################################
+"""
+Check the test results
+"""
+import argparse
+import os
+
+import yaml
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--testpaths', help='Enter the paths for the test', required=True)
+args = parser.parse_args()
+test_path = args.testpaths
+
+# Open the YAML file
+with open(test_path, 'r', encoding='utf-8') as path_file:
+    # Load the YAML content into a Python dictionary
+    pathconfig = yaml.safe_load(path_file)
+
+eval_path = pathconfig["eval_path"]
+eval_file = os.listdir(eval_path)[-1]
+directory_path = os.path.join(eval_path, eval_file)
+passed = []
+failed = []
+
+for filename in sorted(os.listdir(directory_path)):
+    path = os.path.join(directory_path, filename)
+    file_path = os.path.join(path, os.listdir(path)[0])
+    with open(file_path, 'r', encoding='utf-8') as file:
+        content = file.read()
+        if "Loss" in content:
+            pass_file = filename.split("___")[0]
+            passed.append(f"\033[32m\u2714\033[0m Evaluation test passed for {pass_file}.")
+        else:
+            fail_file = filename.split("___")[0]
+            failed.append(f"\033[31m\u2718\033[0m Evaluation test failed for {fail_file}.")
+
+for filename in failed:
+    print(filename)
+for filename in passed:
+    print(filename)

regression/last_dev.py

@@ -91,18 +91,27 @@ def dev_scripts(script_pth, output_file_pth):
                     if log_data == "ai85tinierssdface":
                         continue
 
-                    temp[i+1] = str(config[log_data][log_model]["epoch"])
+                    try:
+                        temp[i+1] = str(config[log_data][log_model]["epoch"])
+                    except KeyError:
+                        print(f"\033[93m\u26A0\033[0m Warning: {temp[j+1]} model is" +
+                              " missing information in test configuration files.")
+                        continue
 
                     if '--deterministic' not in temp:
                         temp.insert(-1, '--deterministic')
 
                     temp.insert(-1, '--name ' + log_name)
 
-                    data_name = temp[k+1]
-                    if data_name in config and "datapath" in config[data_name]:
+                    try:
                         path_data = config[log_data]["datapath"]
-                        temp.insert(-1, '--data ' + path_data)
+                        temp[i+1] = str(config[log_data][log_model]["epoch"])
+                    except KeyError:
+                        print(f"\033[93m\u26A0\033[0m Warning: {temp[j+1]} model is" +
+                              " missing information in test configuration files.")
+                        continue
 
+                    temp.insert(-1, '--data ' + path_data)
                     temp.append("\n")
                     contents = joining(temp)
                     output_file.write(contents)

regression/log_comparison.py

@@ -126,7 +126,7 @@ def compare_logs(old_log, new_log, output_name, output_pth):
         i = 0
         for (map1, map2) in zip(mAP_list1, mAP_list2):
             if float(map1[1]) == 0:
-                print("Map value of " + output_name + " is 0.00.")
+                print(f"Map value of {output_name} is 0.00 at epoch {i}.")
                 map1[1] = 0.000001
             i = i+1
             if '[mAP:' in map2:

regression/paths.yaml

@@ -10,7 +10,7 @@ train_path: /home/test/max7800x/test_scripts/output_file.sh
 
 script_path: ./scripts
 output_file_path: ./scripts/output_file.sh
-
+output_file_path_evaluation: ./scripts/evaluation_file.sh
 
 # last_dev.py
 
@@ -31,3 +31,4 @@ output_path: /home/test/max7800x/log_diff/
 # pass_fail.py
 
 log_path: /home/test/max7800x/log_diff
+eval_path: /home/test/max7800x/evaluation_logs/

regression/test_config.yaml

@@ -54,7 +54,12 @@ ImageNet:
   datapath: "/data_ssd"
   ai87imageneteffnetv2:
     threshold: -5
-    epoch: 13
+    epoch: 15
+ImageNet_Bayer:
+  datapath: "/data_ssd"
+  bayer2rgbnet:
+    threshold: -5
+    epoch: 15
 KWS_20:
   datapath: "/data_ssd"
   ai85kws20net: