Log comparison parallelization (#21)

* Parallel Processing --------- Co-authored-by: Aswin Arun <[email protected]>
incoresemi · May 31, 2024 · 8d6ad62 · 8d6ad62
1 parent db1d2ea
commit 8d6ad62
Show file tree

Hide file tree

Showing 7 changed files with 55 additions and 45 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,10 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.7.0] - 2024-05-28
+- parallelization of the log comparisons
+- added --nproc to river_core compile api
+
 ## [1.6.0] - 2024-05-27
 - smart-diff integration with river_core
 

diff --git a/docs/source/installation.rst b/docs/source/installation.rst
@@ -273,6 +273,7 @@ Output for ``river_core compile --help``:
                                     detects river_core.ini in current directory
                                     or in the ~ directory
     -t, --test_list FILE            Test List file to pass  [required]
+    --nproc JOBS                    Number of processes dedicated to river_core
     -v, --verbosity TEXT            Set the verbosity level for the framework
     --version                       Show the version and exit.
     --help                          Show this message and exit.

diff --git a/river_core/__init__.py b/river_core/__init__.py
@@ -4,4 +4,4 @@
 
 __author__ = """InCore Semiconductors"""
 __email__ = '[email protected]'
-__version__ = '1.6.0'
+__version__ = '1.7.0'
diff --git a/river_core/main.py b/river_core/main.py
@@ -143,9 +143,13 @@ def setup(config, dut, gen, ref, verbosity):
     '--coverage',
     is_flag=True,
     help='Enable collection of coverage statistics from the DuT plugin')
+@click.option(
+    '--nproc',
+    default=1,
+    help='Number of processes dedicated to rivercore framework')
 @cli.command()
 def compile(config, test_list, coverage, verbosity, dut_stage, ref_stage,
-            compare):
+            compare, nproc):
     '''
         subcommand to compile generated programs.
     '''
@@ -175,7 +179,7 @@ def compile(config, test_list, coverage, verbosity, dut_stage, ref_stage,
                     'Compare is enabled\nThis will be generating incomplete reports'
                 )
     rivercore_compile(config, test_list, coverage, verbosity, dut_stage,
-                      ref_stage, compare)
+                      ref_stage, compare, nproc)
 
 
 @click.version_option(version=__version__)

diff --git a/river_core/rivercore.py b/river_core/rivercore.py
@@ -25,6 +25,7 @@
 yaml.allow_unicode = True
 yaml.compact(seq_seq=False, seq_map=False)
 
+from multiprocessing import Pool
 
 # Misc Helper Functions
 def sanitise_pytest_json(json):
@@ -47,7 +48,6 @@ def sanitise_pytest_json(json):
 
     return return_data
 
-
 def generate_coverage_report(output_dir, config, coverage_report,
                              coverage_rank_report, db_files):
     '''
@@ -412,7 +412,7 @@ def rivercore_generate(config_file, verbosity, filter_testgen):
 
 
 def rivercore_compile(config_file, test_list, coverage, verbosity, dut_flags,
-                      ref_flags, compare):
+                      ref_flags, compare, process_count):
     '''
 
         Function to compile generated assembly programs using the plugin as configured in the config.ini.
@@ -619,59 +619,34 @@ def rivercore_compile(config_file, test_list, coverage, verbosity, dut_flags,
                 logger.warning('Ref Plugin disabled')
 
         ## Comparing Dumps
-        success = True
         if compare:
             test_dict = utils.load_yaml(test_list)
             gen_json_data = []
             target_json_data = []
             ref_json_data = []
-            for test, attr in test_dict.items():
-                test_wd = attr['work_dir']
-                is_self_checking = attr['self_checking']
-                if not is_self_checking:
-                  if not os.path.isfile(test_wd + '/dut.dump'):
-                      logger.error(f'{test:<30} : DUT dump is missing')
-                      test_dict[test]['result'] = 'Unavailable'
-                      test_dict[test]['log'] = "DUT dump is missing"
-                      success = False
-                      continue
-                  if not os.path.isfile(test_wd + '/ref.dump'):
-                      logger.error(f'{test:<30} : REF dump is missing')
-                      test_dict[test]['result'] = 'Unavailable'
-                      test_dict[test]['log'] = "REF dump is missing"
-                      success = False
-                      continue
-                  result, log, insnsize = utils.compare_dumps(test_wd + '/dut.dump', test_wd + '/ref.dump')
-                else:
-                  if not os.path.isfile(test_wd + '/dut.signature'):
-                      logger.error(f'{test:<30} : DUT signature is missing')
-                      test_dict[test]['result'] = 'Unavailable'
-                      test_dict[test]['log'] = "DUT signature is missing"
-                      success = False
-                      continue
-                  result, log = utils.self_check(test_wd + '/dut.signature')
-                  insnsize = utils.get_file_size(test_wd + '/dut.dump')
-                test_dict[test]['num_instr'] = insnsize
-                test_dict[test]['result'] = result
-                test_dict[test]['log'] = log
-                if result == 'Passed':
-                    logger.info(f"{test:<30} : TEST {result.upper()}")
-                else:
-                    success = False
-                    logger.error(f"{test:<30} : TEST {result.upper()}")
-
+            # parallelized
+            success = True
+            items = test_dict.items()
+            with Pool(processes = process_count) as process_pool:
+                output = process_pool.map(logcomparison, items) #Collecting the return values from each process in the Pool
+            #Updating values
+            for i in output:
+                success = success and i[0]
+                test_dict[i[1]]['result'] = i[2]
+                test_dict[i[1]]['log'] = i[3]
+                test_dict[i[1]]['num_instr'] = i[4]
             utils.save_yaml(test_dict, output_dir+'/result_list.yaml')
             failed_dict = {}
             for test, attr in test_dict.items():
                 if attr['result'] == 'Failed' or 'Unavailable' in attr['result']:
                     failed_dict[test] = attr
+
             if len(failed_dict) != 0:
                 logger.error(f'Total Tests that Failed :{len(failed_dict)}')
                 failed_dict_file = output_dir+'/failed_list.yaml'
                 logger.error(f'Saving failed list of tests in {failed_dict_file}')
                 utils.save_yaml(failed_dict, failed_dict_file)
 
-
             # Start checking things after running the commands
             # Report generation starts here
             # Target
@@ -961,7 +936,32 @@ def rivercore_merge(verbosity, db_folders, output, config_file):
         except:
             logger.info("Couldn't open the browser")
 
-
+#Helper function for parallel processing
+#Returns success,test,attr['result'],attr['log'],attr['numinstr']
+def logcomparison(item):
+    test, attr = item
+    test_wd = attr['work_dir']
+    is_self_checking = attr['self_checking']
+    if not is_self_checking:
+        if not os.path.isfile(test_wd + '/dut.dump'):
+            logger.error(f'{test:<30} : DUT dump is missing')
+            return False, test, 'Unavailable', "DUT dump is missing", None
+        if not os.path.isfile(test_wd + '/ref.dump'):
+            logger.error(f'{test:<30} : REF dump is missing')
+            return False, test, 'Unavailable', 'REF dump is missing', None
+        result, log, insnsize = utils.compare_dumps(test_wd + '/dut.dump', test_wd + '/ref.dump')
+    else:
+        if not os.path.isfile(test_wd + '/dut.signature'):
+            logger.error(f'{test:<30} : DUT signature is missing')
+            return False, test, 'Unavailable',"DUT signature is missing", None
+        result, log = utils.self_check(test_wd + '/dut.signature')
+        insnsize = utils.get_file_size(test_wd + '/dut.dump')
+    if result == 'Passed':
+        logger.info(f"{test:<30} : TEST {result.upper()}")
+        return True, test, result, log, insnsize
+    else:
+        logger.error(f"{test:<30} : TEST {result.upper()}")
+        return False, test, result, log, insnsize
 def rivercore_setup(config, dut, gen, ref, verbosity):
     '''
         Function to generate sample plugins 
@@ -1123,3 +1123,4 @@ def rivercore_setup(config, dut, gen, ref, verbosity):
 
         logger.info(
             'Created {0} Plugin in the current working directory'.format(ref))
+
diff --git a/setup.cfg b/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.6.0
+current_version = 1.7.0
 commit = True
 tag = True
 

diff --git a/setup.py b/setup.py
@@ -62,6 +62,6 @@ def read_requires():
     tests_require=test_requirements,
     url=
     'https://github.com/incoresemi/river_core',
-    version='1.6.0',
+    version='1.7.0',
     zip_safe=False,
 )