Run different simulator threads in different directories.

This avoids interference between different runner.cfg files.

tests/unit/test_incisive_interface.py

@@ -503,9 +503,9 @@ def test_simulate_vhdl(self, run_command, find_cds_root_irun, find_cds_root_virt
             simif.compile_project(project)
 
         config = make_config()
-        self.assertTrue(simif.simulate("suite_output_path", "test_suite_name", config))
-        elaborate_args_file = str(Path("suite_output_path") / simif.name / "irun_elaborate.args")
-        simulate_args_file = str(Path("suite_output_path") / simif.name / "irun_simulate.args")
+        self.assertTrue(simif.simulate("suite_output_path", "simulator_output_path", "test_suite_name", config))
+        elaborate_args_file = str(Path("simulator_output_path") / "irun_elaborate.args")
+        simulate_args_file = str(Path("simulator_output_path") / "irun_simulate.args")
         run_command.assert_has_calls(
             [
                 mock.call(
@@ -541,7 +541,7 @@ def test_simulate_vhdl(self, run_command, find_cds_root_irun, find_cds_root_virt
                 "-work work",
                 '-nclibdirname "%s"' % str(Path(self.output_path) / "libraries"),
                 '-cdslib "%s"' % str(Path(self.output_path) / "cds.lib"),
-                '-log "%s"' % str(Path("suite_output_path") / simif.name / "irun_elaborate.log"),
+                '-log "%s"' % str(Path("simulator_output_path") / "irun_elaborate.log"),
                 "-quiet",
                 '-reflib "lib_path"',
                 "-access +r",
@@ -565,7 +565,7 @@ def test_simulate_vhdl(self, run_command, find_cds_root_irun, find_cds_root_virt
                 "-work work",
                 '-nclibdirname "%s"' % str(Path(self.output_path) / "libraries"),
                 '-cdslib "%s"' % str(Path(self.output_path) / "cds.lib"),
-                '-log "%s"' % str(Path("suite_output_path") / simif.name / "irun_simulate.log"),
+                '-log "%s"' % str(Path("simulator_output_path") / "irun_simulate.log"),
                 "-quiet",
                 '-reflib "lib_path"',
                 "-access +r",
@@ -591,9 +591,9 @@ def test_simulate_verilog(self, run_command, find_cds_root_irun, find_cds_root_v
             simif.compile_project(project)
 
         config = make_config(verilog=True)
-        self.assertTrue(simif.simulate("suite_output_path", "test_suite_name", config))
-        elaborate_args_file = str(Path("suite_output_path") / simif.name / "irun_elaborate.args")
-        simulate_args_file = str(Path("suite_output_path") / simif.name / "irun_simulate.args")
+        self.assertTrue(simif.simulate("suite_output_path", "simulator_output_path", "test_suite_name", config))
+        elaborate_args_file = str(Path("simulator_output_path") / "irun_elaborate.args")
+        simulate_args_file = str(Path("simulator_output_path") / "irun_simulate.args")
         run_command.assert_has_calls(
             [
                 mock.call(
@@ -629,7 +629,7 @@ def test_simulate_verilog(self, run_command, find_cds_root_irun, find_cds_root_v
                 "-work work",
                 '-nclibdirname "%s"' % str(Path(self.output_path) / "libraries"),
                 '-cdslib "%s"' % str(Path(self.output_path) / "cds.lib"),
-                '-log "%s"' % str(Path("suite_output_path") / simif.name / "irun_elaborate.log"),
+                '-log "%s"' % str(Path("simulator_output_path") / "irun_elaborate.log"),
                 "-quiet",
                 '-reflib "lib_path"',
                 "-access +r",
@@ -653,7 +653,7 @@ def test_simulate_verilog(self, run_command, find_cds_root_irun, find_cds_root_v
                 "-work work",
                 '-nclibdirname "%s"' % str(Path(self.output_path) / "libraries"),
                 '-cdslib "%s"' % str(Path(self.output_path) / "cds.lib"),
-                '-log "%s"' % str(Path("suite_output_path") / simif.name / "irun_simulate.log"),
+                '-log "%s"' % str(Path("simulator_output_path") / "irun_simulate.log"),
                 "-quiet",
                 '-reflib "lib_path"',
                 "-access +r",
@@ -670,9 +670,9 @@ def test_simulate_extra_flags(self, run_command, find_cds_root_irun, find_cds_ro
         find_cds_root_virtuoso.return_value = None
         simif = IncisiveInterface(prefix="prefix", output_path=self.output_path)
         config = make_config(sim_options={"incisive.irun_sim_flags": ["custom", "flags"]})
-        self.assertTrue(simif.simulate("suite_output_path", "test_suite_name", config))
-        elaborate_args_file = str(Path("suite_output_path") / simif.name / "irun_elaborate.args")
-        simulate_args_file = str(Path("suite_output_path") / simif.name / "irun_simulate.args")
+        self.assertTrue(simif.simulate("suite_output_path", "simulator_output_path", "test_suite_name", config))
+        elaborate_args_file = str(Path("simulator_output_path") / "irun_elaborate.args")
+        simulate_args_file = str(Path("simulator_output_path") / "irun_simulate.args")
         run_command.assert_has_calls(
             [
                 mock.call(
@@ -708,9 +708,9 @@ def test_simulate_generics_and_parameters(self, run_command, find_cds_root_irun,
         find_cds_root_virtuoso.return_value = None
         simif = IncisiveInterface(prefix="prefix", output_path=self.output_path)
         config = make_config(verilog=True, generics={"genstr": "genval", "genint": 1, "genbool": True})
-        self.assertTrue(simif.simulate("suite_output_path", "test_suite_name", config))
-        elaborate_args_file = str(Path("suite_output_path") / simif.name / "irun_elaborate.args")
-        simulate_args_file = str(Path("suite_output_path") / simif.name / "irun_simulate.args")
+        self.assertTrue(simif.simulate("suite_output_path", "simulator_output_path", "test_suite_name", config))
+        elaborate_args_file = str(Path("simulator_output_path") / "irun_elaborate.args")
+        simulate_args_file = str(Path("simulator_output_path") / "irun_simulate.args")
         run_command.assert_has_calls(
             [
                 mock.call(
@@ -744,9 +744,9 @@ def test_simulate_hdlvar(self, run_command, find_cds_root_irun, find_cds_root_vi
         find_cds_root_virtuoso.return_value = None
         simif = IncisiveInterface(prefix="prefix", output_path=self.output_path, hdlvar="custom_hdlvar")
         config = make_config()
-        self.assertTrue(simif.simulate("suite_output_path", "test_suite_name", config))
-        elaborate_args_file = str(Path("suite_output_path") / simif.name / "irun_elaborate.args")
-        simulate_args_file = str(Path("suite_output_path") / simif.name / "irun_simulate.args")
+        self.assertTrue(simif.simulate("suite_output_path", "simulator_output_path", "test_suite_name", config))
+        elaborate_args_file = str(Path("simulator_output_path") / "irun_elaborate.args")
+        simulate_args_file = str(Path("simulator_output_path") / "irun_simulate.args")
         run_command.assert_has_calls(
             [
                 mock.call(
@@ -778,8 +778,10 @@ def test_elaborate(self, run_command, find_cds_root_irun, find_cds_root_virtuoso
         find_cds_root_virtuoso.return_value = None
         simif = IncisiveInterface(prefix="prefix", output_path=self.output_path)
         config = make_config(verilog=True)
-        self.assertTrue(simif.simulate("suite_output_path", "test_suite_name", config, elaborate_only=True))
-        elaborate_args_file = str(Path("suite_output_path") / simif.name / "irun_elaborate.args")
+        self.assertTrue(
+            simif.simulate("suite_output_path", "simulator_output_path", "test_suite_name", config, elaborate_only=True)
+        )
+        elaborate_args_file = str(Path("simulator_output_path") / "irun_elaborate.args")
         run_command.assert_has_calls(
             [
                 mock.call(
@@ -810,7 +812,7 @@ def test_elaborate(self, run_command, find_cds_root_irun, find_cds_root_virtuoso
                 "-work work",
                 '-nclibdirname "%s"' % str(Path(self.output_path) / "libraries"),
                 '-cdslib "%s"' % str(Path(self.output_path) / "cds.lib"),
-                '-log "%s"' % str(Path("suite_output_path") / simif.name / "irun_elaborate.log"),
+                '-log "%s"' % str(Path("simulator_output_path") / "irun_elaborate.log"),
                 "-quiet",
                 "-access +r",
                 '-input "@run"',
@@ -826,8 +828,8 @@ def test_elaborate_fail(self, run_command, find_cds_root_irun, find_cds_root_vir
         find_cds_root_virtuoso.return_value = None
         simif = IncisiveInterface(prefix="prefix", output_path=self.output_path)
         config = make_config()
-        self.assertFalse(simif.simulate("suite_output_path", "test_suite_name", config))
-        elaborate_args_file = str(Path("suite_output_path") / simif.name / "irun_elaborate.args")
+        self.assertFalse(simif.simulate("suite_output_path", "simulator_output_path", "test_suite_name", config))
+        elaborate_args_file = str(Path("simulator_output_path") / "irun_elaborate.args")
         run_command.assert_has_calls(
             [
                 mock.call(
@@ -850,9 +852,9 @@ def test_simulate_fail(self, run_command, find_cds_root_irun, find_cds_root_virt
         find_cds_root_virtuoso.return_value = None
         simif = IncisiveInterface(prefix="prefix", output_path=self.output_path)
         config = make_config()
-        self.assertFalse(simif.simulate("suite_output_path", "test_suite_name", config))
-        elaborate_args_file = str(Path("suite_output_path") / simif.name / "irun_elaborate.args")
-        simulate_args_file = str(Path("suite_output_path") / simif.name / "irun_simulate.args")
+        self.assertFalse(simif.simulate("suite_output_path", "simulator_output_path", "test_suite_name", config))
+        elaborate_args_file = str(Path("simulator_output_path") / "irun_elaborate.args")
+        simulate_args_file = str(Path("simulator_output_path") / "irun_simulate.args")
         run_command.assert_has_calls(
             [
                 mock.call(
@@ -888,9 +890,9 @@ def test_simulate_gui(self, run_command, find_cds_root_irun, find_cds_root_virtu
         with mock.patch("vunit.sim_if.check_output", autospec=True, return_value="") as dummy:
             simif.compile_project(project)
         config = make_config()
-        self.assertTrue(simif.simulate("suite_output_path", "test_suite_name", config))
-        elaborate_args_file = str(Path("suite_output_path") / simif.name / "irun_elaborate.args")
-        simulate_args_file = str(Path("suite_output_path") / simif.name / "irun_simulate.args")
+        self.assertTrue(simif.simulate("suite_output_path", "simulator_output_path", "test_suite_name", config))
+        elaborate_args_file = str(Path("simulator_output_path") / "irun_elaborate.args")
+        simulate_args_file = str(Path("simulator_output_path") / "irun_simulate.args")
         run_command.assert_has_calls(
             [
                 mock.call(
@@ -925,7 +927,7 @@ def test_simulate_gui(self, run_command, find_cds_root_irun, find_cds_root_virtu
                 "-work work",
                 '-nclibdirname "%s"' % str(Path(self.output_path) / "libraries"),
                 '-cdslib "%s"' % str(Path(self.output_path) / "cds.lib"),
-                '-log "%s"' % str(Path("suite_output_path") / simif.name / "irun_elaborate.log"),
+                '-log "%s"' % str(Path("simulator_output_path") / "irun_elaborate.log"),
                 "-quiet",
                 '-reflib "lib_path"',
                 "-access +rwc",
@@ -949,7 +951,7 @@ def test_simulate_gui(self, run_command, find_cds_root_irun, find_cds_root_virtu
                 "-work work",
                 '-nclibdirname "%s"' % str(Path(self.output_path) / "libraries"),
                 '-cdslib "%s"' % str(Path(self.output_path) / "cds.lib"),
-                '-log "%s"' % str(Path("suite_output_path") / simif.name / "irun_simulate.log"),
+                '-log "%s"' % str(Path("simulator_output_path") / "irun_simulate.log"),
                 "-quiet",
                 '-reflib "lib_path"',
                 "-access +rwc",

tests/unit/test_test_runner.py

@@ -11,6 +11,7 @@
 from pathlib import Path
 import unittest
 from unittest import mock
+from time import sleep
 from tests.common import with_tempdir
 from vunit.hashing import hash_string
 from vunit.test.runner import TestRunner
@@ -44,6 +45,27 @@ def test_runs_testcases_in_order(self, tempdir):
         self.assertTrue(report.result_of("test1").passed)
         self.assertTrue(report.result_of("test2").failed)
         self.assertTrue(report.result_of("test3").passed)
+        self.assertEqual(test_case1.thread_id, 0)
+        self.assertEqual(test_case2.thread_id, 0)
+        self.assertEqual(test_case3.thread_id, 0)
+
+    @with_tempdir
+    def test_runs_testcases_in_multiple_threads(self, tempdir):
+        report = TestReport()
+        runner = TestRunner(report, tempdir, num_threads=3)
+
+        test_cases = [self.create_test(f"test{n}", True, delay=n / 2) for n in range(1, 6)]
+        test_list = TestList()
+        for test_case in test_cases:
+            test_list.add_test(test_case)
+
+        runner.run(test_list)
+
+        self.assertListEqual(
+            [test_case.thread_id for test_case in test_cases],
+            [1, 2, 0, 1, 2],
+            str([(test_case.name, test_case.thread_id) for test_case in test_cases]),
+        )
 
     @with_tempdir
     def test_fail_fast(self, tempdir):
@@ -68,6 +90,9 @@ def test_fail_fast(self, tempdir):
         self.assertEqual(order, ["test1", "test2"])
         self.assertTrue(report.result_of("test1").passed)
         self.assertTrue(report.result_of("test2").failed)
+        self.assertEqual(test_case1.thread_id, 0)
+        self.assertEqual(test_case2.thread_id, 0)
+        self.assertEqual(test_case3.thread_id, None)
 
     @with_tempdir
     def test_handles_python_exeception(self, tempdir):
@@ -196,7 +221,7 @@ def test_get_output_path_on_windows(self):
                 )
 
     @staticmethod
-    def create_test(name, passed, order=None):
+    def create_test(name, passed, order=None, delay=None):
         """
         Utility function to create a mocked test with name
         that is either passed or failed
@@ -208,6 +233,8 @@ def run_side_effect(*args, **kwargs):  # pylint: disable=unused-argument
             """
             if order is not None:
                 order.append(name)
+            if delay is not None:
+                sleep(delay)
             return passed
 
         test_case = TestCaseMock(name=name, run_side_effect=run_side_effect)
@@ -223,15 +250,17 @@ def __init__(self, name, run_side_effect):
         self.name = name
         self.output_path = None
         self.read_output = None
+        self.thread_id = None
         self.called = False
         self.run_side_effect = run_side_effect
 
-    def run(self, output_path, read_output):
+    def run(self, output_path, read_output, thread_id):
         """
         Mock run method that just records the arguments
         """
         assert not self.called
         self.called = True
         self.output_path = output_path
         self.read_output = read_output
+        self.thread_id = thread_id
         return self.run_side_effect(output_path=output_path, read_output=read_output)

tests/unit/test_test_suites.py

@@ -194,37 +194,47 @@ def test_runner_cfg_location(self):
         with create_tempdir() as tempdir:
             design_unit = Entity("tb_entity", file_name=str(Path(tempdir) / "file.vhd"))
             design_unit.generic_names = ["runner_cfg"]
-            output_path = str(Path(__file__).parent / "sim_out")
-            renew_path(output_path)
+            vunit_output_path = str(Path(tempdir) / "vunit_out")
+            renew_path(vunit_output_path)
+            test_output_path = str(Path(vunit_output_path) / "test_out")
+            simulator_root = str(Path(vunit_output_path) / "my_simulator")
 
             class TestSimIf(SimulatorInterface):
-                def __init__(self, output_path, gui, expect_runner_cfg_generic):
+                def __init__(self, output_path, gui, expect_runner_cfg_generic, thread_id):
                     super().__init__(output_path, gui)
                     self._expect_runner_cfg_generic = expect_runner_cfg_generic
+                    self._thread_id = thread_id
 
-                def simulate(self, output_path, test_suite_name, config, elaborate_only):
+                def simulate(self, output_path, simulator_output_path, test_suite_name, config, elaborate_only):
                     tc = TestCase()
+                    tc.assertEqual(Path(output_path), Path(test_output_path))
+                    tc.assertEqual(Path(simulator_output_path), Path(simulator_root) / str(self._thread_id))
                     if self._expect_runner_cfg_generic:
                         tc.assertIn("runner_cfg", config.generics)
-                        tc.assertFalse((Path(output_path) / "test_sim" / "runner.cfg").exists())
+                        tc.assertFalse((Path(simulator_output_path) / "runner.cfg").exists())
                     else:
                         tc.assertNotIn("runner_cfg", config.generics)
-                        tc.assertTrue((Path(output_path) / "test_sim" / "runner.cfg").exists())
-
-                def get_simulator_output_path(self, output_path):
-                    return Path(output_path) / "test_sim"
+                        tc.assertTrue((Path(simulator_output_path) / "runner.cfg").exists())
 
             for expect_runner_cfg_generic in [False, True]:
-                config = Configuration(
-                    "name", design_unit, vhdl_config_name=None if expect_runner_cfg_generic else "cfg"
-                )
-                sim_if = TestSimIf(output_path, gui=False, expect_runner_cfg_generic=expect_runner_cfg_generic)
-
-                run = TestRun(
-                    simulator_if=sim_if,
-                    config=config,
-                    elaborate_only=False,
-                    test_suite_name=None,
-                    test_cases=["foo"],
-                )
-                run._simulate(output_path)  # pylint: disable=protected-access
+                for thread_id in range(2):
+                    renew_path(test_output_path)
+                    renew_path(simulator_root)
+                    config = Configuration(
+                        "name", design_unit, vhdl_config_name=None if expect_runner_cfg_generic else "cfg"
+                    )
+                    sim_if = TestSimIf(
+                        simulator_root,
+                        gui=False,
+                        expect_runner_cfg_generic=expect_runner_cfg_generic,
+                        thread_id=thread_id,
+                    )
+
+                    test_run = TestRun(
+                        simulator_if=sim_if,
+                        config=config,
+                        elaborate_only=False,
+                        test_suite_name=None,
+                        test_cases=["foo"],
+                    )
+                    test_run.run(test_output_path, thread_id=thread_id, read_output=None)