Improve pref tests logging info, group duplicate codes to common.py (a…

…ws#6336) - Avoid Error message when the application to be used is not found, convert it to a INFO - Improve outcome when difference is 0 - Group duplicate codes to common.py
hanwen-pcluste · Jul 8, 2024 · e8d25cf · e8d25cf
1 parent 60184ba
commit e8d25cf
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 46 deletions.
diff --git a/tests/integration-tests/tests/performance_tests/common.py b/tests/integration-tests/tests/performance_tests/common.py
@@ -27,6 +27,7 @@
 PYTEST_PARAMETERIZE_VALUES = [(NUM_COMPUTE_NODES, 1)]
 TEST_RUNNER_SCRIPT = "/shared/assets/workloads/scale-test/run-scale-test.sh"
 ROUND_UP_FACTOR = 100_000_000
+PERF_TEST_DIFFERENCE_TOLERANCE = 3
 
 METRICS = [
     dict(name="jobRunTime", unit="ms"),
@@ -222,3 +223,29 @@ def write_results_to_output_dir(
         paths["baseline"]["statistics.json"],
         paths[candidate_configuration]["statistics.json"],
     )
+
+
+def perf_test_difference(observed_value, baseline_value):
+    percentage_difference = 100 * (observed_value - baseline_value) / baseline_value
+    return percentage_difference
+
+
+def _log_output_performance_difference(node, performance_degradation, observed_value, baseline_value):
+    percentage_difference = perf_test_difference(observed_value, baseline_value)
+    if percentage_difference < 0:
+        outcome = "improvement"
+    elif percentage_difference == 0:
+        outcome = "matching baseline"
+    elif percentage_difference <= PERF_TEST_DIFFERENCE_TOLERANCE:
+        outcome = "degradation (within tolerance)"
+    else:
+        outcome = "degradation (above tolerance)"
+        performance_degradation[node] = {
+            "baseline": baseline_value,
+            "observed": observed_value,
+            "percentage_difference": percentage_difference,
+        }
+    logging.info(
+        f"Nodes: {node}, Baseline: {baseline_value} seconds, Observed: {observed_value} seconds, "
+        f"Percentage difference: {percentage_difference}%, Outcome: {outcome}"
+    )
diff --git a/tests/integration-tests/tests/performance_tests/test_openfoam.py b/tests/integration-tests/tests/performance_tests/test_openfoam.py
@@ -4,6 +4,8 @@
 import pytest
 from remote_command_executor import RemoteCommandExecutionError, RemoteCommandExecutor
 
+from tests.performance_tests.common import _log_output_performance_difference
+
 # timeout in seconds
 OPENFOAM_INSTALLATION_TIMEOUT = 300
 OPENFOAM_JOB_TIMEOUT = 5400  # Takes long time because during the first time, it's not only execute the job but also
@@ -17,20 +19,15 @@
     "rhel8": {8: 742, 16: 376, 32: 185},  # v3.6.0 just a placeholder, RHEL8 not supported
     "rocky8": {8: 742, 16: 376, 32: 185},  # v3.8.0 just a placeholder, Rocky8 not supported
 }
-PERF_TEST_DIFFERENCE_TOLERANCE = 3
-
-
-def perf_test_difference(observed_value, baseline_value):
-    percentage_difference = 100 * (observed_value - baseline_value) / baseline_value
-    return percentage_difference
 
 
 def openfoam_installed(headnode):
     cmd = '[ -d "/shared/SubspaceBenchmarks" ]'
     try:
-        headnode.run_remote_command(cmd)
+        headnode.run_remote_command(cmd, log_error=False)
         return True
     except RemoteCommandExecutionError:
+        logging.info("OpenFOAM is not installed on the head node.")
         return False
 
 
@@ -94,22 +91,7 @@ def test_openfoam(
     # Check results and log performance degradation
     for node, observed_value in zip(number_of_nodes, [observed_value_8, observed_value_16, observed_value_32]):
         baseline_value = BASELINE_CLUSTER_SIZE_ELAPSED_SECONDS[os][node]
-        percentage_difference = perf_test_difference(observed_value, baseline_value)
-        if percentage_difference < 0:
-            outcome = "improvement"
-        elif percentage_difference <= PERF_TEST_DIFFERENCE_TOLERANCE:
-            outcome = "degradation (within tolerance)"
-        else:
-            outcome = "degradation (above tolerance)"
-            performance_degradation[node] = {
-                "baseline": baseline_value,
-                "observed": observed_value,
-                "percentage_difference": percentage_difference,
-            }
-        logging.info(
-            f"Nodes: {node}, Baseline: {baseline_value} seconds, Observed: {observed_value} seconds, "
-            f"Percentage difference: {percentage_difference}%, Outcome: {outcome}"
-        )
+        _log_output_performance_difference(node, performance_degradation, observed_value, baseline_value)
 
     if performance_degradation:
         pytest.fail(f"Performance degradation detected: {performance_degradation}")

diff --git a/tests/integration-tests/tests/performance_tests/test_starccm.py b/tests/integration-tests/tests/performance_tests/test_starccm.py
@@ -6,6 +6,7 @@
 from remote_command_executor import RemoteCommandExecutionError, RemoteCommandExecutor
 
 from tests.common.utils import assert_no_file_handler_leak, get_compute_ip_to_num_files
+from tests.performance_tests.common import _log_output_performance_difference
 
 # timeout in seconds
 STARCCM_INSTALLATION_TIMEOUT = 1800
@@ -21,7 +22,6 @@
     "rhel8": {8: 66.494, 16: 36.154, 32: 20.347},  # v3.6.0
     "rocky8": {8: 66.859, 16: 36.184, 32: 21.090},  # v3.8.0
 }
-PERF_TEST_DIFFERENCE_TOLERANCE = 3
 
 OSS_REQUIRING_EXTRA_DEPS = ["alinux2023", "rhel8", "rocky8"]
 
@@ -33,17 +33,13 @@ def get_starccm_secrets(region_name):
     return secrets["podkey"], secrets["licpath"]
 
 
-def perf_test_difference(observed_value, baseline_value):
-    percentage_difference = 100 * (observed_value - baseline_value) / baseline_value
-    return percentage_difference
-
-
 def starccm_installed(headnode):
     cmd = "/shared/STAR-CCM+/18.02.008/STAR-CCM+18.02.008/star/bin/starccm+ --version"
     try:
-        headnode.run_remote_command(cmd)
+        headnode.run_remote_command(cmd, log_error=False)
         return True
     except RemoteCommandExecutionError:
+        logging.info("STAR-CCM+ is not installed on the head node.")
         return False
 
 
@@ -131,22 +127,7 @@ def test_starccm(
     # Check results and log performance degradation
     for node, observed_value in zip(number_of_nodes, [observed_value_8, observed_value_16, observed_value_32]):
         baseline_value = BASELINE_CLUSTER_SIZE_ELAPSED_SECONDS[os][node]
-        percentage_difference = perf_test_difference(observed_value, baseline_value)
-        if percentage_difference < 0:
-            outcome = "improvement"
-        elif percentage_difference <= PERF_TEST_DIFFERENCE_TOLERANCE:
-            outcome = "degradation (within tolerance)"
-        else:
-            outcome = "degradation (above tolerance)"
-            performance_degradation[node] = {
-                "baseline": baseline_value,
-                "observed": observed_value,
-                "percentage_difference": percentage_difference,
-            }
-        logging.info(
-            f"Nodes: {node}, Baseline: {baseline_value} seconds, Observed: {observed_value} seconds, "
-            f"Percentage difference: {percentage_difference}%, Outcome: {outcome}"
-        )
+        _log_output_performance_difference(node, performance_degradation, observed_value, baseline_value)
 
     assert_no_file_handler_leak(init_num_files, remote_command_executor, scheduler_commands)