From 0f192f0ef36303b56e2950961fada5957d2051d5 Mon Sep 17 00:00:00 2001 From: AdrickTench Date: Fri, 23 Aug 2024 10:41:33 -0500 Subject: [PATCH 01/18] have ci.yml & run_commit_tests.sh install mettalog and run tests --- .github/workflows/ci.yml | 4 +++- scripts/run_commit_tests.sh | 16 +++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b70a5e2f36a..41d49fdab3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,8 +26,10 @@ jobs: with: python-version: '3.x' - - name: Install junit2html + - name: Install python packages run: | + pip install ansi2html + pip install hyperon pip install junit2html - name: Make Shell Script Executable diff --git a/scripts/run_commit_tests.sh b/scripts/run_commit_tests.sh index ae0e0a22c9f..2945d9de2f0 100755 --- a/scripts/run_commit_tests.sh +++ b/scripts/run_commit_tests.sh @@ -1,8 +1,14 @@ #!/bin/bash +set -e # stop if any step fails -# This script generates the input file used by the Python script. -# Replace the following lines with the actual commands to generate the input file. +# install mettalog +chmod +x INSTALL.sh # Make sure the script is executable +. ./INSTALL.sh --easy -#echo "| ANTI-REGRESSION.BC-COMP.01 | PASS |(https://example.com/test-report) | (assertEqualToResult (add-atom &kb (: axiom (nums 2 3)))) | (()) | (()) |" > /tmp/SHARED.UNITS -cat ./reports/SHARED.UNITS.PREV.md > /tmp/SHARED.UNITS -# You can add more lines or commands to generate additional input data +# generate the output directory with timestamp +timestamp=$(date +"%Y-%m-%dT%H:%M:%S") +output=reports/tests_output/baseline-compat-$timestamp/ + +# run the tests +echo Running baseline_compat tests to $output +mettalog --output=$output tests/baseline_compat > /dev/null 2>&1 From ecd8316af0cf906d5a50cc0b75f988d6d663d7e6 Mon Sep 17 00:00:00 2001 From: AdrickTench Date: Fri, 23 Aug 2024 11:03:56 -0500 Subject: [PATCH 02/18] separate install and run tests step, show test output (for now) --- .github/workflows/ci.yml | 10 ++++++++-- scripts/run_commit_tests.sh | 7 +------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41d49fdab3b..1e92d61c23a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,10 +32,16 @@ jobs: pip install hyperon pip install junit2html - - name: Make Shell Script Executable + - name: Make Install Script Executable + run: chmod +x INSTALL.sh + + - name: Run Install Script to install Mettalog + run: . ./INSTALL.sh --easy + + - name: Make Test Script Executable run: chmod +x scripts/run_commit_tests.sh - - name: Run Shell Script to Generate Input File + - name: Run Test Script to Generate Input File continue-on-error: true run: | ./scripts/run_commit_tests.sh diff --git a/scripts/run_commit_tests.sh b/scripts/run_commit_tests.sh index 2945d9de2f0..43b61bf5930 100755 --- a/scripts/run_commit_tests.sh +++ b/scripts/run_commit_tests.sh @@ -1,9 +1,4 @@ #!/bin/bash -set -e # stop if any step fails - -# install mettalog -chmod +x INSTALL.sh # Make sure the script is executable -. ./INSTALL.sh --easy # generate the output directory with timestamp timestamp=$(date +"%Y-%m-%dT%H:%M:%S") @@ -11,4 +6,4 @@ output=reports/tests_output/baseline-compat-$timestamp/ # run the tests echo Running baseline_compat tests to $output -mettalog --output=$output tests/baseline_compat > /dev/null 2>&1 +mettalog --output=$output tests/baseline_compat #> /dev/null 2>&1 From cde5a73ec680b64be0ce49a0b15b011d89de4d80 Mon Sep 17 00:00:00 2001 From: AdrickTench Date: Fri, 23 Aug 2024 11:11:41 -0500 Subject: [PATCH 03/18] try running tests with venv --- scripts/run_commit_tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/run_commit_tests.sh b/scripts/run_commit_tests.sh index 43b61bf5930..54ec025e754 100755 --- a/scripts/run_commit_tests.sh +++ b/scripts/run_commit_tests.sh @@ -5,5 +5,6 @@ timestamp=$(date +"%Y-%m-%dT%H:%M:%S") output=reports/tests_output/baseline-compat-$timestamp/ # run the tests +. ./scripts/ensure_venv echo Running baseline_compat tests to $output mettalog --output=$output tests/baseline_compat #> /dev/null 2>&1 From 6cc93fde03aa20c756e8ade3fe79677afccfef53 Mon Sep 17 00:00:00 2001 From: AdrickTench Date: Fri, 23 Aug 2024 11:26:33 -0500 Subject: [PATCH 04/18] store /home/atench/.vscode-server/bin/fee1edb8d6d72a0ddff41e5f71a671c23ed924b9/bin/remote-cli:/home/atench/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/usr/lib/wsl/lib:/mnt/c/windows/system32:/mnt/c/windows:/mnt/c/windows/System32/Wbem:/mnt/c/windows/System32/WindowsPowerShell/v1.0/:/mnt/c/windows/System32/OpenSSH/:/mnt/c/Program Files (x86)/NVIDIA Corporation/PhysX/Common:/mnt/c/WINDOWS/system32:/mnt/c/WINDOWS:/mnt/c/WINDOWS/System32/Wbem:/mnt/c/WINDOWS/System32/WindowsPowerShell/v1.0/:/mnt/c/WINDOWS/System32/OpenSSH/:/mnt/c/Program Files/NVIDIA Corporation/NVIDIA NvDLISR:/mnt/c/Program Files/dotnet/:/mnt/c/Program Files/Git/cmd:/mnt/c/Program Files/HP/HP One Agent:/mnt/c/Users/adric/AppData/Local/Microsoft/WindowsApps:/mnt/c/Users/adric/AppData/Local/Programs/Microsoft VS Code/bin:/snap/bin:/home/atench/.local/bin:/home/atench/mettalog/metta-wam to after mettalog install --- .github/workflows/ci.yml | 4 +++- scripts/run_commit_tests.sh | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e92d61c23a..28b72599974 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,9 @@ jobs: run: chmod +x INSTALL.sh - name: Run Install Script to install Mettalog - run: . ./INSTALL.sh --easy + run: | + . ./INSTALL.sh --easy + echo $PATH >> $GITHUB_PATH - name: Make Test Script Executable run: chmod +x scripts/run_commit_tests.sh diff --git a/scripts/run_commit_tests.sh b/scripts/run_commit_tests.sh index 54ec025e754..43b61bf5930 100755 --- a/scripts/run_commit_tests.sh +++ b/scripts/run_commit_tests.sh @@ -5,6 +5,5 @@ timestamp=$(date +"%Y-%m-%dT%H:%M:%S") output=reports/tests_output/baseline-compat-$timestamp/ # run the tests -. ./scripts/ensure_venv echo Running baseline_compat tests to $output mettalog --output=$output tests/baseline_compat #> /dev/null 2>&1 From 857228eb078abae051656ac1d8bfe15a07571360 Mon Sep 17 00:00:00 2001 From: AdrickTench Date: Fri, 23 Aug 2024 11:49:07 -0500 Subject: [PATCH 05/18] set xterm-256color and only run subset of tests --- .github/workflows/ci.yml | 1 + scripts/run_commit_tests.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28b72599974..1e6caa44f52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,7 @@ jobs: - name: Run Test Script to Generate Input File continue-on-error: true run: | + TERM=xterm-256color ./scripts/run_commit_tests.sh - name: Run JUnit Report Generation Script diff --git a/scripts/run_commit_tests.sh b/scripts/run_commit_tests.sh index 43b61bf5930..7390ff29064 100755 --- a/scripts/run_commit_tests.sh +++ b/scripts/run_commit_tests.sh @@ -6,4 +6,4 @@ output=reports/tests_output/baseline-compat-$timestamp/ # run the tests echo Running baseline_compat tests to $output -mettalog --output=$output tests/baseline_compat #> /dev/null 2>&1 +mettalog --output=$output tests/baseline_compat/anti-regression #> /dev/null 2>&1 From b2252798e1b6b8f395ca2378c532db505209567c Mon Sep 17 00:00:00 2001 From: AdrickTench Date: Fri, 23 Aug 2024 11:58:02 -0500 Subject: [PATCH 06/18] set xterm-256color in yml --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e6caa44f52..b269a406d56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,8 +46,9 @@ jobs: - name: Run Test Script to Generate Input File continue-on-error: true run: | - TERM=xterm-256color ./scripts/run_commit_tests.sh + env: + TERM: xterm-256color - name: Run JUnit Report Generation Script continue-on-error: true From 5ed9a5497a9e0caf8068fc8dbfaf4ec09ccc99da Mon Sep 17 00:00:00 2001 From: AdrickTench Date: Fri, 23 Aug 2024 12:26:14 -0500 Subject: [PATCH 07/18] print SHARED.UNITS --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b269a406d56..d4e465b17b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,7 @@ jobs: continue-on-error: true run: | ./scripts/run_commit_tests.sh + cat /tmp/SHARED.UNITS env: TERM: xterm-256color From 9807c5b2d56f7976c64c228c67af0aebefb2c2a5 Mon Sep 17 00:00:00 2001 From: AdrickTench Date: Fri, 23 Aug 2024 12:43:36 -0500 Subject: [PATCH 08/18] only run one test file --- .github/workflows/ci.yml | 1 + scripts/run_commit_tests.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4e465b17b0..d0803416598 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,7 @@ jobs: - name: Run JUnit Report Generation Script continue-on-error: true run: | + cat /tmp/SHARED.UNITS python scripts/into_junit.py /tmp/SHARED.UNITS > junit.xml - name: Convert JUnit XML to Standard HTML Report diff --git a/scripts/run_commit_tests.sh b/scripts/run_commit_tests.sh index 7390ff29064..35de3a54c43 100755 --- a/scripts/run_commit_tests.sh +++ b/scripts/run_commit_tests.sh @@ -6,4 +6,4 @@ output=reports/tests_output/baseline-compat-$timestamp/ # run the tests echo Running baseline_compat tests to $output -mettalog --output=$output tests/baseline_compat/anti-regression #> /dev/null 2>&1 +mettalog --output=$output tests/baseline_compat/anti-regression/comma_is_not_special.metta #> /dev/null 2>&1 From b002706c5affd5d59e3edec098b204b78113adc4 Mon Sep 17 00:00:00 2001 From: AdrickTench Date: Fri, 23 Aug 2024 13:47:58 -0500 Subject: [PATCH 09/18] don't run tests for now --- .github/workflows/ci.yml | 1 - scripts/run_commit_tests.sh | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0803416598..0ee3f84b983 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,6 @@ jobs: continue-on-error: true run: | ./scripts/run_commit_tests.sh - cat /tmp/SHARED.UNITS env: TERM: xterm-256color diff --git a/scripts/run_commit_tests.sh b/scripts/run_commit_tests.sh index 35de3a54c43..2be15520ddf 100755 --- a/scripts/run_commit_tests.sh +++ b/scripts/run_commit_tests.sh @@ -6,4 +6,5 @@ output=reports/tests_output/baseline-compat-$timestamp/ # run the tests echo Running baseline_compat tests to $output -mettalog --output=$output tests/baseline_compat/anti-regression/comma_is_not_special.metta #> /dev/null 2>&1 +cat ./reports/SHARED.UNITS.PREV.md > /tmp/SHARED.UNITS +#mettalog --output=$output tests/baseline_compat/anti-regression/comma_is_not_special.metta #> /dev/null 2>&1 From a313f613e931a115c8cf61656a13eb1b46607312 Mon Sep 17 00:00:00 2001 From: AdrickTench Date: Fri, 23 Aug 2024 14:14:29 -0500 Subject: [PATCH 10/18] update auto-approve-action version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ee3f84b983..4d276f798c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -150,7 +150,7 @@ jobs: - name: Auto-Approve the Pull Request if: github.event_name == 'pull_request_target' - uses: hmarr/auto-approve-action@v3 + uses: hmarr/auto-approve-action@v4 with: github-token: ${{ secrets.GITHUB_TOKEN }} From c247f6bb3bc856bc14a97f518f9bb7d8e77a1765 Mon Sep 17 00:00:00 2001 From: "Douglas R. Miles" Date: Fri, 23 Aug 2024 12:30:01 -0700 Subject: [PATCH 11/18] Run 1 test again to see output --- scripts/run_commit_tests.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/run_commit_tests.sh b/scripts/run_commit_tests.sh index 2be15520ddf..1f33cab79a8 100755 --- a/scripts/run_commit_tests.sh +++ b/scripts/run_commit_tests.sh @@ -6,5 +6,6 @@ output=reports/tests_output/baseline-compat-$timestamp/ # run the tests echo Running baseline_compat tests to $output -cat ./reports/SHARED.UNITS.PREV.md > /tmp/SHARED.UNITS -#mettalog --output=$output tests/baseline_compat/anti-regression/comma_is_not_special.metta #> /dev/null 2>&1 +#cat ./reports/SHARED.UNITS.PREV.md > /tmp/SHARED.UNITS +cat /dev/null> /tmp/SHARED.UNITS +mettalog --output=$output tests/baseline_compat/anti-regression/comma_is_not_special.metta From 7b1c87e3f77e0ae96e340f6b08a82a98cae89959 Mon Sep 17 00:00:00 2001 From: "Douglas R. Miles" Date: Fri, 23 Aug 2024 12:52:59 -0700 Subject: [PATCH 12/18] to run one test you have to use --test --- scripts/run_commit_tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/run_commit_tests.sh b/scripts/run_commit_tests.sh index 1f33cab79a8..912c99e3b2b 100755 --- a/scripts/run_commit_tests.sh +++ b/scripts/run_commit_tests.sh @@ -6,6 +6,6 @@ output=reports/tests_output/baseline-compat-$timestamp/ # run the tests echo Running baseline_compat tests to $output -#cat ./reports/SHARED.UNITS.PREV.md > /tmp/SHARED.UNITS -cat /dev/null> /tmp/SHARED.UNITS -mettalog --output=$output tests/baseline_compat/anti-regression/comma_is_not_special.metta +cat ./reports/SHARED.UNITS.PREV.md > /tmp/SHARED.UNITS +#cat /dev/null> /tmp/SHARED.UNITS +#mettalog --output=$output --test --clean tests/baseline_compat/anti-regression/comma_is_not_special.metta From 38fc3b36127e91e564cdd82bf58191770a167449 Mon Sep 17 00:00:00 2001 From: AdrickTench Date: Fri, 23 Aug 2024 15:08:32 -0500 Subject: [PATCH 13/18] make test run timestamp available --- .github/workflows/ci.yml | 5 ++++- scripts/run_commit_tests.sh | 18 +++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d276f798c8..cec2507a341 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,13 +46,16 @@ jobs: - name: Run Test Script to Generate Input File continue-on-error: true run: | - ./scripts/run_commit_tests.sh + TIMESTAMP=$(date +"%Y-%m-%dT%H:%M:%S") + ./scripts/run_commit_tests.sh -t $TIMESTAMP + echo "TIMESTAMP=$(echo $TIMESTAMP)" >> $GITHUB_ENV env: TERM: xterm-256color - name: Run JUnit Report Generation Script continue-on-error: true run: | + echo $TIMESTAMP cat /tmp/SHARED.UNITS python scripts/into_junit.py /tmp/SHARED.UNITS > junit.xml diff --git a/scripts/run_commit_tests.sh b/scripts/run_commit_tests.sh index 912c99e3b2b..b4677f8c50f 100755 --- a/scripts/run_commit_tests.sh +++ b/scripts/run_commit_tests.sh @@ -1,7 +1,23 @@ #!/bin/bash +# parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + -t|--timestamp) + timestamp="$2" + shift # past argument + shift # past value + ;; + *) + # Ignore unknown options + ;; + esac +done + # generate the output directory with timestamp -timestamp=$(date +"%Y-%m-%dT%H:%M:%S") +if [ -z $timestamp ]; then + timestamp=$(date +"%Y-%m-%dT%H:%M:%S") +fi output=reports/tests_output/baseline-compat-$timestamp/ # run the tests From cbe82d8fdaa22fdb1a619ace41a57c28ebc3f90f Mon Sep 17 00:00:00 2001 From: AdrickTench Date: Fri, 23 Aug 2024 15:25:42 -0500 Subject: [PATCH 14/18] include timestamp in junit.xml --- .github/workflows/ci.yml | 4 +--- scripts/into_junit.py | 11 ++++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cec2507a341..7d10f2aea8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,9 +55,7 @@ jobs: - name: Run JUnit Report Generation Script continue-on-error: true run: | - echo $TIMESTAMP - cat /tmp/SHARED.UNITS - python scripts/into_junit.py /tmp/SHARED.UNITS > junit.xml + python scripts/into_junit.py /tmp/SHARED.UNITS $TIMESTAMP > junit.xml - name: Convert JUnit XML to Standard HTML Report continue-on-error: true diff --git a/scripts/into_junit.py b/scripts/into_junit.py index b08a9334a18..e09f10e0fa8 100644 --- a/scripts/into_junit.py +++ b/scripts/into_junit.py @@ -47,8 +47,8 @@ def parse_test_line(line): return testpackage, testname, stdout, full_identifier, got, expected, status, url -def generate_junit_xml(input_file): - testsuites = ET.Element("testsuites") +def generate_junit_xml(input_file, timestamp): + testsuites = ET.Element("testsuites", timestamp=timestamp) packages_dict = defaultdict(list) # Dictionary to group test cases by their testpackage with open(input_file, 'r') as file: @@ -76,10 +76,11 @@ def generate_junit_xml(input_file): return ET.tostring(testsuites, encoding="utf-8", xml_declaration=True).decode("utf-8") if __name__ == "__main__": - if len(sys.argv) != 2: - print("Usage: python scripts/into_junit.py ") + if len(sys.argv) != 3: + print("Usage: python scripts/into_junit.py ") sys.exit(1) input_file = sys.argv[1] - junit_xml = generate_junit_xml(input_file) + timestamp = sys.argv[2] + junit_xml = generate_junit_xml(input_file, timestamp) print(junit_xml) From b6e84de498e1019e032be93f8bc09b2b632ebeca Mon Sep 17 00:00:00 2001 From: AdrickTench Date: Fri, 23 Aug 2024 16:13:58 -0500 Subject: [PATCH 15/18] include test time in junit.xml --- scripts/into_junit.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/scripts/into_junit.py b/scripts/into_junit.py index e09f10e0fa8..2c400f15792 100644 --- a/scripts/into_junit.py +++ b/scripts/into_junit.py @@ -3,9 +3,9 @@ import re from collections import defaultdict -def create_testcase_element(testclass, testname, stdout, identifier, got, expected, status, url): +def create_testcase_element(testclass, testname, stdout, identifier, got, expected, status, url, time): # Create the testcase XML element with the class and test name attributes - testcase = ET.Element("testcase", classname=testclass, name=testname) + testcase = ET.Element("testcase", classname=testclass, name=testname, time=time) test_res = f"Assertion: {stdout}\nExpected: {expected}\nActual: {got}" sys_out_text = f"Test Report\n\n{test_res}\n]]>" @@ -44,11 +44,12 @@ def parse_test_line(line): raise ValueError("Test package or test name is empty after splitting.") except ValueError as e: raise ValueError(f"Identifier does not contain the expected format: {full_identifier}. Error: {str(e)}") + + time = '.01' # bogus time until tests actually note their runtime - return testpackage, testname, stdout, full_identifier, got, expected, status, url + return testpackage, testname, stdout, full_identifier, got, expected, status, url, time def generate_junit_xml(input_file, timestamp): - testsuites = ET.Element("testsuites", timestamp=timestamp) packages_dict = defaultdict(list) # Dictionary to group test cases by their testpackage with open(input_file, 'r') as file: @@ -57,19 +58,26 @@ def generate_junit_xml(input_file, timestamp): if line.startswith("|"): try: parts = re.split(r'\s*\|\s*(?![^()]*\))', line.strip()) - testpackage, testname, stdout, full_identifier, got, expected, status, url = parse_test_line(line) - testcase = create_testcase_element(testpackage, testname, stdout, full_identifier, got, expected, status, url) + testpackage, testname, stdout, full_identifier, got, expected, status, url, time = parse_test_line(line) + testcase = create_testcase_element(testpackage, testname, stdout, full_identifier, got, expected, status, url, time) packages_dict[testpackage].append(testcase) print(f"Processing {testpackage}.{testname}: {status}", file=sys.stderr) except ValueError as e: print(f"Skipping line due to error: {e}\nLine: {line}\nParts: {parts}", file=sys.stderr) # Create a testsuite for each testpackage group + testsuites = ET.Element("testsuites", timestamp=timestamp) + testsuites_time = 0.0 for testpackage, testcases in packages_dict.items(): testsuite = ET.Element("testsuite", name=testpackage) + testsuite_time = 0.0 for testcase in testcases: + testsuite_time += float(testcase.get('time')) testsuite.append(testcase) + testsuites_time += testsuite_time + testsuite.set('time', str(testsuite_time)) testsuites.append(testsuite) + testsuites.set('time', str(testsuites_time)) # Generate the XML tree and return it as a string tree = ET.ElementTree(testsuites) From d91b91a01b332f8223aac6f4720e622a72515d64 Mon Sep 17 00:00:00 2001 From: AdrickTench Date: Fri, 23 Aug 2024 16:52:55 -0500 Subject: [PATCH 16/18] track commit SHA and branch --- .github/workflows/ci.yml | 1 + scripts/generate_allure_environment.py | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 scripts/generate_allure_environment.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d10f2aea8b..ac3a086ee08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -133,6 +133,7 @@ jobs: if [ -f "previous-junit.xml" ]; then cp previous-junit.xml ./allure-results/ fi + python scripts/generate_allure_environment.py ${{ github.sha }} ${{ github.ref_name }} > ./allure-results/environment.properties - name: Generate Allure Report run: | diff --git a/scripts/generate_allure_environment.py b/scripts/generate_allure_environment.py new file mode 100644 index 00000000000..6e7f2809916 --- /dev/null +++ b/scripts/generate_allure_environment.py @@ -0,0 +1,10 @@ +import sys + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage: python scripts/generate_allure_environment.py ") + sys.exit(1) + + commit_SHA = sys.argv[1] + branch = sys.argv[2] + print("COMMIT_SHA = {}\nBRANCH = {}".format(commit_SHA, branch)) \ No newline at end of file From 8703ebd3ef8ec80dd09c55e976bae17642e182a5 Mon Sep 17 00:00:00 2001 From: AdrickTench Date: Fri, 23 Aug 2024 17:40:11 -0500 Subject: [PATCH 17/18] generate executor.json --- .github/workflows/ci.yml | 1 + scripts/generate_allure_executor.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 scripts/generate_allure_executor.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac3a086ee08..a8bffaedc70 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,6 +134,7 @@ jobs: cp previous-junit.xml ./allure-results/ fi python scripts/generate_allure_environment.py ${{ github.sha }} ${{ github.ref_name }} > ./allure-results/environment.properties + python scripts/generate_allure_executor.py ${{ github.server_url }} ${{ github.repository }} ${{ github.run_id }} > ./allure-results/executor.json - name: Generate Allure Report run: | diff --git a/scripts/generate_allure_executor.py b/scripts/generate_allure_executor.py new file mode 100644 index 00000000000..73bf4e63a4b --- /dev/null +++ b/scripts/generate_allure_executor.py @@ -0,0 +1,15 @@ +import sys, json + +if __name__ == "__main__": + if len(sys.argv) != 4: + print("Usage: python scripts/generate_allure_executor.py ") + sys.exit(1) + + server_url = sys.argv[1] + repo = sys.argv[2] + run_id = sys.argv[3] + data = { 'name':'GitHub Actions', 'type':'github' } + data['buildUrl'] = '{}/{}/actions/runs/{}'.format(server_url, repo, run_id) + data['buildName'] = 'GitHub Actions Run #{}'.format(run_id) + + print(json.dumps(data)) \ No newline at end of file From 6870334aac969974ba86f54d53389c137216b363 Mon Sep 17 00:00:00 2001 From: AdrickTench Date: Fri, 23 Aug 2024 18:16:25 -0500 Subject: [PATCH 18/18] include timestamps for test suites --- scripts/into_junit.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/into_junit.py b/scripts/into_junit.py index 2c400f15792..a45466ba2fd 100644 --- a/scripts/into_junit.py +++ b/scripts/into_junit.py @@ -2,6 +2,7 @@ import sys import re from collections import defaultdict +import datetime def create_testcase_element(testclass, testname, stdout, identifier, got, expected, status, url, time): # Create the testcase XML element with the class and test name attributes @@ -50,6 +51,8 @@ def parse_test_line(line): return testpackage, testname, stdout, full_identifier, got, expected, status, url, time def generate_junit_xml(input_file, timestamp): + dt = datetime.datetime.fromisoformat(timestamp) + timestamps_dict = {} packages_dict = defaultdict(list) # Dictionary to group test cases by their testpackage with open(input_file, 'r') as file: @@ -60,6 +63,9 @@ def generate_junit_xml(input_file, timestamp): parts = re.split(r'\s*\|\s*(?![^()]*\))', line.strip()) testpackage, testname, stdout, full_identifier, got, expected, status, url, time = parse_test_line(line) testcase = create_testcase_element(testpackage, testname, stdout, full_identifier, got, expected, status, url, time) + dt += datetime.timedelta(seconds=float(time)) + if testpackage not in timestamps_dict: + timestamps_dict[testpackage] = dt packages_dict[testpackage].append(testcase) print(f"Processing {testpackage}.{testname}: {status}", file=sys.stderr) except ValueError as e: @@ -69,7 +75,8 @@ def generate_junit_xml(input_file, timestamp): testsuites = ET.Element("testsuites", timestamp=timestamp) testsuites_time = 0.0 for testpackage, testcases in packages_dict.items(): - testsuite = ET.Element("testsuite", name=testpackage) + testsuite_timestamp = timestamps_dict[testpackage].isoformat(timespec='seconds') + testsuite = ET.Element("testsuite", name=testpackage, timestamp=testsuite_timestamp) testsuite_time = 0.0 for testcase in testcases: testsuite_time += float(testcase.get('time'))