Merge pull request #99 from AdrickTench/main

ci.yml installs mettalog

.github/workflows/ci.yml

@@ -26,22 +26,36 @@ jobs:
       with:
         python-version: '3.x'
 
-    - name: Install junit2html
+    - name: Install python packages
       run: |
+        pip install ansi2html
+        pip install hyperon
         pip install junit2html
 
-    - name: Make Shell Script Executable
+    - name: Make Install Script Executable
+      run: chmod +x INSTALL.sh
+
+    - name: Run Install Script to install Mettalog
+      run: |
+        . ./INSTALL.sh --easy
+        echo $PATH >> $GITHUB_PATH
+
+    - name: Make Test Script Executable
       run: chmod +x scripts/run_commit_tests.sh
 
-    - name: Run Shell Script to Generate Input File
+    - name: Run Test Script to Generate Input File
       continue-on-error: true
       run: |
-        ./scripts/run_commit_tests.sh
+        TIMESTAMP=$(date +"%Y-%m-%dT%H:%M:%S")
+        ./scripts/run_commit_tests.sh -t $TIMESTAMP
+        echo "TIMESTAMP=$(echo $TIMESTAMP)" >> $GITHUB_ENV
+      env:
+        TERM: xterm-256color
 
     - name: Run JUnit Report Generation Script
       continue-on-error: true
       run: |
-        python scripts/into_junit.py /tmp/SHARED.UNITS > junit.xml
+        python scripts/into_junit.py /tmp/SHARED.UNITS $TIMESTAMP > junit.xml
 
     - name: Convert JUnit XML to Standard HTML Report
       continue-on-error: true
@@ -119,6 +133,8 @@ jobs:
         if [ -f "previous-junit.xml" ]; then
           cp previous-junit.xml ./allure-results/
         fi
+        python scripts/generate_allure_environment.py ${{ github.sha }} ${{ github.ref_name }} > ./allure-results/environment.properties
+        python scripts/generate_allure_executor.py ${{ github.server_url }} ${{ github.repository }} ${{ github.run_id }} > ./allure-results/executor.json
 
     - name: Generate Allure Report
       run: |
@@ -137,7 +153,7 @@ jobs:
 
     - name: Auto-Approve the Pull Request
       if: github.event_name == 'pull_request_target'
-      uses: hmarr/auto-approve-action@v3
+      uses: hmarr/auto-approve-action@v4
       with:
         github-token: ${{ secrets.GITHUB_TOKEN }}
 

scripts/generate_allure_environment.py

@@ -0,0 +1,10 @@
+import sys
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print("Usage: python scripts/generate_allure_environment.py <commit SHA> <branch>")
+        sys.exit(1)
+
+    commit_SHA = sys.argv[1]
+    branch = sys.argv[2]
+    print("COMMIT_SHA = {}\nBRANCH = {}".format(commit_SHA, branch))

scripts/generate_allure_executor.py

@@ -0,0 +1,15 @@
+import sys, json
+
+if __name__ == "__main__":
+    if len(sys.argv) != 4:
+        print("Usage: python scripts/generate_allure_executor.py <server-url> <repo> <run-id>")
+        sys.exit(1)
+
+    server_url = sys.argv[1]
+    repo = sys.argv[2]
+    run_id = sys.argv[3]
+    data = { 'name':'GitHub Actions', 'type':'github' }
+    data['buildUrl'] = '{}/{}/actions/runs/{}'.format(server_url, repo, run_id)
+    data['buildName'] = 'GitHub Actions Run #{}'.format(run_id)
+
+    print(json.dumps(data))

scripts/into_junit.py

@@ -2,10 +2,11 @@
 import sys
 import re
 from collections import defaultdict
+import datetime
 
-def create_testcase_element(testclass, testname, stdout, identifier, got, expected, status, url):
+def create_testcase_element(testclass, testname, stdout, identifier, got, expected, status, url, time):
     # Create the testcase XML element with the class and test name attributes
-    testcase = ET.Element("testcase", classname=testclass, name=testname)
+    testcase = ET.Element("testcase", classname=testclass, name=testname, time=time)
 
     test_res = f"Assertion: {stdout}\nExpected: {expected}\nActual: {got}"
     sys_out_text = f"<![CDATA[\n<a href=\"{url}\">Test Report</a>\n\n{test_res}\n]]>"
@@ -44,11 +45,14 @@ def parse_test_line(line):
             raise ValueError("Test package or test name is empty after splitting.")
     except ValueError as e:
         raise ValueError(f"Identifier does not contain the expected format: {full_identifier}. Error: {str(e)}")
+
+    time = '.01' # bogus time until tests actually note their runtime
 
-    return testpackage, testname, stdout, full_identifier, got, expected, status, url
+    return testpackage, testname, stdout, full_identifier, got, expected, status, url, time
 
-def generate_junit_xml(input_file):
-    testsuites = ET.Element("testsuites")
+def generate_junit_xml(input_file, timestamp):
+    dt = datetime.datetime.fromisoformat(timestamp)
+    timestamps_dict = {}
     packages_dict = defaultdict(list)  # Dictionary to group test cases by their testpackage
 
     with open(input_file, 'r') as file:
@@ -57,29 +61,41 @@ def generate_junit_xml(input_file):
             if line.startswith("|"):
                 try:
                     parts = re.split(r'\s*\|\s*(?![^()]*\))', line.strip())
-                    testpackage, testname, stdout, full_identifier, got, expected, status, url = parse_test_line(line)
-                    testcase = create_testcase_element(testpackage, testname, stdout, full_identifier, got, expected, status, url)
+                    testpackage, testname, stdout, full_identifier, got, expected, status, url, time = parse_test_line(line)
+                    testcase = create_testcase_element(testpackage, testname, stdout, full_identifier, got, expected, status, url, time)
+                    dt += datetime.timedelta(seconds=float(time))
+                    if testpackage not in timestamps_dict:
+                        timestamps_dict[testpackage] = dt
                     packages_dict[testpackage].append(testcase)
                     print(f"Processing {testpackage}.{testname}: {status}", file=sys.stderr)
                 except ValueError as e:
                     print(f"Skipping line due to error: {e}\nLine: {line}\nParts: {parts}", file=sys.stderr)
 
     # Create a testsuite for each testpackage group
+    testsuites = ET.Element("testsuites", timestamp=timestamp)
+    testsuites_time = 0.0
     for testpackage, testcases in packages_dict.items():
-        testsuite = ET.Element("testsuite", name=testpackage)
+        testsuite_timestamp = timestamps_dict[testpackage].isoformat(timespec='seconds')
+        testsuite = ET.Element("testsuite", name=testpackage, timestamp=testsuite_timestamp)
+        testsuite_time = 0.0
         for testcase in testcases:
+            testsuite_time += float(testcase.get('time'))
             testsuite.append(testcase)
+        testsuites_time += testsuite_time
+        testsuite.set('time', str(testsuite_time))
         testsuites.append(testsuite)
+    testsuites.set('time', str(testsuites_time))
 
     # Generate the XML tree and return it as a string
     tree = ET.ElementTree(testsuites)
     return ET.tostring(testsuites, encoding="utf-8", xml_declaration=True).decode("utf-8")
 
 if __name__ == "__main__":
-    if len(sys.argv) != 2:
-        print("Usage: python scripts/into_junit.py <input_file>")
+    if len(sys.argv) != 3:
+        print("Usage: python scripts/into_junit.py <input_file> <timestamp>")
         sys.exit(1)
 
     input_file = sys.argv[1]
-    junit_xml = generate_junit_xml(input_file)
+    timestamp = sys.argv[2]
+    junit_xml = generate_junit_xml(input_file, timestamp)
     print(junit_xml)

scripts/run_commit_tests.sh

@@ -1,8 +1,27 @@
 #!/bin/bash
 
-# This script generates the input file used by the Python script.
-# Replace the following lines with the actual commands to generate the input file.
+# parse arguments
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -t|--timestamp)
+      timestamp="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    *)
+      # Ignore unknown options
+      ;;
+  esac
+done
 
-#echo "| ANTI-REGRESSION.BC-COMP.01 | PASS |(https://example.com/test-report) | (assertEqualToResult (add-atom &kb (: axiom (nums 2 3)))) | (()) | (()) |" > /tmp/SHARED.UNITS
+# generate the output directory with timestamp
+if [ -z $timestamp ]; then
+    timestamp=$(date +"%Y-%m-%dT%H:%M:%S")
+fi
+output=reports/tests_output/baseline-compat-$timestamp/
+
+# run the tests
+echo Running baseline_compat tests to $output
 cat ./reports/SHARED.UNITS.PREV.md > /tmp/SHARED.UNITS
-# You can add more lines or commands to generate additional input data
+#cat /dev/null> /tmp/SHARED.UNITS
+#mettalog --output=$output --test --clean tests/baseline_compat/anti-regression/comma_is_not_special.metta