Merge pull request #423 from perftool-incubator/cyclictest-runfile-ci

Add cyclictest runfile

.github/workflows/unittest.yaml

@@ -36,13 +36,13 @@ jobs:
         run: |
           cd util
           source .venv/bin/activate
-          pytest -v --html=report.html --self-contained-html blockbreaker.py tests/*.py
+          pytest -v --html=report.html --self-contained-html blockbreaker.py validate_run_file.py tests/*.py
 
       - name: Upload html report
         uses: actions/upload-artifact@v3
         with:
           name: report
-          path: bin/report.html
+          path: util/report.html
 
   unittest-complete:
     runs-on: ubuntu-latest

util/tests/JSON/ci-cyclictest-run-file-remotehost-default.json

@@ -0,0 +1,98 @@
+{
+    "benchmarks": [
+        {
+            "name": "cyclictest",
+            "ids": "1",
+            "mv-params": {
+                "global-options": [
+                {
+                    "name": "required",
+                    "params": [
+                            { "arg": "duration", "vals": [ "10" ], "role": "client" },
+                        { "arg": "priority", "vals": [ "1" ], "role": "client" },
+                            { "arg": "smi", "vals": [ "off" ], "role": "client" }
+                    ]
+                }
+                ],
+                "sets": [
+                {
+                    "include": "required"
+                    }
+                ]
+            }
+        }
+    ],
+    "tool-params": [
+        {
+            "tool": "sysstat",
+            "params": [
+                { "arg": "subtools", "val": "mpstat", "enabled": "yes" }
+            ]
+        },
+        {
+            "tool": "procstat"
+        }
+    ],
+    "tags": {
+        "run": "cyclictest-run-file-json",
+        "userenv": "default"
+    },
+    "endpoints": [
+        {
+            "type": "remotehost",
+            "controller-ip": "CONTROLLER_IP",
+            "host": "CI_ENDPOINT_HOST",
+            "user": "CI_ENDPOINT_USER",
+            "userenv": "default",
+            "server": "1",
+            "client": "1-2",
+            "config": [
+                {
+                    "targets": [
+                        { "role": "client", "ids": "1" },
+                        { "role": "server", "ids": "1" }
+                    ],
+                    "settings": {
+                        "osruntime": "podman"
+                    }
+                },
+                {
+                    "targets": [
+                        { "role": "client", "ids": "2" }
+                    ],
+                   "settings": {
+                        "osruntime": "chroot"
+                    }
+                },
+                {
+                   "targets": "default",
+                   "settings": {
+                        "cpu-partitioning": "1"
+                    }
+                }             
+            ]
+        },
+        {
+            "type": "remotehost",
+            "controller-ip": "CONTROLLER_IP",
+            "host": "CI_ENDPOINT_HOST",
+            "user": "CI_ENDPOINT_USER",
+            "userenv": "default",
+            "profiler": "1",
+            "config": [
+                {
+                    "targets": [
+                        { "role": "profiler", "ids": "1" }
+                    ],
+                    "settings": {
+                        "osruntime": "chroot"
+                    }
+                }
+            ]
+        }
+    ],
+    "run-params": {
+	    "num-samples": 1,
+	    "test-order": "s"
+    }
+}

util/tests/test-validate_run_file.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+
+import pytest
+import glob
+import validate_run_file
+import sys
+
+class TestValidateRunFile:
+
+    """Validate all ci runfiles"""
+    @pytest.mark.parametrize("runfile", glob.glob("tests/JSON/ci-*.json"))
+    def test_validate_run_file(self, runfile, capsys):
+        validate = validate_run_file.validate(runfile)
+        out, err = capsys.readouterr()
+        assert validate == 0
+        assert "[ OK ]" in out
+        assert err == ""

util/validate_run_file.py

@@ -19,39 +19,39 @@ def process_options():
     args = parser.parse_args()
     return args
 
-def main():
-    """Main function of the validate.py utility"""
-
-    global args
+def validate(filename):
+    """Validate json with generic schema and endpoint specific schema"""
     err_msg=None
-
-    input_json = load_json_file(args.json_file)
+    input_json = load_json_file(filename)
     if input_json is None:
-        err_msg=f"{ err }: Failed to load run-file JSON { args.json_file }"
+        err_msg=f"Failed to load run-file JSON { filename }"
         rc=1
 
     if not validate_schema(input_json):
-        err_msg=(
-            f"{ err }: Failed to validate run-file JSON "
-            f"{ args.json_file } against schema."
-        )
+        err_msg=f"Failed to validate run-file JSON { filename } against schema."
         rc=2
 
     for json_blk in input_json["endpoints"]:
         endpoint_type = json_blk["type"]
         if not validate_schema(json_blk, "schema-" + endpoint_type + ".json"):
             err_msg=(
-                f"{ err }: Failed to validate the 'endpoints' block from "
-                f" the JSON run-file { args.json_file } against the "
+                f"Failed to validate the 'endpoints' block from "
+                f" the JSON run-file { filename } against the "
                 f"{ endpoint_type }'s schema."
             )
             rc=3
 
     if err_msg is not None:
-        print(f"ERROR: { err_msg }")
+        print(f"ERROR: { err_msg }", file=sys.stderr)
         return rc
 
-    print(f"[ OK ] run-file JSON { args.json_file } validated!")
+    print(f"[ OK ] run-file JSON { filename } validated!")
+    return 0
+
+def main():
+    """Main function of the validate.py utility"""
+    global args
+    return validate(args.json_file) 
 
 if __name__ == "__main__":
     args = process_options()