fixes related to "auto-unsupported" (#389)

* fix: Fixes related to "auto-unsupported"

* Add a `--no_auto_unsupported` argument to exclude implementations
  from auto-marking (e.g., when QNS is run as part of CI for an implementation)
* If there is only a single server or client implementation given,
  also don't automatically mark as unsupported
* Add some log messages when auto-marking to explain what is going on

* Update run.py

interop.py

@@ -51,6 +51,7 @@ class InteropRunner:
     _markdown = False
     _log_dir = ""
     _save_files = False
+    _no_auto_unsupported = []
 
     def __init__(
         self,
@@ -63,6 +64,7 @@ def __init__(
         debug: bool,
         save_files=False,
         log_dir="",
+        no_auto_unsupported=[],
     ):
         logger = logging.getLogger()
         logger.setLevel(logging.DEBUG)
@@ -81,6 +83,7 @@ def __init__(
         self._markdown = markdown
         self._log_dir = log_dir
         self._save_files = save_files
+        self._no_auto_unsupported = no_auto_unsupported
         if len(self._log_dir) == 0:
             self._log_dir = "logs_{:%Y-%m-%dT%H:%M:%S}".format(self._start_time)
         if os.path.exists(self._log_dir):
@@ -179,17 +182,23 @@ def _postprocess_results(self):
         servers = list(set(server for _, server in self._client_server_pairs))
         # If a client failed a test against all servers, make the test unsupported for the client
         questionable = [TestResult.FAILED, TestResult.UNSUPPORTED]
-        for c in clients:
+        for c in set(clients) - set(self._no_auto_unsupported):
             for t in self._tests:
                 if all(self.test_results[s][c][t] in questionable for s in servers):
-                    # Client failed test against all servers
+                    print(
+                        f'Client {c} failed test "{t.name()}" against all servers, '
+                        + 'marking the entire test as "unsupported"'
+                    )
                     for s in servers:
                         self.test_results[s][c][t] = TestResult.UNSUPPORTED
         # If a server failed a test against all clients, make the test unsupported for the server
-        for s in servers:
+        for s in set(servers) - set(self._no_auto_unsupported):
             for t in self._tests:
                 if all(self.test_results[s][c][t] in questionable for c in clients):
-                    # Server failed test against all clients
+                    print(
+                        f'Server {s} failed test "{t.name()}" against all clients, '
+                        + 'marking the entire test as "unsupported"'
+                    )
                     for c in clients:
                         self.test_results[s][c][t] = TestResult.UNSUPPORTED
 

run.py

@@ -78,6 +78,11 @@ def get_args():
             "--must-include",
             help="implementation that must be included",
         )
+        parser.add_argument(
+            "-n",
+            "--no-auto-unsupported",
+            help="implementations for which auto-marking as unsupported when all tests fail should be skipped",
+        )
         return parser.parse_args()
 
     replace_arg = get_args().replace
@@ -147,20 +152,30 @@ def get_tests_and_measurements(
         return tests, measurements
 
     t = get_tests_and_measurements(get_args().test)
+    clients = get_impls(get_args().client, client_implementations, "Client")
+    servers = get_impls(get_args().server, server_implementations, "Server")
+    # If there is only one client or server, we should not automatically mark tests as unsupported
+    no_auto_unsupported = set()
+    for kind in [clients, servers]:
+        if len(kind) == 1:
+            no_auto_unsupported.add(kind[0])
     return InteropRunner(
         implementations=implementations,
-        client_server_pairs=get_impl_pairs(
-            get_impls(get_args().client, client_implementations, "Client"),
-            get_impls(get_args().server, server_implementations, "Server"),
-            get_args().must_include,
-        ),
+        client_server_pairs=get_impl_pairs(clients, servers, get_args().must_include),
         tests=t[0],
         measurements=t[1],
         output=get_args().json,
         markdown=get_args().markdown,
         debug=get_args().debug,
         log_dir=get_args().log_dir,
         save_files=get_args().save_files,
+        no_auto_unsupported=(
+            no_auto_unsupported
+            if get_args().no_auto_unsupported is None
+            else get_impls(
+                get_args().no_auto_unsupported, clients + servers, "Client/Server"
+            )
+        ),
     ).run()