acceptance: de-flake cert_renewal and hidden_paths

acceptance/cert_renewal/test.py

@@ -53,6 +53,7 @@ class Test(base.TestTopogen):
     """
 
     def _run(self):
+        self.await_connectivity()
 
         isd_ases = scion.ASList.load("%s/gen/as_list.yml" %
                                      self.artifacts).all
@@ -84,7 +85,7 @@ def _run(self):
 
         logger.info("==> Restart containers")
         self.setup_start()
-        time.sleep(5)
+        self.await_connectivity()
 
         logger.info("==> Check connectivity")
         end2end.run_fg()

acceptance/common/base.py

@@ -213,6 +213,26 @@ def teardown(self):
         if re.search(r"Exit\s+[1-9]\d*", ps):
             raise Exception("Failed services.\n" + ps)
 
+    def await_connectivity(self, quiet_seconds=None, timeout_seconds=None):
+        """
+        Wait for the beaconing process in a local topology to establish full connectivity, i.e. at
+        least one path between any two ASes.
+        Runs the tool/await-connectivity script.
+
+        Returns success when full connectivity is established or an error (exception) at
+        timeout (default 20s).
+
+        Remains quiet for a configurable time (default 10s). After that,
+        it reports the missing segments at 1s interval.
+        """
+        cmd = self.get_executable("await-connectivity")
+        cmd.cwd = self.artifacts
+        if quiet_seconds is not None:
+            cmd = cmd["-q", str(quiet_seconds)]
+        if timeout_seconds is not None:
+            cmd = cmd["-t", str(timeout_seconds)]
+        cmd.run_fg()
+
     def execute_tester(self, isd_as: ISD_AS, cmd: str, *args: str) -> str:
         """Executes a command in the designated "tester" container for the specified ISD-AS.
 

acceptance/common/topogen.bzl

@@ -45,6 +45,7 @@ def topogen_test(
     common_args = [
         "--executable=scion-pki:$(location //scion-pki/cmd/scion-pki)",
         "--executable=topogen:$(location //tools:topogen)",
+        "--executable=await-connectivity:$(location //tools:await_connectivity)",
         "--topo=$(location %s)" % topo,
     ]
     if gateway:
@@ -54,6 +55,7 @@ def topogen_test(
         "//scion-pki/cmd/scion-pki",
         "//tools:topogen",
         "//tools:docker_ip",
+        "//tools:await_connectivity",
         topo,
     ]
     loaders = container_loaders(tester, gateway)

acceptance/hidden_paths/test.py

@@ -3,7 +3,6 @@
 # Copyright 2020 Anapaya Systems
 
 import http.server
-import time
 import threading
 
 from plumbum import cmd
@@ -105,9 +104,9 @@ def setup_start(self):
                 ("0.0.0.0", self.http_server_port), http.server.SimpleHTTPRequestHandler)
         server_thread = threading.Thread(target=configuration_server, args=[server])
         server_thread.start()
+        self._server = server
 
         super().setup_start()
-        time.sleep(4)  # Give applications time to download configurations
 
         self._testers = {
             "2": "tester_1-ff00_0_2",
@@ -121,9 +120,11 @@ def setup_start(self):
             "4": "1-ff00:0:4",
             "5": "1-ff00:0:5",
         }
-        server.shutdown()
 
     def _run(self):
+        self.await_connectivity()
+        self._server.shutdown()  # by now configuration must have been downloaded everywhere
+
         # Group 3
         self._showpaths_bidirectional("2", "3", 0)
         self._showpaths_bidirectional("2", "5", 0)

tools/BUILD.bazel

@@ -7,6 +7,12 @@ sh_binary(
     visibility = ["//visibility:public"],
 )
 
+sh_binary(
+    name = "await_connectivity",
+    srcs = ["await-connectivity"],
+    visibility = ["//visibility:public"],
+)
+
 py_binary(
     name = "gomocks",
     srcs = ["gomocks.py"],

tools/await-connectivity

@@ -13,7 +13,7 @@
 # 
 # Usage: await-connectivity -q QUIET -t TIMEOUT
 
-set -euo pipefail
+set -Eeuo pipefail
 
 QUIET=10
 TIMEOUT=20
@@ -83,6 +83,7 @@ main() {
   parse_opts "$@"
 
   # poor bash-man's yaml parser
+  stat gen/as_list.yml > /dev/null # ensure file exists; command substitutions below don't fail because local (!?)
   local cores=$(sed -n '1,/Non-core/{s/^- //p}' gen/as_list.yml)
   local noncores=$(sed -n '/Non-core/,${s/^- //p}' gen/as_list.yml)