From 70e6b2b9e1f43fb60a7359e71c28b034d48de163 Mon Sep 17 00:00:00 2001 From: Matthias Frei Date: Tue, 19 Sep 2023 21:40:51 +0200 Subject: [PATCH] tools: specify project name in docker-compose files --- .buildkite/hooks/bazel-remote.yml | 1 + .buildkite/hooks/go-module-proxy.yml | 2 +- .buildkite/hooks/pre-command | 4 ++-- acceptance/cert_renewal/test.py | 4 ++-- acceptance/common/docker.py | 5 +---- demo/file_transfer/file_transfer.py | 5 ++--- scion.sh | 4 ++-- tools/dc | 3 +-- tools/topology/common.py | 4 ---- tools/topology/docker.py | 32 +++++++++++----------------- tools/topology/docker_utils.py | 7 +++--- tools/topology/jaeger.py | 3 +-- tools/topology/prometheus.py | 5 ++--- tools/topology/sig.py | 23 +++++++------------- 14 files changed, 39 insertions(+), 63 deletions(-) diff --git a/.buildkite/hooks/bazel-remote.yml b/.buildkite/hooks/bazel-remote.yml index 07523b1c2a..a45379295d 100644 --- a/.buildkite/hooks/bazel-remote.yml +++ b/.buildkite/hooks/bazel-remote.yml @@ -8,6 +8,7 @@ services: "--experimental_remote_asset_api", "--profile_port=0", "--s3.auth_method=access_key", + "--s3.disable_ssl", # Speed up (?) https://github.com/buchgr/bazel-remote/issues/675 "--enable_endpoint_metrics", ] network_mode: host diff --git a/.buildkite/hooks/go-module-proxy.yml b/.buildkite/hooks/go-module-proxy.yml index 4c852825f1..eabbc49796 100644 --- a/.buildkite/hooks/go-module-proxy.yml +++ b/.buildkite/hooks/go-module-proxy.yml @@ -1,8 +1,8 @@ --- version: "2.4" +name: athens services: go-module-proxy: - container_name: go-module-proxy network_mode: host # gomods/athens:canary of 29.04.2022 -- using unstable version for the "ATHENS_NETWORK_MODE=offline" feature, see below image: $REGISTRY/gomods/athens@sha256:319701a7541a3e70ff45f290b185364738cac62b845eeb13b25c258f6189af7f diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 0650d36442..fea93b377f 100755 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -43,7 +43,7 @@ echo "~~~ Starting bazel remote cache proxy" # Start bazel remote cache proxy for S3 # Note that S3 keys are injected by buildkite, see # https://buildkite.com/docs/pipelines/secrets#storing-secrets-with-the-elastic-ci-stack-for-aws -docker-compose -f .buildkite/hooks/bazel-remote.yml -p bazel_remote up -d +docker-compose -f .buildkite/hooks/bazel-remote.yml up -d echo "~~~ Starting go module proxy" -docker-compose -f .buildkite/hooks/go-module-proxy.yml -p athens up -d +docker-compose -f .buildkite/hooks/go-module-proxy.yml up -d diff --git a/acceptance/cert_renewal/test.py b/acceptance/cert_renewal/test.py index 0b53b2883a..12cf3a963b 100755 --- a/acceptance/cert_renewal/test.py +++ b/acceptance/cert_renewal/test.py @@ -71,9 +71,9 @@ def _run(self): end2end.run_fg() logger.info("==> Shutting down control servers and purging caches") - for container in self.dc.list_containers("scion_sd.*"): + for container in self.dc.list_containers("sd.*"): self.dc("rm", container) - for container in self.dc.list_containers("scion_cs.*"): + for container in self.dc.list_containers("cs.*"): self.dc.stop_container(container) for cs_config in cs_configs: files = list((pathlib.Path(self.artifacts) / diff --git a/acceptance/common/docker.py b/acceptance/common/docker.py index b77a7f8b4b..47128ce6ac 100644 --- a/acceptance/common/docker.py +++ b/acceptance/common/docker.py @@ -35,15 +35,12 @@ from plumbum import cmd SCION_DC_FILE = "gen/scion-dc.yml" -DC_PROJECT = "scion" SCION_TESTING_DOCKER_ASSERTIONS_OFF = 'SCION_TESTING_DOCKER_ASSERTIONS_OFF' class Compose(object): def __init__(self, - project: str = DC_PROJECT, compose_file: str = SCION_DC_FILE): - self.project = project self.compose_file = compose_file def __call__(self, *args, **kwargs) -> str: @@ -51,7 +48,7 @@ def __call__(self, *args, **kwargs) -> str: # Note: not using plumbum here due to complications with encodings in the captured output try: res = subprocess.run( - ["docker-compose", "-f", self.compose_file, "-p", self.project, *args], + ["docker-compose", "-f", self.compose_file, *args], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8") except subprocess.CalledProcessError as e: raise _CalledProcessErrorWithOutput(e) from None diff --git a/demo/file_transfer/file_transfer.py b/demo/file_transfer/file_transfer.py index 6207159e35..e520c8c6f5 100644 --- a/demo/file_transfer/file_transfer.py +++ b/demo/file_transfer/file_transfer.py @@ -43,7 +43,7 @@ def _set_path_count(self, path_count): with open(config_name, "w") as f: json.dump(t, f, indent=2) # Reload the config. - self.dc("kill", "-s", "SIGHUP", "scion_sig_1-ff00_0_111") + self.dc("kill", "-s", "SIGHUP", "sig_1-ff00_0_111") # Give gateway some time to start using the new path count. time.sleep(2) @@ -82,7 +82,6 @@ def setup_prepare(self): with open(scion_dc, "r") as file: dc = yaml.load(file, Loader=yaml.FullLoader) dc["services"]["tc_setup"] = { - "container_name": "tc_setup", "image": "tester:latest", "cap_add": ["NET_ADMIN"], "volumes": [{ @@ -93,7 +92,7 @@ def setup_prepare(self): "entrypoint": ["/bin/sh", "-exc", "ls -l /share; /share/tc_setup.sh scn_000 16.0mbit ;" " /share/tc_setup.sh scn_001 16.0mbit"], - "depends_on": ["scion_br1-ff00_0_111-1", "scion_br1-ff00_0_111-2"], + "depends_on": ["br1-ff00_0_111-1", "br1-ff00_0_111-2"], "network_mode": "host", } with open(scion_dc, "w") as file: diff --git a/scion.sh b/scion.sh index c437e32125..d72896aaae 100755 --- a/scion.sh +++ b/scion.sh @@ -6,7 +6,7 @@ cmd_bazel_remote() { mkdir -p "$HOME/.cache/bazel/remote" uid=$(id -u) gid=$(id -g) - USER_ID="$uid" GROUP_ID="$gid" docker-compose -f bazel-remote.yml -p bazel_remote up -d + USER_ID="$uid" GROUP_ID="$gid" docker-compose -f bazel-remote.yml up -d } cmd_topo_clean() { @@ -36,7 +36,7 @@ cmd_run() { run_jaeger echo "Running the network..." if is_docker_be; then - docker-compose -f gen/scion-dc.yml -p scion up -d + docker-compose -f gen/scion-dc.yml up -d return 0 else run_setup diff --git a/tools/dc b/tools/dc index 7901082a31..0a5939187f 100755 --- a/tools/dc +++ b/tools/dc @@ -80,10 +80,9 @@ cmd_prom() { # Runs docker compose for the given project dc() { - local project="$1" local dc_file="gen/$1-dc.yml" shift - COMPOSE_FILE="$dc_file" docker-compose -p "$project" --ansi never "$@" + COMPOSE_FILE="$dc_file" docker-compose --ansi never "$@" } cmd_collect_logs() { diff --git a/tools/topology/common.py b/tools/topology/common.py index 934725effe..7f9b74a7eb 100644 --- a/tools/topology/common.py +++ b/tools/topology/common.py @@ -182,10 +182,6 @@ def sciond_name(topo_id): return 'sd%s' % topo_id.file_fmt() -def sciond_svc_name(topo_id): - return 'scion_%s' % sciond_name(topo_id) - - def json_default(o): if isinstance(o, AddressProxy): return str(o.ip) diff --git a/tools/topology/docker.py b/tools/topology/docker.py index b0594925e4..8957070591 100644 --- a/tools/topology/docker.py +++ b/tools/topology/docker.py @@ -25,7 +25,7 @@ ArgsTopoDicts, docker_host, docker_image, - sciond_svc_name, + sciond_name, ) from topology.docker_utils import DockerUtilsGenArgs, DockerUtilsGenerator from topology.net import NetworkDescription, IPNetwork @@ -54,6 +54,7 @@ def __init__(self, args): self.args = args self.dc_conf = { 'version': DOCKER_COMPOSE_CONFIG_VERSION, + 'name': 'scion', 'services': {}, 'networks': {}, 'volumes': {} @@ -62,7 +63,6 @@ def __init__(self, args): self.bridges = {} self.output_base = os.environ.get('SCION_OUTPUT_BASE', os.getcwd()) self.user = '%d:%d' % (os.getuid(), os.getgid()) - self.prefix = 'scion_' def generate(self): self._create_networks() @@ -145,18 +145,17 @@ def _br_conf(self, topo_id, topo, base): image = docker_image(self.args, 'posix-router') entry = { 'image': image, - 'container_name': self.prefix + k, 'networks': {}, 'user': self.user, 'volumes': ['%s:/share/conf:ro' % base], 'environment': { 'SCION_EXPERIMENTAL_BFD_DETECT_MULT': - '${SCION_EXPERIMENTAL_BFD_DETECT_MULT}', + '${SCION_EXPERIMENTAL_BFD_DETECT_MULT:-}', 'SCION_EXPERIMENTAL_BFD_DESIRED_MIN_TX': - '${SCION_EXPERIMENTAL_BFD_DESIRED_MIN_TX}', + '${SCION_EXPERIMENTAL_BFD_DESIRED_MIN_TX:-}', 'SCION_EXPERIMENTAL_BFD_REQUIRED_MIN_RX': - '${SCION_EXPERIMENTAL_BFD_REQUIRED_MIN_RX}', + '${SCION_EXPERIMENTAL_BFD_REQUIRED_MIN_RX:-}', }, 'command': ['--config', '/share/conf/%s.toml' % k] } @@ -170,18 +169,16 @@ def _br_conf(self, topo_id, topo, base): entry['networks'][self.bridges[net['net']]] = { '%s_address' % ipv: str(net[ipv]) } - self.dc_conf['services']['scion_%s' % k] = entry + self.dc_conf['services'][k] = entry def _control_service_conf(self, topo_id, topo, base): for k in topo.get("control_service", {}).keys(): entry = { 'image': docker_image(self.args, 'control'), - 'container_name': - self.prefix + k, - 'depends_on': ['scion_disp_%s' % k], + 'depends_on': ['disp_%s' % k], 'network_mode': - 'service:scion_disp_%s' % k, + 'service:disp_%s' % k, 'user': self.user, 'volumes': [ @@ -192,7 +189,7 @@ def _control_service_conf(self, topo_id, topo, base): ], 'command': ['--config', '/share/conf/%s.toml' % k] } - self.dc_conf['services']['scion_%s' % k] = entry + self.dc_conf['services'][k] = entry def _dispatcher_conf(self, topo_id, topo, base): image = 'dispatcher' @@ -221,7 +218,6 @@ def _dispatcher_conf(self, topo_id, topo, base): entry['networks'][self.bridges[net['net']]] = { '%s_address' % ipv: ip } - entry['container_name'] = '%sdisp_%s' % (self.prefix, disp_id) entry['volumes'].append(self._disp_vol(disp_id)) conf = '%s:/share/conf:rw' % base entry['volumes'].append(conf) @@ -229,12 +225,12 @@ def _dispatcher_conf(self, topo_id, topo, base): '--config', '/share/conf/disp_%s.toml' % disp_id ] - self.dc_conf['services']['scion_disp_%s' % disp_id] = entry + self.dc_conf['services']['disp_%s' % disp_id] = entry self.dc_conf['volumes'][self._disp_vol(disp_id).split(':') [0]] = None def _sciond_conf(self, topo_id, base): - name = sciond_svc_name(topo_id) + name = sciond_name(topo_id) net = self.elem_networks["sd" + topo_id.file_fmt()][0] ipv = 'ipv4' if ipv not in net: @@ -245,9 +241,7 @@ def _sciond_conf(self, topo_id, base): 'extra_hosts': ['jaeger:%s' % docker_host(self.args.docker)], 'image': docker_image(self.args, 'daemon'), - 'container_name': - '%ssd%s' % (self.prefix, topo_id.file_fmt()), - 'depends_on': ['scion_disp_%s' % disp_id], + 'depends_on': ['disp_%s' % disp_id], 'user': self.user, 'volumes': [ @@ -266,7 +260,7 @@ def _sciond_conf(self, topo_id, base): self.dc_conf['services'][name] = entry def _disp_vol(self, disp_id): - return 'vol_%sdisp_%s:/run/shm/dispatcher:rw' % (self.prefix, disp_id) + return 'vol_disp_%s:/run/shm/dispatcher:rw' % disp_id def _cache_vol(self): return self.output_base + '/gen-cache:/share/cache:rw' diff --git a/tools/topology/docker_utils.py b/tools/topology/docker_utils.py index fc15124bd7..90d4041f51 100644 --- a/tools/topology/docker_utils.py +++ b/tools/topology/docker_utils.py @@ -75,19 +75,18 @@ def _test_conf(self, topo_id): name = 'tester_%s' % topo_id.file_fmt() entry = { 'image': docker_image(self.args, 'tester'), - 'container_name': 'tester_%s' % topo_id.file_fmt(), - 'depends_on': ['scion_disp_%s' % name], + 'depends_on': ['disp_%s' % name], 'privileged': True, 'entrypoint': 'sh tester.sh', 'environment': {}, # 'user': self.user, 'volumes': [ - 'vol_scion_disp_%s:/run/shm/dispatcher:rw' % name, + 'vol_disp_%s:/run/shm/dispatcher:rw' % name, self.output_base + '/logs:' + cntr_base + '/logs:rw', self.output_base + '/gen:' + cntr_base + '/gen:rw', self.output_base + '/gen-certs:' + cntr_base + '/gen-certs:rw' ], - 'network_mode': 'service:scion_disp_%s' % name, + 'network_mode': 'service:disp_%s' % name, } net = self.args.networks[name][0] ipv = 'ipv4' diff --git a/tools/topology/jaeger.py b/tools/topology/jaeger.py index 7355694c55..c21da963b2 100644 --- a/tools/topology/jaeger.py +++ b/tools/topology/jaeger.py @@ -43,13 +43,12 @@ def generate(self): yaml.dump(dc_conf, default_flow_style=False)) def _generate_dc(self): - name = 'jaeger' entry = { 'version': '2', + 'name': 'jaeger', 'services': { 'jaeger': { 'image': 'jaegertracing/all-in-one:1.22.0', - 'container_name': name, 'user': '%s:%s' % (str(os.getuid()), str(os.getgid())), 'ports': [ '6831:6831/udp', diff --git a/tools/topology/prometheus.py b/tools/topology/prometheus.py index 9a896a64fc..edd71e27a3 100644 --- a/tools/topology/prometheus.py +++ b/tools/topology/prometheus.py @@ -149,13 +149,12 @@ def _write_disp_file(self): write_file(targets_path, yaml.dump(target_config, default_flow_style=False)) def _write_dc_file(self): - name = 'prometheus' prom_dc = { 'version': DOCKER_COMPOSE_CONFIG_VERSION, + 'name': 'prometheus', 'services': { - name: { + 'prometheus': { 'image': 'prom/prometheus:v2.6.0', - 'container_name': name, 'network_mode': 'host', 'volumes': [ self.output_base + '/gen:/prom-config:ro' diff --git a/tools/topology/sig.py b/tools/topology/sig.py index eb7855c0ae..355214343b 100644 --- a/tools/topology/sig.py +++ b/tools/topology/sig.py @@ -23,7 +23,7 @@ from topology.common import ( ArgsBase, json_default, - sciond_svc_name, + sciond_name, SD_API_PORT, SIG_CONFIG_NAME, translate_features, @@ -55,7 +55,6 @@ def __init__(self, args): self.dc_conf = args.dc_conf self.user = '%d:%d' % (os.getuid(), os.getgid()) self.output_base = os.environ.get('SCION_OUTPUT_BASE', os.getcwd()) - self.prefix = '' def generate(self): for topo_id, topo in self.args.topo_dicts.items(): @@ -72,8 +71,6 @@ def _dispatcher_conf(self, topo_id, base): entry = { 'image': 'dispatcher', - 'container_name': - 'scion_%sdisp_sig_%s' % (self.prefix, topo_id.file_fmt()), 'depends_on': { 'utils_chowner': { 'condition': 'service_started' @@ -98,15 +95,14 @@ def _dispatcher_conf(self, topo_id, base): entry['networks'][self.args.bridges[net['net']]] = { '%s_address' % ipv: str(net[ipv]) } - self.dc_conf['services']['scion_disp_sig_%s' % + self.dc_conf['services']['disp_sig_%s' % topo_id.file_fmt()] = entry - vol_name = 'vol_scion_%sdisp_sig_%s' % (self.prefix, - topo_id.file_fmt()) + vol_name = 'vol_disp_sig_%s' % topo_id.file_fmt() self.dc_conf['volumes'][vol_name] = None def _sig_dc_conf(self, topo_id, base): - setup_name = 'scion_sig_setup_%s' % topo_id.file_fmt() - disp_id = 'scion_disp_sig_%s' % topo_id.file_fmt() + setup_name = 'sig_setup_%s' % topo_id.file_fmt() + disp_id = 'disp_sig_%s' % topo_id.file_fmt() self.dc_conf['services'][setup_name] = { 'image': 'tester:latest', 'depends_on': [disp_id], @@ -114,14 +110,12 @@ def _sig_dc_conf(self, topo_id, base): 'privileged': True, 'network_mode': 'service:%s' % disp_id, } - self.dc_conf['services']['scion_sig_%s' % topo_id.file_fmt()] = { + self.dc_conf['services']['sig_%s' % topo_id.file_fmt()] = { 'image': 'posix-gateway:latest', - 'container_name': - 'scion_%ssig_%s' % (self.prefix, topo_id.file_fmt()), 'depends_on': [ disp_id, - sciond_svc_name(topo_id), + sciond_name(topo_id), setup_name, ], 'environment': { @@ -196,5 +190,4 @@ def _sig_toml(self, topo_id, topo): write_file(path, toml.dumps(sig_conf)) def _disp_vol(self, topo_id): - return 'vol_scion_%sdisp_sig_%s:/run/shm/dispatcher:rw' % ( - self.prefix, topo_id.file_fmt()) + return 'vol_disp_sig_%s:/run/shm/dispatcher:rw' % topo_id.file_fmt()