From eeaebcf43f42818cfbd828bc44d3b480a56abb41 Mon Sep 17 00:00:00 2001 From: Dimi Kot Date: Sun, 3 Mar 2024 00:42:45 -0800 Subject: [PATCH] Add support for custom SSH port --- .github/workflows/ci.yml | 2 +- README.md | 4 +- action.yml | 2 +- ci-storage | 59 ++++++++++++++++++------- docker/compose.yml | 2 +- docker/self-hosted-runner/Dockerfile | 6 +-- docker/self-hosted-runner/entrypoint.sh | 6 +-- 7 files changed, 53 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f3c520..7085743 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ on: - "v*" jobs: - # Tests ci-storage tool itself. + # Tests ci-storage tool and action itself. ci-storage-tool-test: runs-on: ubuntu-latest steps: diff --git a/README.md b/README.md index ec5e881..7b351ed 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,8 @@ ones, so rsync can run efficiently. # Required. action: '' - # Storage host in the format [user@]host; it must have password-free - # SSH key access. + # Storage host in the format [user@]host[:port]; it must allow password-free + # SSH key based access. # Default: the content of ~/ci-storage-host file. storage-host: '' diff --git a/action.yml b/action.yml index c8698cd..8bae11e 100644 --- a/action.yml +++ b/action.yml @@ -8,7 +8,7 @@ inputs: description: "What to do (store or load)." required: true storage-host: - description: "Storage host in the format [user@]host; it must have password-free SSH key access. If not passed, tries to read it from ~/ci-storage-host file." + description: "Storage host in the format [user@]host[:port]; it must allow password-free SSH key based access. If not passed, tries to read it from ~/ci-storage-host file." required: false storage-dir: description: "Storage directory on the remote host. If not set, uses ~/ci-storage/{owner}/{repo}" diff --git a/ci-storage b/ci-storage index d90de52..15df229 100755 --- a/ci-storage +++ b/ci-storage @@ -54,7 +54,7 @@ def main(): "--storage-host", type=str, required=False, - help="storage host in the format [user@]host; it must have password-free SSH key access; if omitted, uses the local filesystem (no SSH)", + help="storage host in the format [user@]host[:port]; it must allow password-free SSH key based access; if omitted, uses the local filesystem (no SSH)", ) parser.add_argument( "--storage-dir", @@ -168,9 +168,11 @@ def action_store( slot_ids_and_ages = list_slots(storage_host=storage_host, storage_dir=storage_dir) slot_id_recent = slot_ids_and_ages[0][0] if len(slot_ids_and_ages) else None slot_id_tmp = f"{slot_id}.tmp.{int(time.time())}" + host, port = parse_host_port(storage_host) check_call( cmd=[ "rsync", + *(["-e", shlex.join(build_ssh_cmd(port=port))] if storage_host else []), "-a", "--delete", "--partial", @@ -180,8 +182,7 @@ def action_store( *[f"--exclude={pattern}" for pattern in exclude], *(["-v"] if verbose else []), f"{local_dir}/", - (f"{storage_host}:" if storage_host else "") - + f"{storage_dir}/{slot_id_tmp}/", + (f"{host}:" if storage_host else "") + f"{storage_dir}/{slot_id_tmp}/", ], print_elapsed=True, ) @@ -223,9 +224,11 @@ def action_load( slot_id = slot_ids_and_ages[0][0] else: slot_id = normalize_slot_id(slot_id) + host, port = parse_host_port(storage_host) check_call( cmd=[ "rsync", + *(["-e", shlex.join(build_ssh_cmd(port=port))] if storage_host else []), "-a", "--delete", "--partial", @@ -233,7 +236,7 @@ def action_load( "--human-readable", *[f"--exclude={pattern}" for pattern in exclude], *(["-v"] if verbose else []), - (f"{storage_host}:" if storage_host else "") + f"{storage_dir}/{slot_id}/", + (f"{host}:" if storage_host else "") + f"{storage_dir}/{slot_id}/", f"{local_dir}/", ], print_elapsed=True, @@ -311,18 +314,11 @@ def check_output( cmd: list[str], indent: bool = False, ) -> str: - if host is not None: - cmd_prefix = [ - "ssh", - host, - ] - print(f"$ {cmd_to_debug_str([*cmd_prefix, *cmd])}") - cmd = [ - *cmd_prefix, - "-oStrictHostKeyChecking=no", - "-oUserKnownHostsFile=/dev/null", - shlex.join(cmd), - ] + host, port = parse_host_port(host) + if host: + ssh_prefix = [*build_ssh_cmd(port=port), host] + print(f"$ {cmd_to_debug_str([*ssh_prefix, *cmd])}") + cmd = [*ssh_prefix, shlex.join(cmd)] else: print(f"$ {cmd_to_debug_str(cmd)}") output = subprocess.check_output(cmd, text=True, stderr=subprocess.PIPE) @@ -362,6 +358,37 @@ def cmd_to_debug_str( return shlex.join([f"<{inv[arg]}>" if arg in inv else arg for arg in cmd]) +# +# Parses host:port pair (with port being optional). +# +def parse_host_port( + host_port: str | None, +) -> tuple[str | None, int | None]: + if not host_port: + return None, None + match = re.match(r"^(.*?)(?::(\d+))?$", host_port) + if match and match.group(2): + return match.group(1), int(match.group(2)) + else: + return host_port, None + + +# +# Builds ssh command line. +# +def build_ssh_cmd( + *, + port: int | None, +) -> list[str]: + return [ + "ssh", + *([f"-p{port}"] if port else []), + "-oStrictHostKeyChecking=no", + "-oUserKnownHostsFile=/dev/null", + "-oLogLevel=error", + ] + + # # A helper class for ArgumentParser. # diff --git a/docker/compose.yml b/docker/compose.yml index feb14d1..d23b0e2 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -24,5 +24,5 @@ services: - GH_REPOSITORY=${GH_REPOSITORY:-dimikot/ci-storage} - GH_LABELS=${GH_LABELS:-ci-storage} - GH_TOKEN - - CI_STORAGE_HOST=${CI_STORAGE_HOST:-host} + - CI_STORAGE_HOST=${CI_STORAGE_HOST:-host:22} - CI_STORAGE_HOST_PRIVATE_KEY=${CI_STORAGE_HOST_PRIVATE_KEY_TEST_ONLY?} diff --git a/docker/self-hosted-runner/Dockerfile b/docker/self-hosted-runner/Dockerfile index fe6642c..25488a0 100644 --- a/docker/self-hosted-runner/Dockerfile +++ b/docker/self-hosted-runner/Dockerfile @@ -37,9 +37,9 @@ RUN true \ && apt-get autoremove \ && apt-get clean \ && apt-get autoclean \ - && rm -rf /var/lib/apt/lists/* \ - && curl https://raw.githubusercontent.com/dimikot/ci-storage/main/ci-storage > /usr/bin/ci-storage \ - && chmod 755 /usr/bin/ci-storage + && rm -rf /var/lib/apt/lists/* + +ADD --chmod=755 https://raw.githubusercontent.com/dimikot/ci-storage/main/ci-storage /usr/bin/ci-storage COPY --chmod=755 --chown=user:user entrypoint.sh /home/user diff --git a/docker/self-hosted-runner/entrypoint.sh b/docker/self-hosted-runner/entrypoint.sh index 40296b2..f345990 100644 --- a/docker/self-hosted-runner/entrypoint.sh +++ b/docker/self-hosted-runner/entrypoint.sh @@ -29,8 +29,8 @@ if [[ "${GH_TOKEN:=}" == "" ]]; then echo "GH_TOKEN must be set."; exit 1; fi -if [[ "${CI_STORAGE_HOST:=}" != "" && ! "$CI_STORAGE_HOST" =~ ^([-.[:alnum:]]+@)?[-.[:alnum:]]+$ ]]; then - echo "If CI_STORAGE_HOST is passed, it must be in form of {hostname} or {user}@{hostname}."; +if [[ "${CI_STORAGE_HOST:=}" != "" && ! "$CI_STORAGE_HOST" =~ ^([-.[:alnum:]]+@)?[-.[:alnum:]]+(:[0-9]+)?$ ]]; then + echo "If CI_STORAGE_HOST is passed, it must be in form of [user@]host[:port]."; exit 1; fi if [[ "${CI_STORAGE_HOST_PRIVATE_KEY:=}" != "" && "$CI_STORAGE_HOST_PRIVATE_KEY" != *OPENSSH\ PRIVATE\ KEY* ]]; then @@ -57,8 +57,6 @@ name="ci-storage-$(hostname)" local_dir=_work/${GH_REPOSITORY##*/}/${GH_REPOSITORY##*/} if [[ "$CI_STORAGE_HOST" != "" && "$CI_STORAGE_HOST_PRIVATE_KEY" != "" ]]; then - ssh-keyscan -H "$CI_STORAGE_HOST" >> ~/.ssh/known_hosts - chmod 600 ~/.ssh/known_hosts mkdir -p "$local_dir" ci-storage load \ --storage-host="$CI_STORAGE_HOST" \