diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7085743..a991385 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,7 +78,7 @@ jobs: with: action: "store" - name: Kill self-hosted runner container - run: kill -SIGINT $(cat ~user/entrypoint.pid) + run: kill -SIGINT $(cat ~user/.entrypoint.pid) # Publishes "host" image to Docker Hub. push-host: diff --git a/docker/compose-up.sh b/docker/compose-up.sh index 0df0c56..35ced2b 100755 --- a/docker/compose-up.sh +++ b/docker/compose-up.sh @@ -1,6 +1,8 @@ #!/bin/bash set -e -echo "Booting containters on the local laptop for debugging purposes..." +echo "Building & booting containters on the local laptop for debugging purposes..." -GH_REPOSITORY=$(gh repo view --json owner,name -q '.owner.login + "/" + .name') GH_TOKEN=$(gh auth token) docker compose up --build "$@" +GH_REPOSITORY=$(gh repo view --json owner,name -q '.owner.login + "/" + .name') \ + GH_TOKEN=$(gh auth token) \ + docker compose up --build "$@" diff --git a/docker/self-hosted-runner/Dockerfile b/docker/self-hosted-runner/Dockerfile index 2fba198..47d25c8 100644 --- a/docker/self-hosted-runner/Dockerfile +++ b/docker/self-hosted-runner/Dockerfile @@ -32,6 +32,7 @@ RUN true \ && curl --no-progress-meter -L https://github.com/actions/runner/releases/download/v$RUNNER_VERSION/actions-runner-$arch-$RUNNER_VERSION.tar.gz | tar xz USER root + RUN true \ && ~user/actions-runner/bin/installdependencies.sh \ && apt-get autoremove \ @@ -39,14 +40,14 @@ RUN true \ && apt-get autoclean \ && rm -rf /var/lib/apt/lists/* -ADD --chmod=755 https://raw.githubusercontent.com/dimikot/ci-storage/main/ci-storage /usr/bin/ci-storage - -COPY --chmod=755 --chown=user:user entrypoint.sh /home/user +# We want the default user to be root, to allow people extend the image with no +# boilerplate. But when someone runs e.g. "docker compose exec bash -l", we want +# the user to be "user". +RUN echo "cd ~user && gosu user:user bash" > ~/.bash_profile -USER user -WORKDIR /home/user -ENTRYPOINT ["./entrypoint.sh"] +ADD --chmod=755 https://raw.githubusercontent.com/dimikot/ci-storage/main/ci-storage /usr/bin/ci-storage +COPY --chmod=755 --chown=root:root root/entrypoint*.sh /root +COPY --chmod=755 --chown=user:user user/entrypoint*.sh /home/user -# If overridden in the derived image, evals this as a shell script after -# config.sh, but before run.sh. -CMD [] +WORKDIR /root +ENTRYPOINT ["/root/entrypoint.sh"] diff --git a/docker/self-hosted-runner/README.md b/docker/self-hosted-runner/README.md index 6ed5c8b..481aaa9 100644 --- a/docker/self-hosted-runner/README.md +++ b/docker/self-hosted-runner/README.md @@ -6,23 +6,38 @@ self-hosted runners as you want. An example scenario: 1. Build an image based off this Dockerfile and publish it. You'll likely want to install some more software into that image (e.g. Node, Python etc.), so it may make sense to extend the base image with your own commands. -2. Run AWS ECS cluster (with e.g. AWS Fargate) and use the image you just - published. Configure its environment variables accordingly: GH_REPOSITORY, - GH_LABELS, GH_TOKEN etc. - see details in entrypoint.sh. -3. Set up auto-scaling rules in the ECS cluster based on the containers' CPU - usage. The running containers are safe to shut down at anytime if it's done - gracefully and with high timeout (to let all the running workflow jobs finish - there and de-register the runner). +2. Run AWS ECS cluster (with e.g. AWS ECS or spot instances with manual docker + container boot) and use the image you just published. Configure its + environment variables accordingly: `GH_REPOSITORY`, `GH_LABELS`, `GH_TOKEN` + etc. - see the full list in `entrypoint.00-validate.sh`. +3. Set up auto-scaling rules based on the containers' CPU usage. The running + containers are safe to shut down at anytime if it's done gracefully and with + high timeout (to let all the running workflow jobs finish there and + de-register the runner). 4. And here comes the perf magic: when the container first boots, but before it becomes available for the jobs, it pre-initializes its work directory from - ci-storage slots storage (see CI_STORAGE_HOST). So when a job is picked up, + ci-storage slots storage (see `CI_STORAGE_HOST`). So when a job is picked up, it already has its work directory pre-created and having most of the build artifacts of someone else. If the job then uses ci-storage GitHub action to restore the files from a slot, it will be very quick, because most of the files are already there. -The container in this Dockerfile is serves only one particular GitHub repository -(controlled by GH_REPOSITORY environment variable at boot time). To serve +The container in this Dockerfile serves only one particular GitHub repository +(controlled by `GH_REPOSITORY` environment variable at boot time). To serve different repositories, boot different containers. +We also expose a naming convention on extra entrypoint files. When extending +this image, one can put custom files like `/root/entrypoint.*.sh` (to be run as +root) or `/home/user/entrypoint.*.sh` (to be run as user `user`). Those files +will be automatically picked up and executed. + +To enter the container, run e.g.: + +``` +docker compose exec self-hosted-runner bash -l +``` + +It will automatically change the user and current directory to `/home/user` +(without `-l`, it will run a root shell session). + See also https://github.com/dimikot/ci-storage diff --git a/docker/self-hosted-runner/root/entrypoint.00-validate.sh b/docker/self-hosted-runner/root/entrypoint.00-validate.sh new file mode 100644 index 0000000..73bc084 --- /dev/null +++ b/docker/self-hosted-runner/root/entrypoint.00-validate.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -u -e + +if [[ "${GH_REPOSITORY:=}" != */* ]]; then + echo "GH_REPOSITORY must be set, and the format should be {owner}/{repo}."; + exit 1; +fi + +if [[ "${GH_LABELS:=}" == "" ]]; then + echo "GH_LABELS must be set."; + exit 1; +fi + +if [[ "${GH_TOKEN:=}" == "" ]]; then + echo "GH_TOKEN must be set."; + exit 1; +fi + +if [[ "${CI_STORAGE_HOST:=}" != "" && ! "$CI_STORAGE_HOST" =~ ^([-.[:alnum:]]+@)?[-.[:alnum:]]+(:[0-9]+)?$ ]]; then + echo "If CI_STORAGE_HOST is passed, it must be in form of [user@]host[:port]."; + exit 1; +fi + +if [[ "${CI_STORAGE_HOST_PRIVATE_KEY:=}" != "" && "$CI_STORAGE_HOST_PRIVATE_KEY" != *OPENSSH\ PRIVATE\ KEY* ]]; then + echo "If CI_STORAGE_HOST_PRIVATE_KEY is passed, it must be an SSH private key."; + exit 1; +fi diff --git a/docker/self-hosted-runner/root/entrypoint.sh b/docker/self-hosted-runner/root/entrypoint.sh new file mode 100644 index 0000000..9034987 --- /dev/null +++ b/docker/self-hosted-runner/root/entrypoint.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -u -e + +if [[ "$(whoami)" != root ]]; then + echo 'This script must be run as "root" user.'; + exit 1; +fi + +cd / + +for entrypoint in ~/entrypoint.*.sh; do + # shellcheck disable=SC1090 + [[ -f "$entrypoint" ]] && { pushd . >/dev/null; source "$entrypoint"; popd >/dev/null; } +done + +"$@" + +exec gosu user:user ~user/entrypoint.sh diff --git a/docker/self-hosted-runner/user/entrypoint.00-ci-storage-load.sh b/docker/self-hosted-runner/user/entrypoint.00-ci-storage-load.sh new file mode 100644 index 0000000..02bf504 --- /dev/null +++ b/docker/self-hosted-runner/user/entrypoint.00-ci-storage-load.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -u -e + +if [[ "$CI_STORAGE_HOST_PRIVATE_KEY" != "" ]]; then + echo "$CI_STORAGE_HOST_PRIVATE_KEY" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 +fi + +echo "$CI_STORAGE_HOST" > ci-storage-host + +if [[ "$CI_STORAGE_HOST" != "" && "$CI_STORAGE_HOST_PRIVATE_KEY" != "" ]]; then + local_dir=~/actions-runner/_work/${GH_REPOSITORY##*/}/${GH_REPOSITORY##*/} + mkdir -p "$local_dir" + ci-storage load \ + --storage-host="$CI_STORAGE_HOST" \ + --storage-dir="~/ci-storage/$GH_REPOSITORY" \ + --slot-id="?" \ + --local-dir="$local_dir" +fi diff --git a/docker/self-hosted-runner/entrypoint.sh b/docker/self-hosted-runner/user/entrypoint.05-config.sh similarity index 55% rename from docker/self-hosted-runner/entrypoint.sh rename to docker/self-hosted-runner/user/entrypoint.05-config.sh index f345990..e89db9f 100644 --- a/docker/self-hosted-runner/entrypoint.sh +++ b/docker/self-hosted-runner/user/entrypoint.05-config.sh @@ -17,60 +17,12 @@ # set -u -e -if [[ "${GH_REPOSITORY:=}" != */* ]]; then - echo "GH_REPOSITORY must be set, and the format should be {owner}/{repo}."; - exit 1; -fi -if [[ "${GH_LABELS:=}" == "" ]]; then - echo "GH_LABELS must be set."; - exit 1; -fi -if [[ "${GH_TOKEN:=}" == "" ]]; then - echo "GH_TOKEN must be set."; - exit 1; -fi -if [[ "${CI_STORAGE_HOST:=}" != "" && ! "$CI_STORAGE_HOST" =~ ^([-.[:alnum:]]+@)?[-.[:alnum:]]+(:[0-9]+)?$ ]]; then - echo "If CI_STORAGE_HOST is passed, it must be in form of [user@]host[:port]."; - exit 1; -fi -if [[ "${CI_STORAGE_HOST_PRIVATE_KEY:=}" != "" && "$CI_STORAGE_HOST_PRIVATE_KEY" != *OPENSSH\ PRIVATE\ KEY* ]]; then - echo "If CI_STORAGE_HOST_PRIVATE_KEY is passed, it must be an SSH private key."; - exit 1; -fi - -if [[ "$(whoami)" != user || ! -d ./actions-runner ]]; then - echo 'This script must be run as "user" user, and ./actions-runner/ should exist.'; - exit 1; -fi - -if [[ "$CI_STORAGE_HOST_PRIVATE_KEY" != "" ]]; then - echo "$CI_STORAGE_HOST_PRIVATE_KEY" > ~/.ssh/id_ed25519 - chmod 600 ~/.ssh/id_ed25519 -fi - -echo $$ > entrypoint.pid -echo "$CI_STORAGE_HOST" > ci-storage-host - -cd ./actions-runner - -name="ci-storage-$(hostname)" -local_dir=_work/${GH_REPOSITORY##*/}/${GH_REPOSITORY##*/} - -if [[ "$CI_STORAGE_HOST" != "" && "$CI_STORAGE_HOST_PRIVATE_KEY" != "" ]]; then - mkdir -p "$local_dir" - ci-storage load \ - --storage-host="$CI_STORAGE_HOST" \ - --storage-dir="~/ci-storage/$GH_REPOSITORY" \ - --slot-id="?" \ - --local-dir="$local_dir" -fi - token=$(gh api -X POST --jq .token "repos/$GH_REPOSITORY/actions/runners/registration-token") -./config.sh \ +cd ~/actions-runner && ./config.sh \ --unattended \ --url https://github.com/$GH_REPOSITORY \ --token "$token" \ - --name "$name" \ + --name "ci-storage-$(cat /proc/sys/kernel/random/uuid)" \ --labels "$GH_LABELS" cleanup() { @@ -83,7 +35,7 @@ cleanup() { echo "Received graceful shutdown signal, removing the runner..." while :; do token=$(gh api -X POST --jq .token "repos/$GH_REPOSITORY/actions/runners/remove-token") - ./config.sh remove --token "$token" && break + cd ~/actions-runner && ./config.sh remove --token "$token" && break sleep 5 echo "Retrying removal till the runner becomes idle and it succeeds..." done @@ -91,7 +43,3 @@ cleanup() { trap "cleanup; exit 130" INT trap "cleanup; exit 143" TERM - -"$@" - -./run.sh & wait $! diff --git a/docker/self-hosted-runner/user/entrypoint.99-run.sh b/docker/self-hosted-runner/user/entrypoint.99-run.sh new file mode 100644 index 0000000..bc8bd3b --- /dev/null +++ b/docker/self-hosted-runner/user/entrypoint.99-run.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -u -e + +cd ~/actions-runner && ./run.sh & wait $! diff --git a/docker/self-hosted-runner/user/entrypoint.sh b/docker/self-hosted-runner/user/entrypoint.sh new file mode 100644 index 0000000..e241f5d --- /dev/null +++ b/docker/self-hosted-runner/user/entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -u -e + +if [[ "$(whoami)" != user ]]; then + echo 'This script must be run as "user" user.'; + exit 1; +fi + +cd ~user + +echo $$ > .entrypoint.pid + +for entrypoint in ~/entrypoint.*.sh; do + # shellcheck disable=SC1090 + [[ -f "$entrypoint" ]] && { pushd . >/dev/null; source "$entrypoint"; popd >/dev/null; } +done