From 903b5903b32a8970d0621f9bc3c9375fe361b098 Mon Sep 17 00:00:00 2001 From: yihuang Date: Thu, 20 Jun 2024 09:58:59 +0800 Subject: [PATCH] Problem: testground image not pushed to registry (#1479) * Problem: testground image not pushed to registry Solution: - build the current version of the image in root flake - push to github container registry * fix entrypoint * check image validity * cleanup * test image --- .github/workflows/container.yml | 52 ++++++++ flake.lock | 18 +-- flake.nix | 4 +- nix/testground-image.nix | 15 +++ testground/benchmark/benchmark/context.py | 11 +- testground/benchmark/benchmark/main.py | 8 ++ testground/benchmark/flake.lock | 145 +++------------------- testground/benchmark/flake.nix | 40 ++---- testground/benchmark/overlay.nix | 45 +++++++ testground/benchmark/pyproject.toml | 2 +- 10 files changed, 169 insertions(+), 171 deletions(-) create mode 100644 .github/workflows/container.yml create mode 100644 nix/testground-image.nix create mode 100644 testground/benchmark/overlay.nix diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml new file mode 100644 index 0000000000..af2186af43 --- /dev/null +++ b/.github/workflows/container.yml @@ -0,0 +1,52 @@ +name: Push Testground Image + +on: + push: + branches: + - main + tags: + - "v*.*.*" + +env: + IMAGE_NAME: cronos-testground + +jobs: + + push: + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v23 + with: + nix_path: nixpkgs=channel:nixos-24.05 + extra_nix_config: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + - uses: cachix/cachix-action@v12 + with: + name: cronos + signingKey: "${{ secrets.CACHIX_SIGNING_KEY }}" + - name: build and push image + run: | + # login to ghcr.io + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + [ "$VERSION" == "main" ] && VERSION=latest + echo "VERSION: $VERSION" + + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + echo "IMAGE_ID: $IMAGE_ID" + + BUILD_TAG="$(nix eval --raw .#testground-image.imageTag)" + echo "BUILD_TAG: $BUILD_TAG" + + docker load -i "$(nix build --no-link --print-out-paths .#testground-image)" + docker run --rm -e TEST_CASE=info $IMAGE_NAME:$BUILD_TAG + docker tag $IMAGE_NAME:$BUILD_TAG $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION diff --git a/flake.lock b/flake.lock index dc81b1613a..26a3262253 100644 --- a/flake.lock +++ b/flake.lock @@ -77,11 +77,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1708341091, - "narHash": "sha256-3R7doGV1AoB5VKFifEd5elj8t4cld6VpJRpn9NaYr1Y=", + "lastModified": 1716715802, + "narHash": "sha256-usk0vE7VlxPX8jOavrtpOqphdfqEQpf9lgedlY/r66c=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "86ef6bd96b6279e1a4a53236d341f5df1ede3803", + "rev": "e2dd4e18cc1c7314e24154331bae07df76eb582f", "type": "github" }, "original": { @@ -104,11 +104,11 @@ "treefmt-nix": "treefmt-nix" }, "locked": { - "lastModified": 1708589824, - "narHash": "sha256-2GOiFTkvs5MtVF65sC78KNVxQSmsxtk0WmV1wJ9V2ck=", + "lastModified": 1718745582, + "narHash": "sha256-TFlVP4YXg6n+MbP/Iv/RIwqvRKuV9KA1JAPihoFmPfo=", "owner": "nix-community", "repo": "poetry2nix", - "rev": "3c92540611f42d3fb2d0d084a6c694cd6544b609", + "rev": "48e7ed4ef7832efa5a5558e573986c4128fc478f", "type": "github" }, "original": { @@ -148,11 +148,11 @@ ] }, "locked": { - "lastModified": 1708335038, - "narHash": "sha256-ETLZNFBVCabo7lJrpjD6cAbnE11eDOjaQnznmg/6hAE=", + "lastModified": 1718522839, + "narHash": "sha256-ULzoKzEaBOiLRtjeY3YoGFJMwWSKRYOic6VNw2UyTls=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "e504621290a1fd896631ddbc5e9c16f4366c9f65", + "rev": "68eb1dc333ce82d0ab0c0357363ea17c31ea1f81", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 5fea04c927..b1fdcda4a3 100644 --- a/flake.nix +++ b/flake.nix @@ -37,7 +37,7 @@ in rec { packages = pkgs.cronos-matrix // { - inherit (pkgs) rocksdb; + inherit (pkgs) rocksdb testground-image; }; apps = { cronosd = mkApp packages.cronosd; @@ -72,12 +72,14 @@ (import ./nix/build_overlay.nix) poetry2nix.overlays.default gomod2nix.overlays.default + (import ./testground/benchmark/overlay.nix) (final: super: { go = super.go_1_22; test-env = final.callPackage ./nix/testenv.nix { }; cronos-matrix = final.callPackage ./nix/cronos-matrix.nix { bundle-exe = final.pkgsBuildBuild.callPackage nix-bundle-exe { }; }; + testground-image = final.callPackage ./nix/testground-image.nix { }; }) ]; }; diff --git a/nix/testground-image.nix b/nix/testground-image.nix new file mode 100644 index 0000000000..e1c3d3a943 --- /dev/null +++ b/nix/testground-image.nix @@ -0,0 +1,15 @@ +{ dockerTools, cronos-matrix, testground-testcase }: +dockerTools.buildLayeredImage { + name = "cronos-testground"; + contents = [ + testground-testcase + cronos-matrix.cronosd + ]; + config = { + Expose = [ 9090 26657 26656 1317 26658 26660 26659 30000 ]; + Cmd = [ "/bin/testground-testcase" ]; + Env = [ + "PYTHONUNBUFFERED=1" + ]; + }; +} diff --git a/testground/benchmark/benchmark/context.py b/testground/benchmark/benchmark/context.py index 014527c8b8..f44cfb062a 100644 --- a/testground/benchmark/benchmark/context.py +++ b/testground/benchmark/benchmark/context.py @@ -12,7 +12,13 @@ def __init__(self, params: RunParams = None): if params is None: params = run_params() self.params = params - self.sync = SyncService(params) + self._sync = None + + @property + def sync(self) -> SyncService: + if self._sync is None: + self._sync = SyncService(self.params) + return self._sync def init_common(self): self.wait_network_ready() @@ -107,4 +113,5 @@ def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): - self.sync.close() + if self._sync is not None: + self._sync.close() diff --git a/testground/benchmark/benchmark/main.py b/testground/benchmark/benchmark/main.py index d968753338..d321845028 100644 --- a/testground/benchmark/benchmark/main.py +++ b/testground/benchmark/benchmark/main.py @@ -46,8 +46,16 @@ def entrypoint(ctx: Context): ctx.record_success() +def info(ctx: Context): + """ + Print the runtime configuration, mainly to check if the image is built successfully. + """ + print(ctx.params) + + TEST_CASES = { "entrypoint": entrypoint, + "info": info, } diff --git a/testground/benchmark/flake.lock b/testground/benchmark/flake.lock index 22feaa71c5..baa53c2408 100644 --- a/testground/benchmark/flake.lock +++ b/testground/benchmark/flake.lock @@ -1,32 +1,5 @@ { "nodes": { - "cronos": { - "inputs": { - "flake-utils": [ - "flake-utils" - ], - "gomod2nix": "gomod2nix", - "nix-bundle-exe": "nix-bundle-exe", - "nixpkgs": [ - "nixpkgs" - ], - "poetry2nix": "poetry2nix" - }, - "locked": { - "lastModified": 1716448317, - "narHash": "sha256-MY2S2YyKWvyj2z086VaFcrWpjjMzb7R10Rprq3zeE+o=", - "owner": "crypto-org-chain", - "repo": "cronos", - "rev": "b2a47a2bdf95dfe8b5d81428f7024ef36b81f32e", - "type": "github" - }, - "original": { - "owner": "crypto-org-chain", - "ref": "main", - "repo": "cronos", - "type": "github" - } - }, "flake-utils": { "inputs": { "systems": "systems" @@ -45,69 +18,9 @@ "type": "github" } }, - "flake-utils_2": { - "inputs": { - "systems": "systems_3" - }, - "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "gomod2nix": { - "inputs": { - "flake-utils": [ - "cronos", - "flake-utils" - ], - "nixpkgs": [ - "cronos", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1705314449, - "narHash": "sha256-yfQQ67dLejP0FLK76LKHbkzcQqNIrux6MFe32MMFGNQ=", - "owner": "nix-community", - "repo": "gomod2nix", - "rev": "30e3c3a9ec4ac8453282ca7f67fca9e1da12c3e6", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "gomod2nix", - "type": "github" - } - }, - "nix-bundle-exe": { - "flake": false, - "locked": { - "lastModified": 1660176694, - "narHash": "sha256-cJGZ/3CjVkoyk1W9mFVs6P/5LbJ8C+42chGYiB/wB/A=", - "owner": "3noch", - "repo": "nix-bundle-exe", - "rev": "91416cec283a33ae3448aacdc5cabdece9c08793", - "type": "github" - }, - "original": { - "owner": "3noch", - "repo": "nix-bundle-exe", - "type": "github" - } - }, "nix-github-actions": { "inputs": { "nixpkgs": [ - "cronos", "poetry2nix", "nixpkgs" ] @@ -127,22 +40,6 @@ } }, "nixpkgs": { - "locked": { - "lastModified": 1716924492, - "narHash": "sha256-9/Ro5/MfI+PNMF8jzh7+gXDPUHeOzL1e/iw3p4z6Ttc=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "4ae13643e7f2cd4bc6555fce074865d9d14e7c24", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-unstable-small", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_2": { "locked": { "lastModified": 1716715802, "narHash": "sha256-usk0vE7VlxPX8jOavrtpOqphdfqEQpf9lgedlY/r66c=", @@ -160,18 +57,22 @@ }, "poetry2nix": { "inputs": { - "flake-utils": "flake-utils", + "flake-utils": [ + "flake-utils" + ], "nix-github-actions": "nix-github-actions", - "nixpkgs": "nixpkgs", + "nixpkgs": [ + "nixpkgs" + ], "systems": "systems_2", "treefmt-nix": "treefmt-nix" }, "locked": { - "lastModified": 1716986791, - "narHash": "sha256-T9Lra3MFVrto23zj2dDZ6bgxz/9gp/wh2x7QhFkpWEs=", + "lastModified": 1718745582, + "narHash": "sha256-TFlVP4YXg6n+MbP/Iv/RIwqvRKuV9KA1JAPihoFmPfo=", "owner": "nix-community", "repo": "poetry2nix", - "rev": "1f32bb76c8e2502d896d38dc08433ac8d1e27468", + "rev": "48e7ed4ef7832efa5a5558e573986c4128fc478f", "type": "github" }, "original": { @@ -182,9 +83,9 @@ }, "root": { "inputs": { - "cronos": "cronos", - "flake-utils": "flake-utils_2", - "nixpkgs": "nixpkgs_2" + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "poetry2nix": "poetry2nix" } }, "systems": { @@ -216,35 +117,19 @@ "type": "indirect" } }, - "systems_3": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - }, "treefmt-nix": { "inputs": { "nixpkgs": [ - "cronos", "poetry2nix", "nixpkgs" ] }, "locked": { - "lastModified": 1708335038, - "narHash": "sha256-ETLZNFBVCabo7lJrpjD6cAbnE11eDOjaQnznmg/6hAE=", + "lastModified": 1718522839, + "narHash": "sha256-ULzoKzEaBOiLRtjeY3YoGFJMwWSKRYOic6VNw2UyTls=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "e504621290a1fd896631ddbc5e9c16f4366c9f65", + "rev": "68eb1dc333ce82d0ab0c0357363ea17c31ea1f81", "type": "github" }, "original": { diff --git a/testground/benchmark/flake.nix b/testground/benchmark/flake.nix index c75f5a1496..e4b9fabafb 100644 --- a/testground/benchmark/flake.nix +++ b/testground/benchmark/flake.nix @@ -2,15 +2,14 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; flake-utils.url = "github:numtide/flake-utils"; - cronos = { - url = "github:crypto-org-chain/cronos/main"; + poetry2nix = { + url = "github:nix-community/poetry2nix"; inputs.nixpkgs.follows = "nixpkgs"; inputs.flake-utils.follows = "flake-utils"; - inputs.poetry2nix.url = "github:nix-community/poetry2nix"; }; }; - outputs = { self, nixpkgs, flake-utils, cronos }: + outputs = { self, nixpkgs, flake-utils, poetry2nix }: let overrides = { lib, poetry2nix }: poetry2nix.overrides.withDefaults (self: super: @@ -50,42 +49,27 @@ overrides = overrides { inherit lib poetry2nix; }; }; - image = { dockerTools, cronos-matrix, test-env, benchmark }: - dockerTools.buildLayeredImage { - name = "cronos-testground"; - contents = [ - benchmark - test-env - cronos-matrix.cronosd - ]; - config = { - Expose = [ 9090 26657 26656 1317 26658 26660 26659 30000 ]; - Cmd = [ "/bin/benchmark" ]; - Env = [ - "PYTHONUNBUFFERED=1" - ]; - }; - }; in (flake-utils.lib.eachDefaultSystem (system: let pkgs = import nixpkgs { inherit system; - overlays = cronos.overlays.default ++ [ - (final: _: { - benchmark = final.callPackage benchmark { }; - benchmark-env = final.callPackage benchmark-env { }; - benchmark-image = final.callPackage image { }; - }) + overlays = [ + poetry2nix.overlays.default + (import ./overlay.nix) ]; config = { }; }; in rec { - packages.default = pkgs.benchmark-image; + packages.default = pkgs.testground-testcase; + apps.default = { + type = "app"; + program = "${pkgs.testground-testcase}/bin/testground-testcase"; + }; devShells.default = pkgs.mkShell { - buildInputs = [ pkgs.benchmark-env ]; + buildInputs = [ pkgs.testground-testcase-env ]; }; legacyPackages = pkgs; }) diff --git a/testground/benchmark/overlay.nix b/testground/benchmark/overlay.nix new file mode 100644 index 0000000000..bb37c0a1ef --- /dev/null +++ b/testground/benchmark/overlay.nix @@ -0,0 +1,45 @@ +final: _: +let + overrides = { lib, poetry2nix }: poetry2nix.overrides.withDefaults + (self: super: + let + buildSystems = { + pystarport = [ "poetry-core" ]; + durations = [ "setuptools" ]; + multitail2 = [ "setuptools" ]; + docker = [ "hatchling" "hatch-vcs" ]; + pyunormalize = [ "setuptools" ]; + }; + in + lib.mapAttrs + (attr: systems: super.${attr}.overridePythonAttrs + (old: { + nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ map (a: self.${a}) systems; + })) + buildSystems + ); + + src = nix-gitignore: nix-gitignore.gitignoreSourcePure [ + "/*" # ignore all, then add whitelists + "!/benchmark/" + "!poetry.lock" + "!pyproject.toml" + ] ./.; + + benchmark = { lib, poetry2nix, python311, nix-gitignore }: poetry2nix.mkPoetryApplication { + projectDir = src nix-gitignore; + python = python311; + overrides = overrides { inherit lib poetry2nix; }; + }; + + benchmark-env = { lib, poetry2nix, python311, nix-gitignore }: poetry2nix.mkPoetryEnv { + projectDir = src nix-gitignore; + python = python311; + overrides = overrides { inherit lib poetry2nix; }; + }; + +in +{ + testground-testcase = final.callPackage benchmark { }; + testground-testcase-env = final.callPackage benchmark-env { }; +} diff --git a/testground/benchmark/pyproject.toml b/testground/benchmark/pyproject.toml index 4133beec7a..7bde6a3907 100644 --- a/testground/benchmark/pyproject.toml +++ b/testground/benchmark/pyproject.toml @@ -23,4 +23,4 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.poetry.scripts] -benchmark = "benchmark.main:main" +testground-testcase = "benchmark.main:main"