From 1bd8355460c25d950e2bee1e8781d6734b1340d8 Mon Sep 17 00:00:00 2001 From: Jakub Walczak Date: Mon, 15 Jan 2024 09:00:37 +0100 Subject: [PATCH 1/7] Prepare single workflow for docker images of all components --- .github/workflows/build-push-docker-prod.yml | 26 -------- .github/workflows/build-push-docker.yml | 32 ---------- .github/workflows/deploy-staging.yml | 65 ++++++++++++++++++++ api/Dockerfile | 2 +- datastore/Dockerfile | 2 +- drivers/README.md | 2 +- drivers/pyproject.toml | 2 +- executor/Dockerfile | 2 +- 8 files changed, 70 insertions(+), 63 deletions(-) delete mode 100644 .github/workflows/build-push-docker-prod.yml delete mode 100644 .github/workflows/build-push-docker.yml create mode 100644 .github/workflows/deploy-staging.yml diff --git a/.github/workflows/build-push-docker-prod.yml b/.github/workflows/build-push-docker-prod.yml deleted file mode 100644 index 3cc1e88..0000000 --- a/.github/workflows/build-push-docker-prod.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Build Docker image of the geodds-api component and push to the production repository - -on: - push: - tags: - - 'v*' -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Get release tag - run: echo "RELEASE_TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - - name: Login to Docker registry - run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login ${{ secrets.DOCKER_PROD_REPO_URL }} -u nologin --password-stdin - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Build and push - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile - push: true - tags: | - ${{ secrets.DOCKER_PROD_REPO_URL }}/geodds-api:${{ env.RELEASE_TAG }} - ${{ secrets.DOCKER_PROD_REPO_URL }}/geodds-api:latest diff --git a/.github/workflows/build-push-docker.yml b/.github/workflows/build-push-docker.yml deleted file mode 100644 index 6365e65..0000000 --- a/.github/workflows/build-push-docker.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Build Docker image of the geodds-api component and push to the dev repository - -on: - pull_request: - types: [opened, synchronize] - workflow_dispatch: -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set Docker image tag name - run: echo "TAG=$(date +'%Y.%m.%d.%H.%M')" >> $GITHUB_ENV - - name: Login to Scaleway Container Registry - uses: docker/login-action@v2 - with: - username: nologin - password: ${{ secrets.DOCKER_PASSWORD }} - registry: ${{ secrets.DOCKER_DEV_REPO_URL }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Build and push - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile - push: true - build-args: | - REGISTRY=${{ secrets.DOCKER_DEV_REPO_URL }} - tags: | - ${{ secrets.DOCKER_DEV_REPO_URL }}/geodds-api:${{ env.TAG }} - ${{ secrets.DOCKER_DEV_REPO_URL }}/geodds-api:latest \ No newline at end of file diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml new file mode 100644 index 0000000..b45bf4e --- /dev/null +++ b/.github/workflows/deploy-staging.yml @@ -0,0 +1,65 @@ +name: Build Docker images for geolake components and push to the repository + +on: + pull_request: + types: [opened, synchronize] + workflow_dispatch: +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set Docker image tag name + run: echo "TAG=$(date +'%Y.%m.%d.%H.%M')" >> $GITHUB_ENV + - name: Login to Scaleway Container Registry + uses: docker/login-action@v2 + with: + username: nologin + password: ${{ secrets.DOCKER_PASSWORD }} + registry: ${{ vars.DOCKER_REGISTRY }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build and push drivers + uses: docker/build-push-action@v4 + with: + context: . + file: ./drivers/Dockerfile + push: true + build-args: | + REGISTRY=${{ vars.DOCKER_REGISTRY }} + tags: | + ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:${{ env.TAG }} + ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:latest + - name: Build and push datastore component + uses: docker/build-push-action@v4 + with: + context: . + file: ./datastore/Dockerfile + push: true + build-args: | + REGISTRY=${{ vars.DOCKER_REGISTRY }} + tags: | + ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:${{ env.TAG }} + ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:latest + - name: Build and push api component + uses: docker/build-push-action@v4 + with: + context: . + file: ./api/Dockerfile + push: true + build-args: | + REGISTRY=${{ vars.DOCKER_REGISTRY }} + tags: | + ${{ vars.DOCKER_REGISTRY }}/geolake-api:${{ env.TAG }} + ${{ vars.DOCKER_REGISTRY }}/geolake-api:latest + - name: Build and push executor component + uses: docker/build-push-action@v4 + with: + context: . + file: ./executor/Dockerfile + push: true + build-args: | + REGISTRY=${{ vars.DOCKER_REGISTRY }} + tags: | + ${{ vars.DOCKER_REGISTRY }}/geolake-executor:${{ env.TAG }} + ${{ vars.DOCKER_REGISTRY }}/geolake-executor:latest \ No newline at end of file diff --git a/api/Dockerfile b/api/Dockerfile index 9ee0633..a2cfea0 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,6 +1,6 @@ ARG REGISTRY=rg.nl-ams.scw.cloud/geodds-production ARG TAG=latest -FROM $REGISTRY/geodds-datastore:$TAG +FROM $REGISTRY/geolake-datastore:$TAG WORKDIR /app COPY requirements.txt /code/requirements.txt RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt diff --git a/datastore/Dockerfile b/datastore/Dockerfile index 9ca2496..018ad5e 100644 --- a/datastore/Dockerfile +++ b/datastore/Dockerfile @@ -1,6 +1,6 @@ ARG REGISTRY=rg.nl-ams.scw.cloud/geokube-production ARG TAG=latest -FROM $REGISTRY/intake-geokube:$TAG +FROM $REGISTRY/geolake-drivers:$TAG RUN conda install -c conda-forge --yes --freeze-installed psycopg2 \ && conda clean -afy COPY requirements.txt /app/requirements.txt diff --git a/drivers/README.md b/drivers/README.md index f08349c..ed98e22 100644 --- a/drivers/README.md +++ b/drivers/README.md @@ -1,2 +1,2 @@ -# intake-geokube +# geolake-drivers GeoKube plugin for Intake \ No newline at end of file diff --git a/drivers/pyproject.toml b/drivers/pyproject.toml index ae138ac..2f0a6d5 100644 --- a/drivers/pyproject.toml +++ b/drivers/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "intake-geokube" +name = "geolake-drivers" description = "opengeokube DDS driver." requires-python = ">=3.10" readme = "README.md" diff --git a/executor/Dockerfile b/executor/Dockerfile index 6a946fd..db3cebb 100644 --- a/executor/Dockerfile +++ b/executor/Dockerfile @@ -2,7 +2,7 @@ ARG REGISTRY=rg.nl-ams.scw.cloud/geodds-production ARG TAG=latest ARG SENTINEL_USERNAME=... ARG SENTINEL_PASSWORD=... -FROM $REGISTRY/geodds-datastore:$TAG +FROM $REGISTRY/geolake-datastore:$TAG WORKDIR /app ENV SENTINEL_USERNAME=$SENTINEL_USERNAME ENV SENTINEL_PASSWORD=$SENTINEL_PASSWORD From 29d93e71ee7a93885cf4ae481e661687b83ca2d4 Mon Sep 17 00:00:00 2001 From: Jakub Walczak Date: Mon, 15 Jan 2024 09:04:53 +0100 Subject: [PATCH 2/7] Build wheel for driver --- .github/workflows/deploy-staging.yml | 12 ++++++++++++ drivers/Dockerfile | 6 +++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index b45bf4e..039f13c 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -9,6 +9,18 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Install build + run: >- + python3 -m + pip install + build + --user + - name: Build a binary wheel and a source for drivers + run: python3 -m build ./drivers - name: Set Docker image tag name run: echo "TAG=$(date +'%Y.%m.%d.%H.%M')" >> $GITHUB_ENV - name: Login to Scaleway Container Registry diff --git a/drivers/Dockerfile b/drivers/Dockerfile index d4f9e76..4980d28 100644 --- a/drivers/Dockerfile +++ b/drivers/Dockerfile @@ -3,6 +3,6 @@ ARG TAG=latest FROM $REGISTRY/geokube:$TAG RUN conda install -c conda-forge --yes --freeze-installed intake=0.6.6 RUN conda clean -afy -COPY dist/intake_geokube-1.0b0-py3-none-any.whl / -RUN pip install /intake_geokube-1.0b0-py3-none-any.whl -RUN rm /intake_geokube-1.0b0-py3-none-any.whl +COPY dist/geolake_drivers-1.0b0-py3-none-any.whl / +RUN pip install /geolake_drivers-1.0b0-py3-none-any.whl +RUN rm /geolake_drivers-1.0b0-py3-none-any.whl From 97b1ef331b8cc06588271fa25b61e8d9eeaa6d67 Mon Sep 17 00:00:00 2001 From: Jakub Walczak Date: Mon, 15 Jan 2024 09:15:46 +0100 Subject: [PATCH 3/7] Update path for intake wheel in Docker use --- .github/workflows/deploy-staging.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 039f13c..8bf91bf 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -34,7 +34,7 @@ jobs: - name: Build and push drivers uses: docker/build-push-action@v4 with: - context: . + context: ./drivers file: ./drivers/Dockerfile push: true build-args: | From bdb77195972a3f280d78418174b79cfac247fb2d Mon Sep 17 00:00:00 2001 From: Jakub Walczak Date: Mon, 15 Jan 2024 09:27:00 +0100 Subject: [PATCH 4/7] Add action for production --- .github/workflows/build-production.yml | 79 +++++++++++++++++++ .../{deploy-staging.yml => build-staging.yml} | 0 2 files changed, 79 insertions(+) create mode 100644 .github/workflows/build-production.yml rename .github/workflows/{deploy-staging.yml => build-staging.yml} (100%) diff --git a/.github/workflows/build-production.yml b/.github/workflows/build-production.yml new file mode 100644 index 0000000..2b04b9a --- /dev/null +++ b/.github/workflows/build-production.yml @@ -0,0 +1,79 @@ +name: Build Docker images for geolake components and push to the repository + +on: + push: + tags: + - 'v*' +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Install build + run: >- + python3 -m + pip install + build + --user + - name: Build a binary wheel and a source for drivers + run: python3 -m build ./drivers + - name: Set Docker image tag name + run: echo "TAG=$(date +'%Y.%m.%d.%H.%M')" >> $GITHUB_ENV + - name: Login to Scaleway Container Registry + uses: docker/login-action@v2 + with: + username: nologin + password: ${{ secrets.DOCKER_PASSWORD }} + registry: ${{ vars.DOCKER_REGISTRY }} + - name: Get release tag + run: echo "RELEASE_TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build and push drivers + uses: docker/build-push-action@v4 + with: + context: ./drivers + file: ./drivers/Dockerfile + push: true + build-args: | + REGISTRY=${{ vars.DOCKER_REGISTRY }} + tags: | + ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:${{ env.RELEASE_TAG }} + ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:latest + - name: Build and push datastore component + uses: docker/build-push-action@v4 + with: + context: . + file: ./datastore/Dockerfile + push: true + build-args: | + REGISTRY=${{ vars.DOCKER_REGISTRY }} + tags: | + ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:${{ env.RELEASE_TAG }} + ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:latest + - name: Build and push api component + uses: docker/build-push-action@v4 + with: + context: . + file: ./api/Dockerfile + push: true + build-args: | + REGISTRY=${{ vars.DOCKER_REGISTRY }} + tags: | + ${{ vars.DOCKER_REGISTRY }}/geolake-api:${{ env.RELEASE_TAG }} + ${{ vars.DOCKER_REGISTRY }}/geolake-api:latest + - name: Build and push executor component + uses: docker/build-push-action@v4 + with: + context: . + file: ./executor/Dockerfile + push: true + build-args: | + REGISTRY=${{ vars.DOCKER_REGISTRY }} + tags: | + ${{ vars.DOCKER_REGISTRY }}/geolake-executor:${{ env.RELEASE_TAG }} + ${{ vars.DOCKER_REGISTRY }}/geolake-executor:latest \ No newline at end of file diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/build-staging.yml similarity index 100% rename from .github/workflows/deploy-staging.yml rename to .github/workflows/build-staging.yml From 5f42f3d3dcb2997bd91d0a5de89db35a490f55ae Mon Sep 17 00:00:00 2001 From: Jakub Walczak Date: Mon, 15 Jan 2024 09:33:43 +0100 Subject: [PATCH 5/7] Update docker context --- .github/workflows/build-production.yml | 6 +++--- .github/workflows/build-staging.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-production.yml b/.github/workflows/build-production.yml index 2b04b9a..608c92b 100644 --- a/.github/workflows/build-production.yml +++ b/.github/workflows/build-production.yml @@ -47,7 +47,7 @@ jobs: - name: Build and push datastore component uses: docker/build-push-action@v4 with: - context: . + context: ./datastore file: ./datastore/Dockerfile push: true build-args: | @@ -58,7 +58,7 @@ jobs: - name: Build and push api component uses: docker/build-push-action@v4 with: - context: . + context: ./api file: ./api/Dockerfile push: true build-args: | @@ -69,7 +69,7 @@ jobs: - name: Build and push executor component uses: docker/build-push-action@v4 with: - context: . + context: ./executor file: ./executor/Dockerfile push: true build-args: | diff --git a/.github/workflows/build-staging.yml b/.github/workflows/build-staging.yml index 8bf91bf..7c16ff2 100644 --- a/.github/workflows/build-staging.yml +++ b/.github/workflows/build-staging.yml @@ -45,7 +45,7 @@ jobs: - name: Build and push datastore component uses: docker/build-push-action@v4 with: - context: . + context: ./datastore file: ./datastore/Dockerfile push: true build-args: | @@ -56,7 +56,7 @@ jobs: - name: Build and push api component uses: docker/build-push-action@v4 with: - context: . + context: ./api file: ./api/Dockerfile push: true build-args: | @@ -67,7 +67,7 @@ jobs: - name: Build and push executor component uses: docker/build-push-action@v4 with: - context: . + context: ./executor file: ./executor/Dockerfile push: true build-args: | From f1771da5b2c6bd463cbfe3a6f0b012d17e811635 Mon Sep 17 00:00:00 2001 From: Valentina Scardigno Date: Tue, 16 Jan 2024 10:34:05 +0000 Subject: [PATCH 6/7] Fix variable name in staging --- .github/workflows/build-staging.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-staging.yml b/.github/workflows/build-staging.yml index 7c16ff2..288f6e2 100644 --- a/.github/workflows/build-staging.yml +++ b/.github/workflows/build-staging.yml @@ -28,7 +28,7 @@ jobs: with: username: nologin password: ${{ secrets.DOCKER_PASSWORD }} - registry: ${{ vars.DOCKER_REGISTRY }} + registry: ${{ vars.STAGING_DOCKER_REGISTRY }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Build and push drivers @@ -38,10 +38,10 @@ jobs: file: ./drivers/Dockerfile push: true build-args: | - REGISTRY=${{ vars.DOCKER_REGISTRY }} + REGISTRY=${{ vars.STAGING_DOCKER_REGISTRY }} tags: | - ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:${{ env.TAG }} - ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:latest + ${{ vars.STAGING_DOCKER_REGISTRY }}/geolake-drivers:${{ env.TAG }} + ${{ vars.STAGING_DOCKER_REGISTRY }}/geolake-drivers:latest - name: Build and push datastore component uses: docker/build-push-action@v4 with: @@ -49,10 +49,10 @@ jobs: file: ./datastore/Dockerfile push: true build-args: | - REGISTRY=${{ vars.DOCKER_REGISTRY }} + REGISTRY=${{ vars.STAGING_DOCKER_REGISTRY }} tags: | - ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:${{ env.TAG }} - ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:latest + ${{ vars.STAGING_DOCKER_REGISTRY }}/geolake-datastore:${{ env.TAG }} + ${{ vars.STAGING_DOCKER_REGISTRY }}/geolake-datastore:latest - name: Build and push api component uses: docker/build-push-action@v4 with: @@ -60,10 +60,10 @@ jobs: file: ./api/Dockerfile push: true build-args: | - REGISTRY=${{ vars.DOCKER_REGISTRY }} + REGISTRY=${{ vars.STAGING_DOCKER_REGISTRY }} tags: | - ${{ vars.DOCKER_REGISTRY }}/geolake-api:${{ env.TAG }} - ${{ vars.DOCKER_REGISTRY }}/geolake-api:latest + ${{ vars.STAGING_DOCKER_REGISTRY }}/geolake-api:${{ env.TAG }} + ${{ vars.STAGING_DOCKER_REGISTRY }}/geolake-api:latest - name: Build and push executor component uses: docker/build-push-action@v4 with: @@ -71,7 +71,7 @@ jobs: file: ./executor/Dockerfile push: true build-args: | - REGISTRY=${{ vars.DOCKER_REGISTRY }} + REGISTRY=${{ vars.STAGING_DOCKER_REGISTRY }} tags: | - ${{ vars.DOCKER_REGISTRY }}/geolake-executor:${{ env.TAG }} - ${{ vars.DOCKER_REGISTRY }}/geolake-executor:latest \ No newline at end of file + ${{ vars.STAGING_DOCKER_REGISTRY }}/geolake-executor:${{ env.TAG }} + ${{ vars.STAGING_DOCKER_REGISTRY }}/geolake-executor:latest \ No newline at end of file From a3a3e999d3ea44346d75936b87240eb512c8e544 Mon Sep 17 00:00:00 2001 From: Valentina Scardigno Date: Tue, 23 Jan 2024 14:11:27 +0000 Subject: [PATCH 7/7] Change var name for registry --- .github/workflows/build-staging.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-staging.yml b/.github/workflows/build-staging.yml index 288f6e2..7c16ff2 100644 --- a/.github/workflows/build-staging.yml +++ b/.github/workflows/build-staging.yml @@ -28,7 +28,7 @@ jobs: with: username: nologin password: ${{ secrets.DOCKER_PASSWORD }} - registry: ${{ vars.STAGING_DOCKER_REGISTRY }} + registry: ${{ vars.DOCKER_REGISTRY }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Build and push drivers @@ -38,10 +38,10 @@ jobs: file: ./drivers/Dockerfile push: true build-args: | - REGISTRY=${{ vars.STAGING_DOCKER_REGISTRY }} + REGISTRY=${{ vars.DOCKER_REGISTRY }} tags: | - ${{ vars.STAGING_DOCKER_REGISTRY }}/geolake-drivers:${{ env.TAG }} - ${{ vars.STAGING_DOCKER_REGISTRY }}/geolake-drivers:latest + ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:${{ env.TAG }} + ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:latest - name: Build and push datastore component uses: docker/build-push-action@v4 with: @@ -49,10 +49,10 @@ jobs: file: ./datastore/Dockerfile push: true build-args: | - REGISTRY=${{ vars.STAGING_DOCKER_REGISTRY }} + REGISTRY=${{ vars.DOCKER_REGISTRY }} tags: | - ${{ vars.STAGING_DOCKER_REGISTRY }}/geolake-datastore:${{ env.TAG }} - ${{ vars.STAGING_DOCKER_REGISTRY }}/geolake-datastore:latest + ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:${{ env.TAG }} + ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:latest - name: Build and push api component uses: docker/build-push-action@v4 with: @@ -60,10 +60,10 @@ jobs: file: ./api/Dockerfile push: true build-args: | - REGISTRY=${{ vars.STAGING_DOCKER_REGISTRY }} + REGISTRY=${{ vars.DOCKER_REGISTRY }} tags: | - ${{ vars.STAGING_DOCKER_REGISTRY }}/geolake-api:${{ env.TAG }} - ${{ vars.STAGING_DOCKER_REGISTRY }}/geolake-api:latest + ${{ vars.DOCKER_REGISTRY }}/geolake-api:${{ env.TAG }} + ${{ vars.DOCKER_REGISTRY }}/geolake-api:latest - name: Build and push executor component uses: docker/build-push-action@v4 with: @@ -71,7 +71,7 @@ jobs: file: ./executor/Dockerfile push: true build-args: | - REGISTRY=${{ vars.STAGING_DOCKER_REGISTRY }} + REGISTRY=${{ vars.DOCKER_REGISTRY }} tags: | - ${{ vars.STAGING_DOCKER_REGISTRY }}/geolake-executor:${{ env.TAG }} - ${{ vars.STAGING_DOCKER_REGISTRY }}/geolake-executor:latest \ No newline at end of file + ${{ vars.DOCKER_REGISTRY }}/geolake-executor:${{ env.TAG }} + ${{ vars.DOCKER_REGISTRY }}/geolake-executor:latest \ No newline at end of file