Overhaul CI, use reusable workflows (#830)

* ci: overhaul, use reusable workflows * fix: json2osm import from osm-fieldwork
hotosm · Sep 20, 2023 · 890c268 · 890c268
1 parent 0356f78
commit 890c268
Show file tree

Hide file tree

Showing 16 changed files with 349 additions and 190 deletions.
diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml
@@ -13,97 +13,105 @@ on:
   # Allow manual trigger
   workflow_dispatch:
 
-env:
-  REGISTRY: ghcr.io
-  GIT_BRANCH: ${{ github.ref_name }}
-
 jobs:
-  build-and-push-images:
+  pytest:
+    uses: ./.github/workflows/r-pytest.yml
+    with:
+      image_tag: ci-${{ github.ref_name }}
+
+  frontend-main-tests:
+    uses: ./.github/workflows/r-frontend_tests.yml
+
+  extract-versions:
+    needs:
+      - pytest
+      - frontend-main-tests
+    uses: ./.github/workflows/r-extract_versions.yml
+
+  backend-build:
+    uses: ./.github/workflows/r-build_backend.yml
+    needs: extract-versions
+    with:
+      api_version: ${{ needs.extract-versions.outputs.api_version }}
+      build_target: prod
+      image_tags: |
+        "ghcr.io/hotosm/fmtm/backend:${{ needs.extract-versions.outputs.api_version }}-${{ github.ref_name }}"
+        "ghcr.io/hotosm/fmtm/backend:latest"
+
+  frontend-main-build:
+    uses: ./.github/workflows/r-build_frontend.yml
+    needs: extract-versions
+    with:
+      environment: ${{ github.ref_name }}
+      name: main
+      app_version: ${{ needs.extract-versions.outputs.frontend_main_version }}
+      build_target: prod
+      image_tags: |
+        "ghcr.io/hotosm/fmtm/frontend/main:${{ needs.extract-versions.outputs.frontend_main_version }}-${{ github.ref_name }}"
+        "ghcr.io/hotosm/fmtm/frontend/main:latest"
+
+  frontend-map-build:
+    uses: ./.github/workflows/r-build_frontend.yml
+    needs: extract-versions
+    with:
+      environment: ${{ github.ref_name }}
+      name: fmtm_openlayer_map
+      app_version: ${{ needs.extract-versions.outputs.frontend_map_version }}
+      build_target: prod
+      image_tags: |
+        "ghcr.io/hotosm/fmtm/frontend/map:${{ needs.extract-versions.outputs.frontend_map_version }}-${{ github.ref_name }}"
+        "ghcr.io/hotosm/fmtm/frontend/map:latest"
+
+  smoke-test-backend:
     runs-on: ubuntu-latest
+    needs:
+      - extract-versions
+      - backend-build
     environment:
       name: ${{ github.ref_name }}
-    permissions:
-      contents: read
-      packages: write
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
 
-      - name: Log in to the Container registry
-        uses: docker/login-action@v2
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Persist env vars
-        run: echo "${{ secrets.DOTENV }}" >> $GITHUB_ENV
-
-      - name: Extract api version
-        id: extract_api_version
+      - name: Environment to .env
+        env:
+          API_VERSION: ${{ needs.extract-versions.outputs.api_version }}
+          FRONTEND_MAIN_VERSION: ${{ needs.extract-versions.outputs.frontend_main_version }}
+          FRONTEND_MAP_VERSION: ${{ needs.extract-versions.outputs.frontend_map_version }}
         run: |
-          cd src/backend
-          echo "API_VERSION=$(python -c 'from app.__version__ import __version__; print(__version__)')" >> $GITHUB_ENV
+          echo "${{ secrets.DOTENV }}" > .env
+          echo "API_VERSION=${API_VERSION}" >> .env
+          echo "FRONTEND_MAIN_VERSION=${FRONTEND_MAIN_VERSION}" >> .env
+          echo "FRONTEND_MAP_VERSION=${FRONTEND_MAP_VERSION}" >> .env
 
-      - name: Extract frontend versions
-        id: extract_frontend_versions
+      - name: Backend smoke test
         run: |
-          cd src/frontend
-          echo "FRONTEND_MAIN_VERSION=$(jq -r '.version' main/package.json)" >> $GITHUB_ENV
-
-      - name: Build and push backend
-        uses: docker/build-push-action@v4
-        with:
-          context: src/backend
-          target: prod
-          push: true
-          tags: |
-            "ghcr.io/hotosm/fmtm/backend:${{ env.API_VERSION }}-${{ github.ref_name }}"
-            "ghcr.io/hotosm/fmtm/backend:latest"
-          build-args: |
-            APP_VERSION=${{ env.API_VERSION }}
-
-      - name: Build and push frontend main
-        uses: docker/build-push-action@v4
-        with:
-          context: src/frontend
-          file: src/frontend/prod.dockerfile
-          push: true
-          tags: "ghcr.io/hotosm/fmtm/frontend/main:${{ env.FRONTEND_MAIN_VERSION }}-${{ github.ref_name }}"
-          build-args: |
-            APP_NAME=main
-            APP_VERSION=${{ env.FRONTEND_MAIN_VERSION }}
-            API_URL=${{ env.URL_SCHEME }}://${{ env.API_URL }}
-            FRONTEND_MAIN_URL=${{ env.URL_SCHEME }}://${{ env.FRONTEND_MAIN_URL }}
+          docker compose --file docker-compose.deploy.yml pull api
+          docker compose --file docker-compose.deploy.yml up api --exit-code-from api
 
   deploy-containers:
     runs-on: ubuntu-latest
-    needs: build-and-push-images
+    needs:
+      - extract-versions
+      - smoke-test-backend
     environment:
       name: ${{ github.ref_name }}
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
 
-      - name: Extract api version
-        id: extract_api_version
-        run: |
-          cd src/backend
-          echo "API_VERSION=$(python -c 'from app.__version__ import __version__; print(__version__)')" >> $GITHUB_OUTPUT
-
-      - name: Extract frontend versions
-        id: extract_frontend_versions
-        run: |
-          cd src/frontend
-          echo "FRONTEND_MAIN_VERSION=$(jq -r '.version' main/package.json)" >> $GITHUB_OUTPUT
-
       - name: Environment to .env
+        env:
+          API_VERSION: ${{ needs.extract-versions.outputs.api_version }}
+          FRONTEND_MAIN_VERSION: ${{ needs.extract-versions.outputs.frontend_main_version }}
+          FRONTEND_MAP_VERSION: ${{ needs.extract-versions.outputs.frontend_map_version }}
         run: |
           echo "${{ secrets.DOTENV }}" > .env
-          echo "API_VERSION=${{ steps.extract_api_version.outputs.API_VERSION }}" >> .env
-          echo "FRONTEND_MAIN_VERSION=${{ steps.extract_frontend_versions.outputs.FRONTEND_MAIN_VERSION }}" >> .env
+          echo "API_VERSION=${API_VERSION}" >> .env
+          echo "FRONTEND_MAIN_VERSION=${FRONTEND_MAIN_VERSION}" >> .env
+          echo "FRONTEND_MAP_VERSION=${FRONTEND_MAP_VERSION}" >> .env
 
       - uses: webfactory/[email protected]
         with:

diff --git a/.github/workflows/build_ci_img.yml b/.github/workflows/build_ci_img.yml
@@ -0,0 +1,29 @@
+name: Build CI Img
+
+on:
+  # Push includes PR merge
+  push:
+    branches:
+      - main
+      - staging
+      - development
+    paths:
+      # Workflow is triggered only if deps change
+      - "src/backend/pyproject.toml"
+      - "src/backend/Dockerfile"
+  # Allow manual trigger
+  workflow_dispatch:
+
+jobs:
+  extract-versions:
+    uses: ./.github/workflows/r-extract_versions.yml
+
+  backend-ci-build:
+    uses: ./.github/workflows/r-build_backend.yml
+    needs: [extract-versions]
+    with:
+      api_version: ${{ needs.extract-versions.outputs.api_version }}
+      build_target: ci
+      image_tags: |
+        "ghcr.io/hotosm/fmtm/backend:${{ needs.extract-versions.outputs.api_version }}-ci-${{ github.ref_name }}"
+        "ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }}"
diff --git a/.github/workflows/odk_image_build.yml → .github/workflows/build_odk_imgs.yml b/.github/workflows/odk_image_build.yml → .github/workflows/build_odk_imgs.yml
@@ -4,20 +4,13 @@ on:
   # Push includes PR merge
   push:
     branches:
-      - main
-      - staging
       - development
-      - "*-development-*"
     paths:
       # Workflow is triggered only if odkcentral dir changes
       - "odkcentral/**"
   # Allow manual trigger
   workflow_dispatch:
 
-env:
-  REGISTRY: ghcr.io
-  ODK_CENTRAL_VERSION: v2023.2.1
-
 jobs:
   build-and-push-images:
     runs-on: ubuntu-latest
@@ -32,7 +25,7 @@ jobs:
       - name: Log in to the Container registry
         uses: docker/login-action@v2
         with:
-          registry: ${{ env.REGISTRY }}
+          registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
@@ -41,13 +34,17 @@ jobs:
         with:
           context: odkcentral/api
           push: true
-          tags: "ghcr.io/hotosm/fmtm/odkcentral:${{ env.ODK_CENTRAL_VERSION }}"
+          tags: |
+            "ghcr.io/hotosm/fmtm/odkcentral:${{ vars.ODK_CENTRAL_VERSION }}"
+            "ghcr.io/hotosm/fmtm/odkcentral:latest"
           build-args: |
-            ODK_CENTRAL_VERSION=${{ env.ODK_CENTRAL_VERSION }}
+            ODK_CENTRAL_VERSION=${{ vars.ODK_CENTRAL_VERSION }}
 
       - name: Build and push odkcentral proxy
         uses: docker/build-push-action@v4
         with:
           context: odkcentral/proxy
           push: true
-          tags: "ghcr.io/hotosm/fmtm/odkcentral-proxy:latest"
+          tags: |
+            "ghcr.io/hotosm/fmtm/odkcentral-proxy:${{ vars.ODK_CENTRAL_VERSION }}"
+            "ghcr.io/hotosm/fmtm/odkcentral-proxy:latest"
diff --git a/.github/workflows/ci_img_build.yml b/.github/workflows/ci_img_build.yml
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -11,6 +11,8 @@ on:
 
 jobs:
   pytest:
-    uses: ./.github/workflows/reusable/pytest.yml
-  frontend-tests:
-    uses: ./.github/workflows/reusable/frontend_tests.yml
+    uses: ./.github/workflows/r-pytest.yml
+    with:
+      image_tag: ci-${{ github.base_ref }}
+  frontend-main-tests:
+    uses: ./.github/workflows/r-frontend_tests.yml
diff --git a/.github/workflows/pr_pytest.yml b/.github/workflows/pr_pytest.yml
diff --git a/.github/workflows/r-build_backend.yml b/.github/workflows/r-build_backend.yml
@@ -0,0 +1,44 @@
+name: Build Backend Imgs
+
+on:
+  workflow_call:
+    paths:
+      - src/backend/**
+    inputs:
+      api_version:
+        required: true
+        type: string
+      build_target:
+        required: true
+        type: string
+      image_tags:
+        required: true
+        type: string
+
+jobs:
+  build-and-push-images:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push backend
+        uses: docker/build-push-action@v4
+        with:
+          context: src/backend
+          target: ${{ inputs.build_target }}
+          push: true
+          tags: ${{ inputs.image_tags }}
+          build-args: |
+            APP_VERSION=${{ inputs.api_version }}