Merge branch 'development'

hotosm · Sep 20, 2023 · 49c9d6e · 49c9d6e
2 parents 92f07fd + cad4f63
commit 49c9d6e
Show file tree

Hide file tree

Showing 200 changed files with 8,093 additions and 23,827 deletions.
diff --git a/.env.example b/.env.example
@@ -1,6 +1,7 @@
 ### copy to .env and set variables
 
 ### ODK Central ###
+ODK_CENTRAL_VERSION=v2023.2.1
 ODK_CENTRAL_URL=https://central-proxy
 ODK_CENTRAL_USER=[email protected]
 ODK_CENTRAL_PASSWD=fmtm
@@ -12,16 +13,15 @@ ODK_CENTRAL_PASSWD=fmtm
 URL_SCHEME=http
 API_URL=127.0.0.1:8000
 FRONTEND_MAIN_URL=127.0.0.1:8080
-FRONTEND_MAP_URL=127.0.0.1:8081
 # API_PREFIX=/api
 
 ### OSM ###
-OSM_CLIENT_ID=
-OSM_CLIENT_SECRET=
+OSM_CLIENT_ID=xxx
+OSM_CLIENT_SECRET=xxx
 OSM_URL=https://www.openstreetmap.org
 OSM_SCOPE=read_prefs
-OSM_LOGIN_REDIRECT_URI=http://127.0.0.1:8000/auth/callback/
-OSM_SECRET_KEY=
+OSM_LOGIN_REDIRECT_URI=http://127.0.0.1:8080/osmauth/
+OSM_SECRET_KEY=xxx
 
 ### Database (optional) ###
 CENTRAL_DB_HOST=central-db
@@ -33,3 +33,6 @@ FMTM_DB_HOST=fmtm-db
 FMTM_DB_USER=fmtm
 FMTM_DB_PASSWORD=fmtm
 FMTM_DB_NAME=fmtm
+
+### Underpass (optional) ###
+# UNDERPASS_API_URL=
diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml
@@ -9,119 +9,141 @@ on:
       - development
     paths:
       # Workflow is triggered only if src changes
-      - "src/**"
+      - src/**
   # Allow manual trigger
   workflow_dispatch:
 
-env:
-  REGISTRY: ghcr.io
-  GIT_BRANCH: ${{ github.ref_name }}
-
 jobs:
-  build-and-push-images:
+  pytest:
+    uses: ./.github/workflows/r-pytest.yml
+    with:
+      image_tag: ci-${{ github.ref_name }}
+
+  frontend-tests:
+    uses: ./.github/workflows/r-frontend_tests.yml
+
+  extract-versions:
+    needs:
+      - pytest
+      - frontend-tests
+    uses: ./.github/workflows/r-extract_versions.yml
+
+  backend-build:
+    uses: ./.github/workflows/r-build_backend.yml
+    needs: extract-versions
+    with:
+      api_version: ${{ needs.extract-versions.outputs.api_version }}
+      build_target: prod
+      image_tags: |
+        "ghcr.io/hotosm/fmtm/backend:${{ needs.extract-versions.outputs.api_version }}-${{ github.ref_name }}"
+        "ghcr.io/hotosm/fmtm/backend:latest"
+
+  frontend-main-build:
+    uses: ./.github/workflows/r-build_frontend.yml
+    needs: extract-versions
+    with:
+      environment: ${{ github.ref_name }}
+      name: main
+      app_version: ${{ needs.extract-versions.outputs.frontend_main_version }}
+      build_target: prod
+      image_tags: |
+        "ghcr.io/hotosm/fmtm/frontend/main:${{ needs.extract-versions.outputs.frontend_main_version }}-${{ github.ref_name }}"
+        "ghcr.io/hotosm/fmtm/frontend/main:latest"
+
+  smoke-test-backend:
     runs-on: ubuntu-latest
+    needs:
+      - extract-versions
+      - backend-build
     environment:
       name: ${{ github.ref_name }}
-    permissions:
-      contents: read
-      packages: write
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
 
-      - name: Log in to the Container registry
-        uses: docker/login-action@v2
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Persist env vars
-        run: echo "${{ secrets.DOTENV }}" >> $GITHUB_ENV
-
-      - name: Extract api version
-        id: extract_api_version
+      - name: Environment to .env
+        env:
+          GIT_BRANCH: ${{ github.ref_name }}
+          API_VERSION: ${{ needs.extract-versions.outputs.api_version }}
+          FRONTEND_MAIN_VERSION: ${{ needs.extract-versions.outputs.frontend_main_version }}
         run: |
-          cd src/backend
-          echo "API_VERSION=$(python -c 'from app.__version__ import __version__; print(__version__)')" >> $GITHUB_ENV
+          echo "${{ secrets.DOTENV }}" > .env
+          echo "GIT_BRANCH=${GIT_BRANCH}" >> .env
+          echo "API_VERSION=${API_VERSION}" >> .env
+          echo "FRONTEND_MAIN_VERSION=${FRONTEND_MAIN_VERSION}" >> .env
 
-      - name: Extract frontend versions
-        id: extract_frontend_versions
+      - name: Backend smoke test
         run: |
-          cd src/frontend
-          echo "FRONTEND_MAIN_VERSION=$(jq -r '.version' main/package.json)" >> $GITHUB_ENV
-          echo "FRONTEND_MAP_VERSION=$(jq -r '.version' fmtm_openlayer_map/package.json)" >> $GITHUB_ENV
+          echo "Not implemented" 
+          # source .env
+          # docker network create fmtm
+
+          # docker pull "postgis/postgis:14-3.3-alpine"
+          # docker run --rm -d \
+          #   --name=fmtm-db \
+          #   --network=fmtm \
+          #   -e POSTGRES_USER=fmtm \
+          #   -e POSTGRES_PASSWORD=fmtm \
+          #   -e POSTGRES_DB=fmtm \
+          #   "postgis/postgis:14-3.3-alpine"
+
+          # docker pull "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}"
+          # docker run --rm -d \
+          #   --network=fmtm \
+          #   -p 8080:8080 \
+          #   -e FRONTEND_MAIN_URL="http://test.com" \
+          #   -e OSM_CLIENT_ID="test" \
+          #   -e OSM_CLIENT_SECRET="test" \
+          #   -e OSM_SECRET_KEY="test" \
+          #   "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}"
+
+          # # First wait 10 seconds for API
+          # sleep 10
+          # # Check the exit status of curl and exit the job if it fails
+          # if ! curl -f http://localhost:8080/docs; then
+          #   echo "curl failed to access http://localhost:8080/docs"
+          #   exit 1
+          # fi
+
+  smoke-test-frontend:
+    runs-on: ubuntu-latest
+    needs:
+      - extract-versions
+      - frontend-main-build
+    environment:
+      name: ${{ github.ref_name }}
 
-      - name: Build and push backend
-        uses: docker/build-push-action@v4
-        with:
-          context: src/backend
-          target: prod
-          push: true
-          tags: |
-            "ghcr.io/hotosm/fmtm/backend:${{ env.API_VERSION }}-${{ github.ref_name }}"
-            "ghcr.io/hotosm/fmtm/backend:latest"
-          build-args: |
-            APP_VERSION=${{ env.API_VERSION }}
-
-      - name: Build and push frontend main
-        uses: docker/build-push-action@v4
-        with:
-          context: src/frontend
-          file: src/frontend/prod.dockerfile
-          push: true
-          tags: "ghcr.io/hotosm/fmtm/frontend/main:${{ env.FRONTEND_MAIN_VERSION }}-${{ github.ref_name }}"
-          build-args: |
-            APP_NAME=main
-            APP_VERSION=${{ env.FRONTEND_MAIN_VERSION }}
-            API_URL=${{ env.URL_SCHEME }}://${{ env.API_URL }}
-            FRONTEND_MAIN_URL=${{ env.URL_SCHEME }}://${{ env.FRONTEND_MAIN_URL }}
-            FRONTEND_MAP_URL=${{ env.URL_SCHEME }}://${{ env.FRONTEND_MAP_URL }}
-
-      - name: Build and push frontend map
-        uses: docker/build-push-action@v4
-        with:
-          context: src/frontend
-          file: src/frontend/prod.dockerfile
-          push: true
-          tags: "ghcr.io/hotosm/fmtm/frontend/map:${{ env.FRONTEND_MAP_VERSION }}-${{ github.ref_name }}"
-          build-args: |
-            APP_NAME=fmtm_openlayer_map
-            APP_VERSION=${{ env.FRONTEND_MAP_VERSION }}
-            API_URL=${{ env.URL_SCHEME }}://${{ env.API_URL }}
-            FRONTEND_MAIN_URL=${{ env.URL_SCHEME }}://${{ env.FRONTEND_MAIN_URL }}
-            FRONTEND_MAP_URL=${{ env.URL_SCHEME }}://${{ env.FRONTEND_MAP_URL }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Frontend smoke test
+        run: echo "Not implemented"
 
   deploy-containers:
     runs-on: ubuntu-latest
-    needs: build-and-push-images
+    needs:
+      - extract-versions
+      - smoke-test-backend
+      - smoke-test-frontend
     environment:
       name: ${{ github.ref_name }}
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
 
-      - name: Extract api version
-        id: extract_api_version
-        run: |
-          cd src/backend
-          echo "API_VERSION=$(python -c 'from app.__version__ import __version__; print(__version__)')" >> $GITHUB_OUTPUT
-
-      - name: Extract frontend versions
-        id: extract_frontend_versions
-        run: |
-          cd src/frontend
-          echo "FRONTEND_MAIN_VERSION=$(jq -r '.version' main/package.json)" >> $GITHUB_OUTPUT
-          echo "FRONTEND_MAP_VERSION=$(jq -r '.version' fmtm_openlayer_map/package.json)" >> $GITHUB_OUTPUT
-
       - name: Environment to .env
+        env:
+          GIT_BRANCH: ${{ github.ref_name }}
+          API_VERSION: ${{ needs.extract-versions.outputs.api_version }}
+          FRONTEND_MAIN_VERSION: ${{ needs.extract-versions.outputs.frontend_main_version }}
         run: |
           echo "${{ secrets.DOTENV }}" > .env
-          echo "API_VERSION=${{ steps.extract_api_version.outputs.API_VERSION }}" >> .env
-          echo "FRONTEND_MAIN_VERSION=${{ steps.extract_frontend_versions.outputs.FRONTEND_MAIN_VERSION }}" >> .env
-          echo "FRONTEND_MAP_VERSION=${{ steps.extract_frontend_versions.outputs.FRONTEND_MAP_VERSION }}" >> .env
+          echo "GIT_BRANCH=${GIT_BRANCH}" >> .env
+          echo "API_VERSION=${API_VERSION}" >> .env
+          echo "FRONTEND_MAIN_VERSION=${FRONTEND_MAIN_VERSION}" >> .env
 
       - uses: webfactory/[email protected]
         with:

diff --git a/.github/workflows/build_ci_img.yml b/.github/workflows/build_ci_img.yml
@@ -0,0 +1,29 @@
+name: Build CI Img
+
+on:
+  # Push includes PR merge
+  push:
+    branches:
+      - main
+      - staging
+      - development
+    paths:
+      # Workflow is triggered only if deps change
+      - "src/backend/pyproject.toml"
+      - "src/backend/Dockerfile"
+  # Allow manual trigger
+  workflow_dispatch:
+
+jobs:
+  extract-versions:
+    uses: ./.github/workflows/r-extract_versions.yml
+
+  backend-ci-build:
+    uses: ./.github/workflows/r-build_backend.yml
+    needs: [extract-versions]
+    with:
+      api_version: ${{ needs.extract-versions.outputs.api_version }}
+      build_target: ci
+      image_tags: |
+        "ghcr.io/hotosm/fmtm/backend:${{ needs.extract-versions.outputs.api_version }}-ci-${{ github.ref_name }}"
+        "ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }}"
diff --git a/.github/workflows/odk_image_build.yml → .github/workflows/build_odk_imgs.yml b/.github/workflows/odk_image_build.yml → .github/workflows/build_odk_imgs.yml
@@ -4,20 +4,13 @@ on:
   # Push includes PR merge
   push:
     branches:
-      - main
-      - staging
       - development
-      - "*-development-*"
     paths:
       # Workflow is triggered only if odkcentral dir changes
       - "odkcentral/**"
   # Allow manual trigger
   workflow_dispatch:
 
-env:
-  REGISTRY: ghcr.io
-  ODK_CENTRAL_VERSION: v2023.2.1
-
 jobs:
   build-and-push-images:
     runs-on: ubuntu-latest
@@ -32,7 +25,7 @@ jobs:
       - name: Log in to the Container registry
         uses: docker/login-action@v2
         with:
-          registry: ${{ env.REGISTRY }}
+          registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
@@ -41,13 +34,17 @@ jobs:
         with:
           context: odkcentral/api
           push: true
-          tags: "ghcr.io/hotosm/fmtm/odkcentral:${{ env.ODK_CENTRAL_VERSION }}"
+          tags: |
+            "ghcr.io/hotosm/fmtm/odkcentral:${{ vars.ODK_CENTRAL_VERSION }}"
+            "ghcr.io/hotosm/fmtm/odkcentral:latest"
           build-args: |
-            ODK_CENTRAL_VERSION=${{ env.ODK_CENTRAL_VERSION }}
+            ODK_CENTRAL_VERSION=${{ vars.ODK_CENTRAL_VERSION }}
 
       - name: Build and push odkcentral proxy
         uses: docker/build-push-action@v4
         with:
           context: odkcentral/proxy
           push: true
-          tags: "ghcr.io/hotosm/fmtm/odkcentral-proxy:latest"
+          tags: |
+            "ghcr.io/hotosm/fmtm/odkcentral-proxy:${{ vars.ODK_CENTRAL_VERSION }}"
+            "ghcr.io/hotosm/fmtm/odkcentral-proxy:latest"
diff --git a/.github/workflows/frontend_test.yml b/.github/workflows/frontend_test.yml
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -0,0 +1,21 @@
+name: PR
+
+on:
+  pull_request:
+    branches:
+      - main
+      - staging
+      - development
+    # Workflow is triggered only if src changes
+    paths:
+      - src/**
+  # Allow manual trigger (workflow_dispatch)
+  workflow_dispatch:
+
+jobs:
+  pytest:
+    uses: ./.github/workflows/r-pytest.yml
+    with:
+      image_tag: ci-${{ github.base_ref }}
+  frontend-tests:
+    uses: ./.github/workflows/r-frontend_tests.yml