diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 2bdfb79533..00465a56e6 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -13,73 +13,88 @@ on: # Allow manual trigger workflow_dispatch: -env: - REGISTRY: ghcr.io - GIT_BRANCH: ${{ github.ref_name }} - jobs: - build-and-push-images: + pytest: + uses: ./.github/workflows/r-pytest.yml + with: + image_tag: ci-${{ github.ref_name }} + + frontend-main-tests: + uses: ./.github/workflows/r-frontend_tests.yml + + extract-versions: + needs: + - pytest + - frontend-main-tests + uses: ./.github/workflows/r-extract_versions.yml + + backend-build: + uses: ./.github/workflows/r-build_backend.yml + needs: extract-versions + with: + api_version: ${{ needs.extract-versions.outputs.api_version }} + build_target: prod + image_tags: | + "ghcr.io/hotosm/fmtm/backend:${{ needs.extract-versions.outputs.api_version }}-${{ github.ref_name }}" + "ghcr.io/hotosm/fmtm/backend:latest" + + frontend-main-build: + uses: ./.github/workflows/r-build_frontend.yml + needs: extract-versions + with: + environment: ${{ github.ref_name }} + name: main + app_version: ${{ needs.extract-versions.outputs.frontend_main_version }} + build_target: prod + image_tags: | + "ghcr.io/hotosm/fmtm/frontend/main:${{ needs.extract-versions.outputs.frontend_main_version }}-${{ github.ref_name }}" + "ghcr.io/hotosm/fmtm/frontend/main:latest" + + frontend-map-build: + uses: ./.github/workflows/r-build_frontend.yml + needs: extract-versions + with: + environment: ${{ github.ref_name }} + name: fmtm_openlayer_map + app_version: ${{ needs.extract-versions.outputs.frontend_map_version }} + build_target: prod + image_tags: | + "ghcr.io/hotosm/fmtm/frontend/map:${{ needs.extract-versions.outputs.frontend_map_version }}-${{ github.ref_name }}" + "ghcr.io/hotosm/fmtm/frontend/map:latest" + + smoke-test-backend: runs-on: ubuntu-latest + needs: + - extract-versions + - backend-build environment: name: ${{ github.ref_name }} - permissions: - contents: read - packages: write steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Log in to the Container registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Persist env vars - run: echo "${{ secrets.DOTENV }}" >> $GITHUB_ENV - - - name: Extract api version - id: extract_api_version + - name: Environment to .env + env: + API_VERSION: ${{ needs.extract-versions.outputs.api_version }} + FRONTEND_MAIN_VERSION: ${{ needs.extract-versions.outputs.frontend_main_version }} + FRONTEND_MAP_VERSION: ${{ needs.extract-versions.outputs.frontend_map_version }} run: | - cd src/backend - echo "API_VERSION=$(python -c 'from app.__version__ import __version__; print(__version__)')" >> $GITHUB_ENV + echo "${{ secrets.DOTENV }}" > .env + echo "API_VERSION=${API_VERSION}" >> .env + echo "FRONTEND_MAIN_VERSION=${FRONTEND_MAIN_VERSION}" >> .env + echo "FRONTEND_MAP_VERSION=${FRONTEND_MAP_VERSION}" >> .env - - name: Extract frontend versions - id: extract_frontend_versions + - name: Backend smoke test run: | - cd src/frontend - echo "FRONTEND_MAIN_VERSION=$(jq -r '.version' main/package.json)" >> $GITHUB_ENV - - - name: Build and push backend - uses: docker/build-push-action@v4 - with: - context: src/backend - target: prod - push: true - tags: | - "ghcr.io/hotosm/fmtm/backend:${{ env.API_VERSION }}-${{ github.ref_name }}" - "ghcr.io/hotosm/fmtm/backend:latest" - build-args: | - APP_VERSION=${{ env.API_VERSION }} - - - name: Build and push frontend main - uses: docker/build-push-action@v4 - with: - context: src/frontend - file: src/frontend/prod.dockerfile - push: true - tags: "ghcr.io/hotosm/fmtm/frontend/main:${{ env.FRONTEND_MAIN_VERSION }}-${{ github.ref_name }}" - build-args: | - APP_NAME=main - APP_VERSION=${{ env.FRONTEND_MAIN_VERSION }} - API_URL=${{ env.URL_SCHEME }}://${{ env.API_URL }} - FRONTEND_MAIN_URL=${{ env.URL_SCHEME }}://${{ env.FRONTEND_MAIN_URL }} + docker compose --file docker-compose.deploy.yml pull api + docker compose --file docker-compose.deploy.yml up api --exit-code-from api deploy-containers: runs-on: ubuntu-latest - needs: build-and-push-images + needs: + - extract-versions + - smoke-test-backend environment: name: ${{ github.ref_name }} @@ -87,23 +102,16 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - - name: Extract api version - id: extract_api_version - run: | - cd src/backend - echo "API_VERSION=$(python -c 'from app.__version__ import __version__; print(__version__)')" >> $GITHUB_OUTPUT - - - name: Extract frontend versions - id: extract_frontend_versions - run: | - cd src/frontend - echo "FRONTEND_MAIN_VERSION=$(jq -r '.version' main/package.json)" >> $GITHUB_OUTPUT - - name: Environment to .env + env: + API_VERSION: ${{ needs.extract-versions.outputs.api_version }} + FRONTEND_MAIN_VERSION: ${{ needs.extract-versions.outputs.frontend_main_version }} + FRONTEND_MAP_VERSION: ${{ needs.extract-versions.outputs.frontend_map_version }} run: | echo "${{ secrets.DOTENV }}" > .env - echo "API_VERSION=${{ steps.extract_api_version.outputs.API_VERSION }}" >> .env - echo "FRONTEND_MAIN_VERSION=${{ steps.extract_frontend_versions.outputs.FRONTEND_MAIN_VERSION }}" >> .env + echo "API_VERSION=${API_VERSION}" >> .env + echo "FRONTEND_MAIN_VERSION=${FRONTEND_MAIN_VERSION}" >> .env + echo "FRONTEND_MAP_VERSION=${FRONTEND_MAP_VERSION}" >> .env - uses: webfactory/ssh-agent@v0.7.0 with: diff --git a/.github/workflows/build_ci_img.yml b/.github/workflows/build_ci_img.yml new file mode 100644 index 0000000000..b803856cf2 --- /dev/null +++ b/.github/workflows/build_ci_img.yml @@ -0,0 +1,29 @@ +name: Build CI Img + +on: + # Push includes PR merge + push: + branches: + - main + - staging + - development + paths: + # Workflow is triggered only if deps change + - "src/backend/pyproject.toml" + - "src/backend/Dockerfile" + # Allow manual trigger + workflow_dispatch: + +jobs: + extract-versions: + uses: ./.github/workflows/r-extract_versions.yml + + backend-ci-build: + uses: ./.github/workflows/r-build_backend.yml + needs: [extract-versions] + with: + api_version: ${{ needs.extract-versions.outputs.api_version }} + build_target: ci + image_tags: | + "ghcr.io/hotosm/fmtm/backend:${{ needs.extract-versions.outputs.api_version }}-ci-${{ github.ref_name }}" + "ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }}" diff --git a/.github/workflows/odk_image_build.yml b/.github/workflows/build_odk_imgs.yml similarity index 70% rename from .github/workflows/odk_image_build.yml rename to .github/workflows/build_odk_imgs.yml index 3d42ad3759..7f1ba7cf3e 100644 --- a/.github/workflows/odk_image_build.yml +++ b/.github/workflows/build_odk_imgs.yml @@ -4,20 +4,13 @@ on: # Push includes PR merge push: branches: - - main - - staging - development - - "*-development-*" paths: # Workflow is triggered only if odkcentral dir changes - "odkcentral/**" # Allow manual trigger workflow_dispatch: -env: - REGISTRY: ghcr.io - ODK_CENTRAL_VERSION: v2023.2.1 - jobs: build-and-push-images: runs-on: ubuntu-latest @@ -32,7 +25,7 @@ jobs: - name: Log in to the Container registry uses: docker/login-action@v2 with: - registry: ${{ env.REGISTRY }} + registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} @@ -41,13 +34,17 @@ jobs: with: context: odkcentral/api push: true - tags: "ghcr.io/hotosm/fmtm/odkcentral:${{ env.ODK_CENTRAL_VERSION }}" + tags: | + "ghcr.io/hotosm/fmtm/odkcentral:${{ vars.ODK_CENTRAL_VERSION }}" + "ghcr.io/hotosm/fmtm/odkcentral:latest" build-args: | - ODK_CENTRAL_VERSION=${{ env.ODK_CENTRAL_VERSION }} + ODK_CENTRAL_VERSION=${{ vars.ODK_CENTRAL_VERSION }} - name: Build and push odkcentral proxy uses: docker/build-push-action@v4 with: context: odkcentral/proxy push: true - tags: "ghcr.io/hotosm/fmtm/odkcentral-proxy:latest" + tags: | + "ghcr.io/hotosm/fmtm/odkcentral-proxy:${{ vars.ODK_CENTRAL_VERSION }}" + "ghcr.io/hotosm/fmtm/odkcentral-proxy:latest" diff --git a/.github/workflows/ci_img_build.yml b/.github/workflows/ci_img_build.yml deleted file mode 100644 index 17e40040e1..0000000000 --- a/.github/workflows/ci_img_build.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Build CI Img - -on: - # Push includes PR merge - push: - branches: - - main - - staging - - development - paths: - # Workflow is triggered only if deps change - - "src/backend/pyproject.toml" - - "src/backend/Dockerfile" - # Allow manual trigger - workflow_dispatch: - -env: - REGISTRY: ghcr.io - GIT_BRANCH: ${{ github.ref_name }} - -jobs: - build-and-push-images: - runs-on: ubuntu-latest - environment: - name: ${{ github.ref_name }} - permissions: - contents: read - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract api version - id: extract_api_version - run: | - cd src/backend - echo "API_VERSION=$(python -c 'from app.__version__ import __version__; print(__version__)')" >> $GITHUB_ENV - - - name: Build image - uses: docker/build-push-action@v4 - with: - context: src/backend - target: ci - push: true - tags: | - "ghcr.io/hotosm/fmtm/backend:${{ env.API_VERSION }}-ci-${{ github.ref_name }}" - "ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }}" - build-args: | - API_VERSION=${{ env.API_VERSION }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 1e3219854a..7929e7a495 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -11,6 +11,8 @@ on: jobs: pytest: - uses: ./.github/workflows/reusable/pytest.yml - frontend-tests: - uses: ./.github/workflows/reusable/frontend_tests.yml + uses: ./.github/workflows/r-pytest.yml + with: + image_tag: ci-${{ github.base_ref }} + frontend-main-tests: + uses: ./.github/workflows/r-frontend_tests.yml diff --git a/.github/workflows/pr_pytest.yml b/.github/workflows/pr_pytest.yml deleted file mode 100644 index 7f1009b55f..0000000000 --- a/.github/workflows/pr_pytest.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: PR - -on: - pull_request: - branches: - - main - - staging - - development - paths: - - src/backend/** - # Allow manual trigger (workflow_dispatch) - workflow_dispatch: - -jobs: - frontend-tests: - runs-on: ubuntu-latest - environment: - name: ${{ github.ref_name }} - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: PyTest - uses: ./.github/workflows/reusable/pytest.yml diff --git a/.github/workflows/r-build_backend.yml b/.github/workflows/r-build_backend.yml new file mode 100644 index 0000000000..2cbe22d166 --- /dev/null +++ b/.github/workflows/r-build_backend.yml @@ -0,0 +1,44 @@ +name: Build Backend Imgs + +on: + workflow_call: + paths: + - src/backend/** + inputs: + api_version: + required: true + type: string + build_target: + required: true + type: string + image_tags: + required: true + type: string + +jobs: + build-and-push-images: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push backend + uses: docker/build-push-action@v4 + with: + context: src/backend + target: ${{ inputs.build_target }} + push: true + tags: ${{ inputs.image_tags }} + build-args: | + APP_VERSION=${{ inputs.api_version }} diff --git a/.github/workflows/r-build_frontend.yml b/.github/workflows/r-build_frontend.yml new file mode 100644 index 0000000000..4b23a7824f --- /dev/null +++ b/.github/workflows/r-build_frontend.yml @@ -0,0 +1,57 @@ +name: Build Frontend Imgs + +on: + workflow_call: + paths: + - src/frontend/** + inputs: + environment: + required: true + type: string + name: + required: true + type: string + app_version: + required: true + type: string + build_target: + required: true + type: string + image_tags: + required: true + type: string + +jobs: + build-and-push-images: + runs-on: ubuntu-latest + environment: + name: ${{ inputs.environment }} + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push frontend + uses: docker/build-push-action@v4 + with: + context: src/frontend + file: src/frontend/prod.dockerfile + target: ${{ inputs.build_target }} + push: true + tags: ${{ inputs.image_tags }} + build-args: | + APP_NAME=${{ inputs.name }} + APP_VERSION=${{ inputs.app_version }} + API_URL=${{ vars.URL_SCHEME }}://${{ vars.API_URL }} + FRONTEND_MAIN_URL=${{ vars.URL_SCHEME }}://${{ vars.FRONTEND_MAIN_URL }} + FRONTEND_MAP_URL=${{ vars.URL_SCHEME }}://${{ vars.FRONTEND_MAP_URL }} diff --git a/.github/workflows/r-extract_versions.yml b/.github/workflows/r-extract_versions.yml new file mode 100644 index 0000000000..cebb301f93 --- /dev/null +++ b/.github/workflows/r-extract_versions.yml @@ -0,0 +1,42 @@ +name: Extract Versions + +on: + workflow_call: + outputs: + api_version: + description: "Backend API Version" + value: ${{ jobs.extract-versions.outputs.api_version }} + frontend_main_version: + description: "Frontend Main Version" + value: ${{ jobs.extract-versions.outputs.frontend_main_version }} + frontend_map_version: + description: "Frontend Map Version" + value: ${{ jobs.extract-versions.outputs.frontend_map_version }} + +jobs: + extract-versions: + runs-on: ubuntu-latest + outputs: + api_version: ${{ steps.extract_api_version.outputs.api_version }} + frontend_main_version: ${{ steps.extract_frontend_versions.outputs.frontend_main_version }} + frontend_map_version: ${{ steps.extract_frontend_versions.outputs.frontend_map_version }} + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Extract api version + id: extract_api_version + run: | + cd src/backend + API_VERSION=$(python -c 'from app.__version__ import __version__; print(__version__)') + echo "api_version=${API_VERSION}" >> $GITHUB_OUTPUT + + - name: Extract frontend versions + id: extract_frontend_versions + run: | + cd src/frontend + FRONTEND_MAIN_VERSION=$(jq -r '.version' main/package.json) + FRONTEND_MAP_VERSION=$(jq -r '.version' fmtm_openlayer_map/package.json) + echo "frontend_main_version=${FRONTEND_MAIN_VERSION}" >> $GITHUB_OUTPUT + echo "frontend_map_version=${FRONTEND_MAP_VERSION}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/r-frontend_tests.yml b/.github/workflows/r-frontend_tests.yml new file mode 100644 index 0000000000..f71021073d --- /dev/null +++ b/.github/workflows/r-frontend_tests.yml @@ -0,0 +1,36 @@ +name: Frontend Tests + +on: + workflow_call: + paths: + - src/frontend/** + +jobs: + test: + name: Run Frontend Tests + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Setup Node.js + uses: actions/setup-node@v3 + with: + node-version: "18" + + - name: Cache Node Modules + uses: actions/cache@v3 + with: + path: | + src/frontend/main/node_modules + src/frontend/fmtm_openlayer_map/node_modules + keys: node-modules-${{ env.cache_id }} + restore-keys: | + node-modules- + + - name: Test Frontend Main + run: | + cd src/frontend/main + npm ci --only-dev + npm run test diff --git a/.github/workflows/reusable/pytest.yml b/.github/workflows/r-pytest.yml similarity index 87% rename from .github/workflows/reusable/pytest.yml rename to .github/workflows/r-pytest.yml index ddc9276005..39af7a3f80 100644 --- a/.github/workflows/reusable/pytest.yml +++ b/.github/workflows/r-pytest.yml @@ -4,6 +4,13 @@ on: workflow_call: paths: - src/backend/** + inputs: + image_tag: + required: true + type: string + environment: + required: false + type: string permissions: contents: read @@ -12,10 +19,10 @@ jobs: test: runs-on: ubuntu-latest environment: - name: test + name: ${{ inputs.environment || test }} container: - image: ghcr.io/hotosm/fmtm/backend:ci-${{ github.base_ref || github.ref_name }} + image: ghcr.io/hotosm/fmtm/backend:${{ inputs.image_tag }} env: ODK_CENTRAL_URL: ${{ vars.ODK_CENTRAL_URL }} ODK_CENTRAL_USER: ${{ vars.ODK_CENTRAL_USER }} @@ -29,7 +36,7 @@ jobs: services: # Start backend database fmtm-db: - image: postgis/postgis:14-3.3-alpine + image: "postgis/postgis:14-3.3-alpine" env: POSTGRES_PASSWORD: fmtm POSTGRES_DB: fmtm diff --git a/.github/workflows/reusable/frontend_tests.yml b/.github/workflows/reusable/frontend_tests.yml deleted file mode 100644 index 56dd3dab30..0000000000 --- a/.github/workflows/reusable/frontend_tests.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Frontend Tests - -on: - workflow_call: - paths: - - src/frontend/** - -jobs: - test: - name: Run Frontend Tests - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Setup Node.js - uses: actions/setup-node@v2 - with: - node-version: "18" # Change this to your preferred Node.js version - - - name: Test Frontend Main - run: | - cd src/frontend/main - npm install - npm run test diff --git a/.github/workflows/tests/pr_payload.json b/.github/workflows/tests/pr_payload.json new file mode 100644 index 0000000000..9046102093 --- /dev/null +++ b/.github/workflows/tests/pr_payload.json @@ -0,0 +1,10 @@ +{ + "pull_request": { + "head": { + "ref": "feat/some-new-thing" + }, + "base": { + "ref": "development" + } + } +} diff --git a/.github/workflows/tests/push_payload.json b/.github/workflows/tests/push_payload.json new file mode 100644 index 0000000000..eb2701ab2c --- /dev/null +++ b/.github/workflows/tests/push_payload.json @@ -0,0 +1,3 @@ +{ + "base_ref ": "development" +} diff --git a/.github/workflows/tests/test_ci.sh b/.github/workflows/tests/test_ci.sh new file mode 100644 index 0000000000..e370d28815 --- /dev/null +++ b/.github/workflows/tests/test_ci.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +set -e + +######################################## +# Note: run this from the repo root. +######################################## + +# TODO read personal access token +# read -p +# GITHUB_TOKEN=input +# Feed to act using -s flag: -s GITHUB_TOKEN=input_personal_access_token + +# PR +act pull_request -W .github/workflows/pr.yml -e .github/workflows/tests/pr_payload.json + +# Build and deploy +act push -W .github/workflows/build_and_deploy.yml -e .github/workflows/tests/push_payload.json + +# CI Img Build +act push -W .github/workflows/build_ci_img.yml -e .github/workflows/tests/push_payload.json + +# ODK Img Build +act push -W .github/workflows/build_odk_imgs.yml -e .github/workflows/tests/push_payload.json + +# Docs +act push -W .github/workflows/docs.yml -e .github/workflows/tests/push_payload.json + +# Wiki +act push -W .github/workflows/wiki.yml -e .github/workflows/tests/push_payload.json diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index adc69cdc9b..c48e8d4678 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -45,7 +45,7 @@ from osm_fieldwork.make_data_extract import PostgresClient from osm_fieldwork.OdkCentral import OdkAppUser from osm_fieldwork.xlsforms import xlsforms_path -from osm_fieldwork import json2osm +from osm_fieldwork.json2osm import json2osm from shapely import wkt from shapely.geometry import MultiPolygon, Polygon, mapping, shape from sqlalchemy import and_, column, func, inspect, select, table