build: major deveops refactor, use nginx/certbot + add install script (…

…#931)

* build: replace docker-compose yamls with new config

* build: replace odkcentral proxy with repo level proxy

* build: replace URL_SCHEME, API_URL, FRONTEND_MAIN_URL --> FMTM_DOMAIN

* docs: update refs to https://central-proxy --> https://proxy

* docs: remove Docker-Tips file

* build: rename docker compose files to match branches

* docs: replace wiki homepage with redirect to docs

* build: set VITE_API_URL automatically, unless override

* build: update lables for all dockerfiles

* build: add certbot and netcat to nginx image

* build: script to build nginx certbot images

* build: add renew-certs.sh script, certbot init template

* docs: extra info for production / vars

* build: certbot compose config, set all depends conditions

* build: compose add required: false for odk central svc

* build: add recommended letsencrypt ssl params nginx

* build: add PUSH_IMGS option to certs-init build

* build: add renew-certs-manual.sh script

* ci: update compose command for pytest

* build: add mc client to backend, migrate backup to s3

* fix: add default for UNDERPASS_API_URL in .env.example

* build: set default dockerfile user to non-system >999

* build: add db backups to production deploys

* build: use staging certbot config

* build: fix app dist dir for proxy copy

* build: fix fmtm-central container name for dev compose

* build: fix default_server for cert init nginx configs

* build: update cert init build stages for clarity

* build: don't cache local cert-init img builds

* build: revert certbot staging, run non-interactive

* build: set certbot non-interactive via -n flag

* build: fix certbot non-interactive before certonly

* build: remove dup ssl_dhparam directive for prod nginx

* build: cache 2nd cert nginx build, logs for backup entry

* build: fix default_server for port 80 minio/odk

* build: combine domains into single cert for nginx

* build: add http --> https redirects on root

* build: add healthcheck on db compose backup service

* docs: update license code for frontend + author

* build: add root dir for nginx frontend conf

* build: move ssl-dhparams to certs dir

* build: remove redundant FMTM_PORT in proxy env

* build: update certbot certs-init to single stage

* build: fix certbot development command

* build: rename build_img --> build_imgs, build all

* build: remove dist from proxy, load via compose volume

* build: allow caching for image_builds.sh

* build: fix FROM in proxy dockerfile

* build: update spa-to-http img, use entrypoint for ui-build

* build: specify certbot img command via array

* build: set certbot domains via entrypoint (vs command)

* build: move frontend volume from /app/dist to /app

* build: rename ui-build service to ui

* build: set frontend build args correctly in prod

* build: update VITE_API_URL without protocol

* build: change spa-to-http to rclone, add sync entrypoint

* build: add entrypoint to prod.dockerfile frontend

* build: use sh for container-entrypoint (no bash)

* build: update frontend prod dockerfile to use root

* build: optimise odk central build (use slim img, save 800mb)

* ci: update pytest to run proxy service instead of api

* ci: bump remote_deploy workflow --> 1.1.3 to force redeploy

* build: allow TAG_OVERRIDE for debug frontend

* build: add VOLUME to prod dockerfile /frontend

* build: add security headers to all prod nginx

* build: add central frontend to dev stack

* ci: remove wait-for-it from api, update test_backend workflows

* build: correctly unset ENTRYPOINT in api ci image

* build: default ci img command sleep infinity

* build: fix renew_certs_manual script

* ci: update pr labeller criteria

* build: add curlable bash script for easy install

* build: fix curlable bash install script

* build: always clone repo during install (frontend build)

* refactor: rename install-fmtm.sh --> install.sh

* build: all changing port during development install

* feat: major updates to install scripts, capture sigterms

* build: update install bash script

* docs: add info about using install script

* build: pass DOCKER_HOST to machinectl cmd

* build: neater install outputs, progress, default redirect

* build: replace frontend as default_server (over api)

* build: install allow priv port access, progress install bar

* build: rename certs-init image to proxy:certs-init tag

* build: check odk password length during install

* build: add install script to nginx proxy

* build: optimise install script, allow existing .env file

* build: add default for FMTM_SCRIPT_DOMAIN

* build: fix using existing .env in install script

* build: use cert init images specific to branches

* build: remove set -u from cert renew (unbound vars)

* build: rename proxy entrypoint file

* build: remove FMTM_SCRIPT_DOMAIN from dev setup

* build: fix ref to container entrypoint for proxy

* build: set default FMTM_SCRIPT_DOMAIN to ignore

* ci: add workflow to build proxy images

* build: fix cert renew with FMTM_SCRIPT_DOMAIN

* build: fix typo in nginx container entrypoint

* build: rename FMTM_PORT --> FMTM_DEV_PORT for clarity

* build: add defaults for FMTM_SCRIPT_DOMAIN (nginx fails)

* build: fix remove user from sudoers in script

* build: install script fix unary operator expected

* build: add names to compose volumes (multiple deploy)

* build: fix add fmtm_data volume to dev compose

* build: fix install script find .env for non-root usr

* build: fix script.conf nginx, get the script only

* build: add DEBIAN_FRONTEND=noninteractive to install script

* build: always revalidate cache for script proxy

* refactor: add s3 bucket name to prod install script

* ci: gh workflow schedule cron in single quotes

* build: install script replace == with =, fix remote execution

* docs: update latest docs with install / prod details

* docs: tweak install docs

* docs: update install docs

* build: replace this branch in install.sh with env var

* fix: set OSM_LOGIN_REDIRECT_URI automatically

* build: fix gen-env scripts for osm redirect uri

* docs: update install docs for .env generation

* docs: add info for restoring from db backup

.env.example

@@ -1,17 +1,15 @@
-### copy to .env and set variables
-
 ### ODK Central ###
-ODK_CENTRAL_URL=${ODK_CENTRAL_URL:-"https://central-proxy"}
-ODK_CENTRAL_USER=${ODK_CENTRAL_USER:-"dev@fmtm.hotosm.org"}
+ODK_CENTRAL_URL=${ODK_CENTRAL_URL:-"https://proxy"}
+ODK_CENTRAL_USER=${ODK_CENTRAL_USER:-"test@fmtm.dev"}
 ODK_CENTRAL_PASSWD=${ODK_CENTRAL_PASSWD:-"testuserpassword"}
 
 ### FMTM ###
 DEBUG=${DEBUG:-False}
 LOG_LEVEL=${LOG_LEVEL:-INFO}
 EXTRA_CORS_ORIGINS=${EXTRA_CORS_ORIGINS}
-URL_SCHEME=${URL_SCHEME:-http}
-API_URL=${API_URL:-"127.0.0.1:8000"}
-FRONTEND_MAIN_URL=${FRONTEND_MAIN_URL:-"127.0.0.1:8080"}
+FMTM_DOMAIN=${FMTM_DOMAIN:-"fmtm.localhost"}
+FMTM_DEV_PORT=${FMTM_DEV_PORT:-7050}
+CERT_EMAIL=${CERT_EMAIL}
 # Use API_PREFIX if running behind a proxy subpath (e.g. /api)
 API_PREFIX=${API_PREFIX:-/}
 
@@ -20,7 +18,7 @@ OSM_CLIENT_ID=${OSM_CLIENT_ID}
 OSM_CLIENT_SECRET=${OSM_CLIENT_SECRET}
 OSM_URL=${OSM_URL:-"https://www.openstreetmap.org"}
 OSM_SCOPE=${OSM_SCOPE:-"read_prefs"}
-OSM_LOGIN_REDIRECT_URI=${OSM_LOGIN_REDIRECT_URI:-"http://127.0.0.1:8080/osmauth/"}
+OSM_LOGIN_REDIRECT_URI="http${FMTM_DOMAIN:+s}://${FMTM_DOMAIN:-127.0.0.1:7051}/osmauth/"
 OSM_SECRET_KEY=${OSM_SECRET_KEY}
 
 ### S3 File Storage ###
@@ -41,4 +39,4 @@ FMTM_DB_PASSWORD=${FMTM_DB_PASSWORD:-"fmtm"}
 FMTM_DB_NAME=${FMTM_DB_NAME:-"fmtm"}
 
 ### Underpass (optional override) ###
-UNDERPASS_API_URL=${UNDERPASS_API_URL}
+UNDERPASS_API_URL=${UNDERPASS_API_URL:-"https://raw-data-api0.hotosm.org/v1"}

.github/labeler.yml

@@ -1,10 +1,12 @@
 "frontend":
-  - "src/frontend/**/*"
+  - any: ["src/frontend/**/*", "!src/frontend/public/install-fmtm.sh"]
 "backend":
   - "src/backend/**/*"
 "devops":
   - ".github/**/*"
+  - "nginx/**/*"
   - "scripts/**/*"
+  - "src/frontend/public/install-fmtm.sh"
   - "docker-*.yml"
   - "**/Dockerfile"
   - "**/*.dockerfile"

.github/workflows/build_and_deploy.yml

@@ -17,12 +17,13 @@ on:
 
 jobs:
   pytest:
-    uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.1.2
+    uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.2.1
     with:
       image_name: ghcr.io/${{ github.repository }}/backend
       build_context: src/backend
+      pre_command: docker compose up -d proxy
       compose_service: api
-      compose_command: wait-for-it fmtm-db:5432 --strict -- wait-for-it central:8383 --strict --timeout=30 -- pytest
+      compose_command: pytest
       tag_override: ci-${{ github.ref_name }}
     secrets: inherit
 
@@ -124,7 +125,7 @@ jobs:
     needs:
       - smoke-test-backend
       - smoke-test-frontend
-    uses: hotosm/gh-workflows/.github/workflows/[email protected].2
+    uses: hotosm/gh-workflows/.github/workflows/[email protected].3
     with:
       environment: ${{ github.ref_name }}
       docker_compose_file: "docker-compose.${{ github.ref_name }}.yml"

.github/workflows/build_odk_imgs.yml

@@ -22,10 +22,12 @@ jobs:
       extra_build_args: |
         ODK_CENTRAL_TAG=${{ vars.ODK_CENTRAL_TAG }}
 
-  build-proxy:
+  build-odkcentral-ui:
     uses: hotosm/gh-workflows/.github/workflows/[email protected]
     with:
-      context: odkcentral/proxy
+      context: odkcentral/ui
       image_tags: |
-        "ghcr.io/${{ github.repository }}/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}"
-        "ghcr.io/${{ github.repository }}/odkcentral-proxy:latest"
+        "ghcr.io/${{ github.repository }}/odkcentral-ui:${{ vars.ODK_CENTRAL_TAG }}"
+        "ghcr.io/${{ github.repository }}/odkcentral-ui:latest"
+      build_args: |
+        ODK_CENTRAL_TAG=${{ vars.ODK_CENTRAL_TAG }}

.github/workflows/build_proxy_imgs.yml

@@ -0,0 +1,45 @@
+name: 🔧 Build Proxy Images
+
+on:
+  # Trigger on schedule
+  schedule:
+    # Run midnight 1st and 15th of every month (must be single quote)
+    # prettier-ignore
+    - cron: '0 0 * * 1,15'
+  # Allow manual trigger
+  workflow_dispatch:
+
+jobs:
+  build-cert-init-main:
+    uses: hotosm/gh-workflows/.github/workflows/[email protected]
+    with:
+      context: nginx
+      target: certs-init-main
+      image_tags: |
+        "ghcr.io/${{ github.repository }}/proxy:certs-init-main"
+
+  build-cert-init-dev:
+    uses: hotosm/gh-workflows/.github/workflows/[email protected]
+    with:
+      context: nginx
+      target: certs-init-development
+      image_tags: |
+        "ghcr.io/${{ github.repository }}/proxy:certs-init-development"
+        "ghcr.io/${{ github.repository }}/proxy:certs-init-staging"
+
+  build-proxy-main:
+    uses: hotosm/gh-workflows/.github/workflows/[email protected]
+    with:
+      context: nginx
+      target: main
+      image_tags: |
+        "ghcr.io/${{ github.repository }}/proxy:main"
+
+  build-proxy-dev:
+    uses: hotosm/gh-workflows/.github/workflows/[email protected]
+    with:
+      context: nginx
+      target: development
+      image_tags: |
+        "ghcr.io/${{ github.repository }}/proxy:development"
+        "ghcr.io/${{ github.repository }}/proxy:staging"

.github/workflows/pr_label.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/labeler@v3
+      - uses: actions/labeler@v4
         # Uses .github/labeler.yml definitions
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/pr_test_backend.yml

@@ -14,16 +14,18 @@ on:
 
 jobs:
   pytest:
-    uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.1.2
+    uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.2.1
     with:
       image_name: ghcr.io/${{ github.repository }}/backend
       build_context: src/backend
+      pre_command: docker compose up -d proxy
       compose_service: api
-      compose_command: wait-for-it fmtm-db:5432 --strict -- wait-for-it central:8383 --strict --timeout=30 -- pytest
+      compose_command: pytest
       cache_extra_imgs: |
         "docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }}"
         "docker.io/minio/minio:${{ vars.MINIO_TAG }}"
-      # For caching odk central images, add:
+      # For caching odk central image & proxy image, add:
       # "ghcr.io/${{ github.repository }}/odkcentral:${{ vars.ODK_CENTRAL_TAG }}"
-      # "ghcr.io/${{ github.repository }}/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}"
+      # "ghcr.io/${{ github.repository }}/proxy:${{ github.head_ref }}"
+      # ${{ github.head_ref }} --> target branch in PR
     secrets: inherit

.github/workflows/wiki.yml

@@ -12,4 +12,4 @@ jobs:
   publish-docs-to-wiki:
     uses: hotosm/gh-workflows/.github/workflows/[email protected]
     with:
-      homepage_path: "index.md"
+      homepage_path: "wiki_redirect.md"

.gitignore

@@ -57,7 +57,7 @@ db.sqlite3
 .DS_Store
 
 # ignore settings
-.env
+**/*.env
 
 # ignore python environments
 venv
@@ -81,3 +81,6 @@ temp_webmaps/Naivasha
 
 # mkdocs
 site
+
+# Bash install script
+envsubst