diff --git a/INSTALL.md b/INSTALL.md index daafad9544..6cd9bb6235 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -158,7 +158,7 @@ The corresponding docker-compose files are: main - docker-compose.main.yml staging - docker-compose.staging.yml development - docker-compose.development.yml -local test - docker-compose.yml +local test - compose.yaml ``` Set your selection to a terminal variable to make the next step easier: diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000000..2085cf21ca --- /dev/null +++ b/compose.yaml @@ -0,0 +1,330 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +name: fmtm + +volumes: + fmtm_data: + fmtm_db_data: + fmtm_logs: + fmtm_images: + fmtm_tiles: + central_db_data: + central_frontend: + +networks: + fmtm-net: + name: fmtm-${GIT_BRANCH:-local} + +services: + proxy: + image: "ghcr.io/hotosm/fmtm/proxy:debug" + build: + context: nginx + target: debug + args: + NGINX_TAG: "${NGINX_TAG:-1.27.0}" + depends_on: + api: + condition: service_started + ui: + condition: service_started + ui-mapper: + condition: service_started + central: + condition: service_started + required: false + central-ui: + condition: service_completed_successfully + required: false + s3: + condition: service_started + electric: + condition: service_started + volumes: + - central_frontend:/usr/share/nginx/html/central + ports: + - ${FMTM_DEV_PORT:-7050}:80 + networks: + - fmtm-net + restart: "unless-stopped" + + api: + image: "ghcr.io/hotosm/fmtm/backend:${TAG_OVERRIDE:-debug}" + build: + context: src/backend + target: "${TARGET_OVERRIDE:-debug}" + args: + APP_VERSION: "${TAG_OVERRIDE:-debug}" + # Uncomment these to debug with a terminal debugger like pdb + # Then `docker attach fmtm_api` to debug + # stdin_open: true + # tty: true + volumes: + - fmtm_logs:/opt/logs + - fmtm_tiles:/opt/tiles + - ./src/backend/pyproject.toml:/opt/pyproject.toml:ro + - ./src/backend/app:/opt/app + - ./src/backend/tests:/opt/tests:ro + # - ../osm-fieldwork/osm_fieldwork:/home/appuser/.local/lib/python3.12/site-packages/osm_fieldwork:ro + # - ../osm-rawdata/osm_rawdata:/home/appuser/.local/lib/python3.12/site-packages/osm_rawdata:ro + # - ../fmtm-splitter/fmtm_splitter:/home/appuser/.local/lib/python3.12/site-packages/fmtm_splitter:ro + depends_on: + fmtm-db: + condition: service_healthy + central: + condition: service_healthy + required: false + migrations: + condition: service_completed_successfully + s3: + condition: service_healthy + env_file: + - .env + ports: + - "7052-7055:8000" + # - "5678-5679:5678" # Debugger port + networks: + - fmtm-net + restart: "unless-stopped" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/__lbheartbeat__"] + start_period: 60s + interval: 10s + timeout: 5s + retries: 10 + deploy: + replicas: ${API_REPLICAS:-1} + resources: + limits: + cpus: "0.9" + memory: 1500M + reservations: + cpus: "0.1" + memory: 100M + + ui: + image: "ghcr.io/hotosm/fmtm/frontend:debug" + build: + context: src + additional_contexts: + - code=src/frontend + dockerfile: Dockerfile.ui.debug + depends_on: + api: + condition: service_started + volumes: + - ./src/frontend:/app + - /app/node_modules/ + environment: + - VITE_API_URL=${API_URL:-http://api.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050}} + ports: + - "7051:7051" + networks: + - fmtm-net + restart: "unless-stopped" + + ui-mapper: + image: "ghcr.io/hotosm/fmtm/frontend:mapper" + build: + context: src + additional_contexts: + - code=src/mapper + dockerfile: Dockerfile.ui.debug + depends_on: + api: + condition: service_started + volumes: + - ./src/mapper:/app + - /app/node_modules/ + - /app/.svelte-kit/ + # - ../ui:/app/node_modules/@hotosm/ui:ro + environment: + - VITE_API_URL=${API_URL:-http://api.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050}} + - VITE_SYNC_URL=http://sync.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050} + networks: + - fmtm-net + restart: "unless-stopped" + + central: + profiles: ["", "central"] + image: "ghcr.io/hotosm/fmtm/odkcentral:${ODK_CENTRAL_TAG:-v2024.2.1}" + build: + context: odkcentral/api + args: + ODK_CENTRAL_TAG: ${ODK_CENTRAL_TAG:-v2024.2.1} + depends_on: + central-db: + condition: service_healthy + environment: + - DOMAIN=${CENTRAL_DOMAIN_OVERRIDE:-odk.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050}} + - SSL_TYPE=upstream + - SYSADMIN_EMAIL=${ODK_CENTRAL_USER} + - SYSADMIN_PASSWD=${ODK_CENTRAL_PASSWD} + - HTTPS_PORT=${HTTPS_PORT:-443} + - DB_HOST=${CENTRAL_DB_HOST:-central-db} + - DB_USER=${CENTRAL_DB_USER:-odk} + - DB_PASSWORD=${CENTRAL_DB_PASSWORD:-odk} + - DB_NAME=${CENTRAL_DB_NAME:-odk} + - DB_SSL=${DB_SSL:-null} + - EMAIL_FROM=${ODK_CENTRAL_USER} + - EMAIL_HOST=${EMAIL_HOST:-mail} + - EMAIL_PORT=${EMAIL_PORT:-25} + - EMAIL_SECURE=${EMAIL_SECURE:-false} + - EMAIL_IGNORE_TLS=${EMAIL_IGNORE_TLS:-true} + - EMAIL_USER=${EMAIL_USER:-''} + - EMAIL_PASSWORD=${EMAIL_PASSWORD:-''} + - OIDC_ENABLED=${OIDC_ENABLED:-false} + - SENTRY_ORG_SUBDOMAIN=${SENTRY_ORG_SUBDOMAIN:-o130137} + - SENTRY_KEY=${SENTRY_KEY:-3cf75f54983e473da6bd07daddf0d2ee} + - SENTRY_PROJECT=${SENTRY_PROJECT:-1298632} + # ports: + # - "8383:8383" + networks: + - fmtm-net + restart: "unless-stopped" + + pyxform: + image: "ghcr.io/getodk/pyxform-http:v2.0.3" + # Temp workaround until multiarch support + build: + context: https://github.com/getodk/pyxform-http.git#v2.0.3 + depends_on: + central-db: + condition: service_healthy + networks: + - fmtm-net + restart: "unless-stopped" + + central-ui: + # This service simply builds the frontend to a volume + # accessible to the proxy, then shuts down + profiles: ["", "central"] + image: "ghcr.io/hotosm/fmtm/odkcentral-ui:${ODK_CENTRAL_TAG:-v2024.2.1}" + build: + context: odkcentral/ui + args: + ODK_CENTRAL_TAG: ${ODK_CENTRAL_TAG:-v2024.2.1} + volumes: + - central_frontend:/frontend + network_mode: none + restart: "on-failure:2" + + s3: + image: "docker.io/minio/minio:${MINIO_TAG:-RELEASE.2024-10-13T13-34-11Z}" + environment: + MINIO_ROOT_USER: ${S3_ACCESS_KEY:-fmtm} + MINIO_ROOT_PASSWORD: ${S3_SECRET_KEY:-somelongpassword} + MINIO_VOLUMES: "/mnt/data" + MINIO_BROWSER: ${MINIO_BROWSER:-off} + MINIO_CONSOLE_ADDRESS: ":9090" + volumes: + - fmtm_data:/mnt/data + # ports: + # - 9000:9000 + # - 9090:9090 + networks: + - fmtm-net + command: minio server + restart: "unless-stopped" + healthcheck: + test: timeout 5s bash -c ':> /dev/tcp/127.0.0.1/9000' || exit 1 + interval: 5s + retries: 3 + start_period: 5s + timeout: 5s + + fmtm-db: + image: "postgis/postgis:${POSTGIS_TAG:-14-3.5-alpine}" + # Temp workaround until https://github.com/postgis/docker-postgis/issues/216 + build: + context: https://github.com/postgis/docker-postgis.git#master:14-3.5/alpine + command: -c 'max_connections=300' -c 'wal_level=logical' + volumes: + - fmtm_db_data:/var/lib/postgresql/data/ + environment: + - POSTGRES_USER=${FMTM_DB_USER:-fmtm} + - POSTGRES_PASSWORD=${FMTM_DB_PASSWORD:-fmtm} + - POSTGRES_DB=${FMTM_DB_NAME:-fmtm} + ports: + - "5438:5432" + networks: + - fmtm-net + restart: "unless-stopped" + healthcheck: + test: pg_isready -U ${FMTM_DB_USER:-fmtm} -d ${FMTM_DB_NAME:-fmtm} + start_period: 5s + interval: 10s + timeout: 5s + retries: 3 + + electric: + image: "electricsql/electric:${ELECTRIC_TAG:-0.9.3}" + depends_on: + fmtm-db: + condition: service_healthy + migrations: + condition: service_completed_successfully + environment: + DATABASE_URL: postgresql://${FMTM_DB_USER:-fmtm}:${FMTM_DB_PASSWORD:-fmtm}@${FMTM_DB_HOST:-fmtm-db}/${FMTM_DB_NAME:-fmtm}?sslmode=disable + # OTEL_EXPORT: otlp + # OTLP_ENDPOINT: https://... + # ELECTRIC_WRITE_TO_PG_MODE: direct_writes + networks: + - fmtm-net + restart: "unless-stopped" + + central-db: + profiles: ["", "central"] + image: "postgis/postgis:${POSTGIS_TAG:-14-3.5-alpine}" + volumes: + - central_db_data:/var/lib/postgresql/data/ + environment: + - POSTGRES_USER=${CENTRAL_DB_USER:-odk} + - POSTGRES_PASSWORD=${CENTRAL_DB_PASSWORD:-odk} + - POSTGRES_DB=${CENTRAL_DB_NAME:-odk} + ports: + - "5434:5432" + networks: + - fmtm-net + restart: "unless-stopped" + healthcheck: + test: pg_isready -U ${CENTRAL_DB_USER:-odk} -d ${CENTRAL_DB_NAME:-odk} + start_period: 5s + interval: 10s + timeout: 5s + retries: 3 + + migrations: + image: "ghcr.io/hotosm/fmtm/backend:${TAG_OVERRIDE:-debug}" + depends_on: + fmtm-db: + condition: service_healthy + s3: + condition: service_healthy + env_file: + - .env + # Hardcode some vars for dev, as not necessarily present in the .env file + environment: + - S3_ENDPOINT=${S3_ENDPOINT:-"http://s3:9000"} + - S3_BACKUP_BUCKET_NAME=${S3_BACKUP_BUCKET_NAME:-"fmtm-db-backups"} + networks: + - fmtm-net + entrypoint: ["/migrate-entrypoint.sh"] + restart: "on-failure:2" + healthcheck: + test: [] # Set the health check test to an empty value to disable it diff --git a/contrib/josm/compose.yaml b/contrib/josm/compose.yaml new file mode 100644 index 0000000000..9fbe243caf --- /dev/null +++ b/contrib/josm/compose.yaml @@ -0,0 +1,52 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +networks: + x11: + +services: + josm: + image: "ghcr.io/hotosm/fmtm/josm:latest" + build: + context: contrib/josm + container_name: josm + environment: + - DISPLAY=josm-novnc:0.0 + depends_on: + - josm-novnc + networks: + - fmtm-net + - x11 + ports: + - "8111:80" + restart: "unless-stopped" + + josm-novnc: + image: "ghcr.io/hotosm/fmtm/josm-novnc:latest" + build: + context: contrib/josm/novnc + container_name: josm_novnc + environment: + - DISPLAY_WIDTH=1280 + - DISPLAY_HEIGHT=600 + - RUN_XTERM=no + - RUN_FLUXBOX=no + ports: + - "8112:8080" + networks: + - x11 + restart: "unless-stopped" diff --git a/contrib/just/start/Justfile b/contrib/just/start/Justfile index 86b2af1c0a..3091a5c120 100644 --- a/contrib/just/start/Justfile +++ b/contrib/just/start/Justfile @@ -57,8 +57,8 @@ without-central: [no-cd] josm: docker compose \ - -f docker-compose.yml \ - -f contrib/josm/docker-compose.yml \ + -f compose.yaml \ + -f contrib/josm/compose.yaml \ up -d @echo @@ -76,8 +76,8 @@ tunnel: #!/usr/bin/env sh docker compose \ - -f docker-compose.yml \ - -f contrib/tunnel/docker-compose.yml \ + -f compose.yaml \ + -f contrib/tunnel/compose.yaml \ up --wait # Workaround to until PR merged: @@ -92,8 +92,8 @@ tunnel: # Is is required to correctly download forms from Collect CENTRAL_DOMAIN_OVERRIDE="$(echo "${odk_url}" | sed 's|^https://||')" \ docker compose \ - -f docker-compose.yml \ - -f contrib/tunnel/docker-compose.yml \ + -f compose.yaml \ + -f contrib/tunnel/compose.yaml \ up -d central just --unstable start _print-tunnel-url "$odk_url" "$button_url" @@ -112,8 +112,8 @@ _get-tunnel-url service_name: #!/usr/bin/env sh service_url=$(docker compose \ - -f docker-compose.yml \ - -f contrib/tunnel/docker-compose.yml \ + -f compose.yaml \ + -f contrib/tunnel/compose.yaml \ logs {{service_name}}-tunnel | \ grep 'Your quick Tunnel' -A 1 | tail -n 1 | \ sed -n 's/.*| *\(https:\/\/[^ ]*\).*/\1/p') diff --git a/contrib/just/stop/Justfile b/contrib/just/stop/Justfile index 8237b78a8b..596c93fdf3 100644 --- a/contrib/just/stop/Justfile +++ b/contrib/just/stop/Justfile @@ -25,15 +25,15 @@ default: [no-cd] josm: docker compose \ - -f docker-compose.yml \ - -f contrib/josm/docker-compose.yml \ + -f compose.yaml \ + -f contrib/josm/compose.yaml \ down # Stop FMTM & tunnels [no-cd] tunnel: docker compose \ - -f docker-compose.yml \ - -f contrib/tunnel/docker-compose.yml \ + -f compose.yaml \ + -f contrib/tunnel/compose.yaml \ down diff --git a/contrib/just/test/Justfile b/contrib/just/test/Justfile index 6cdb0d2e29..a70d10da87 100644 --- a/contrib/just/test/Justfile +++ b/contrib/just/test/Justfile @@ -30,24 +30,24 @@ backend: [no-cd] frontend: docker compose \ - -f docker-compose.yml \ - -f contrib/playwright/docker-compose.yml \ + -f compose.yaml \ + -f contrib/playwright/compose.yaml \ run --rm --service-ports ui-test 'npm run test:e2e-report' # View Playwright tests as they happen in browser [no-cd] frontend-debug: docker compose \ - -f docker-compose.yml \ - -f contrib/playwright/docker-compose.yml \ + -f compose.yaml \ + -f contrib/playwright/compose.yaml \ run --rm ui-test 'npm run test:e2e-debug' # Create Playwright tests interactively [no-cd] frontend-interactive: docker compose \ - -f docker-compose.yml \ - -f contrib/playwright/docker-compose.yml \ + -f compose.yaml \ + -f contrib/playwright/compose.yaml \ run --rm ui-test 'npm run test:e2e-interactive' # Check coverage for backend tests diff --git a/contrib/openobserve/compose.yaml b/contrib/openobserve/compose.yaml new file mode 100644 index 0000000000..1595fc6c90 --- /dev/null +++ b/contrib/openobserve/compose.yaml @@ -0,0 +1,49 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +volumes: + fmtm_monitoring_data: + name: fmtm-monitoring-data-${GIT_BRANCH:-local} + +networks: + fmtm-net: + name: fmtm-${GIT_BRANCH:-local} + +services: + openobserve: + image: public.ecr.aws/zinclabs/openobserve:latest + environment: + ZO_ROOT_USER_EMAIL: ${OPENOBSERVE_USER} + ZO_ROOT_USER_PASSWORD: ${OPENOBSERVE_PASSWORD} + # Keep data 90 days + ZO_COMPACT_DATA_RETENTION_DAYS: ${OPENOBSERVE_RETENTION_DAYS} + # S3 config + # ZO_LOCAL_MODE_STORAGE: s3 + # ZO_S3_SERVER_URL: ${S3_xx} + # ZO_S3_REGION_NAME: us-west-1 + # ZO_S3_ACCESS_KEY: ${S3_xx} + # ZO_S3_SECRET_KEY: ${S3_xx} + # ZO_S3_BUCKET_NAME: ${S3_xx} + # ZO_S3_PROVIDER: s3 + ports: + - "5080:5080" + volumes: + - fmtm_monitoring_data:/data + networks: + - fmtm-net + restart: unless-stopped + # TODO add healthcheck diff --git a/contrib/playwright/compose.yaml b/contrib/playwright/compose.yaml new file mode 100644 index 0000000000..2670ee449f --- /dev/null +++ b/contrib/playwright/compose.yaml @@ -0,0 +1,87 @@ +# Uses hostname fmtm.dev.test for api and ui on same domain +# otherwise tests will not work using the cookie for auth + +networks: + fmtm-net: + name: fmtm-${GIT_BRANCH:-local} + +services: + api: + hostname: fmtm.dev.test + environment: + # Run via alias fmtm.dev.test + FMTM_DOMAIN: "fmtm.dev.test" + # Do not use dev port, use port 80 + FMTM_DEV_PORT: "false" + # The ci image has overrides for CMD and ENTRYPOINT, so re-apply + entrypoint: ["/app-entrypoint.sh"] + # API_REPLICAS can be increased via CI env vars + command: + [ + "uvicorn", + "app.main:api", + "--host", + "0.0.0.0", + "--port", + "8000", + "--workers", + "1", + "--log-level", + "critical", + "--no-access-log", + ] + + ui: + # This hostname is used for Playwright test internal networking + hostname: fmtm.dev.test + environment: + VITE_API_URL: "http://fmtm.dev.test:8000" + command: --port 80 + healthcheck: + test: timeout 5s bash -c ':> /dev/tcp/127.0.0.1/80' || exit 1 + interval: 5s + retries: 3 + start_period: 5s + timeout: 5s + + # ui-mapper: + # # This hostname is used for Playwright test internal networking + # hostname: fmtm.dev.test + # environment: + # VITE_API_URL: "http://fmtm.dev.test:8000" + # command: --port 80 + # healthcheck: + # test: timeout 5s bash -c ':> /dev/tcp/127.0.0.1/80' || exit 1 + # interval: 5s + # retries: 3 + # start_period: 5s + # timeout: 5s + + ui-test: + image: "mcr.microsoft.com/playwright:${PLAYWRIGHT_TAG:-v1.48.1}" + depends_on: + # Starts the proxy and all other services + proxy: + condition: service_healthy + api: + # Override: must be healthy before tests run + condition: service_healthy + ui: + # Override: must be healthy before tests run + condition: service_healthy + working_dir: /app + environment: + DISPLAY: :0 + volumes: + - ./src/frontend:/app + - /tmp/.X11-unix:/tmp/.X11-unix + entrypoint: /bin/sh -c + command: + - | + npm install --legacy-peer-deps + npm run test:e2e + ports: + - "9323:9323" + networks: + - fmtm-net + restart: "no" diff --git a/contrib/tunnel/compose.yaml b/contrib/tunnel/compose.yaml new file mode 100644 index 0000000000..4128e4a713 --- /dev/null +++ b/contrib/tunnel/compose.yaml @@ -0,0 +1,61 @@ +# Copyright (c) Humanitarian OpenStreetMap Team +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +networks: + fmtm-net: + name: fmtm-${GIT_BRANCH:-local} + +services: + central-tunnel: + image: "docker.io/cloudflare/cloudflared:2024.10.1" + depends_on: + central: + condition: service_healthy + networks: + - fmtm-net + restart: "unless-stopped" + command: tunnel --url http://central:8383 + + button: + image: "ghcr.io/hotosm/fmtm/frontend:button" + build: + context: ./src + additional_contexts: + - code=./contrib/tunnel/button + dockerfile: Dockerfile.ui.debug + volumes: + - ./contrib/tunnel/button:/app + - /app/node_modules/ + networks: + - fmtm-net + restart: "unless-stopped" + healthcheck: + test: timeout 5s bash -c ':> /dev/tcp/127.0.0.1/3001' || exit 1 + interval: 5s + retries: 3 + start_period: 5s + timeout: 5s + + button-tunnel: + image: "docker.io/cloudflare/cloudflared:2024.10.1" + depends_on: + button: + condition: service_healthy + networks: + - fmtm-net + restart: "unless-stopped" + command: tunnel --url http://button:3001 diff --git a/docs/dev/Backend.md b/docs/dev/Backend.md index 4c64121957..ff34eabeb6 100644 --- a/docs/dev/Backend.md +++ b/docs/dev/Backend.md @@ -116,13 +116,13 @@ just migrate ### Interactive Debugging -- The `docker-compose.yml` builds FMTM using the `debug` target in the Dockerfile. +- The `compose.yaml` builds FMTM using the `debug` target in the Dockerfile. - The debug image contains `debugpy` to assist debugging in the container. To use it: 1. Re-build the docker image `docker compose build api` -2. Uncomment the debug port in docker-compose.yml: +2. Uncomment the debug port in compose.yaml: ```yml services: @@ -201,7 +201,7 @@ Creating a new release during development may not always be feasible. - A development version of osm-fieldwork can be mounted into the FMTM container via bind mount. - Clone the osm-fieldwork repo to the same root directory as FMTM. -- Uncomment the line in docker-compose.yml +- Uncomment the line in compose.yaml ```yaml - ../osm-fieldwork/osm_fieldwork:/home/appuser/.local/lib/python3.12/site-packages/osm_fieldwork