Merge remote-tracking branch 'upstream/development' into fix-xlsforms

.env.example

@@ -34,3 +34,6 @@ FMTM_DB_HOST=fmtm-db
 FMTM_DB_USER=fmtm
 FMTM_DB_PASSWORD=fmtm
 FMTM_DB_NAME=fmtm
+
+### Underpass (optional) ###
+# UNDERPASS_API_URL=

.github/workflows/pr.yml

@@ -0,0 +1,16 @@
+name: PR
+
+on:
+  pull_request:
+    branches:
+      - main
+      - staging
+      - development
+  # Allow manual trigger (workflow_dispatch)
+  workflow_dispatch:
+
+jobs:
+  pytest:
+    uses: ./.github/workflows/reusable/pytest.yml
+  frontend-tests:
+    uses: ./.github/workflows/reusable/frontend_tests.yml

.github/workflows/pr_pytest.yml

@@ -0,0 +1,24 @@
+name: PR
+
+on:
+  pull_request:
+    branches:
+      - main
+      - staging
+      - development
+    paths:
+      - src/backend/**
+  # Allow manual trigger (workflow_dispatch)
+  workflow_dispatch:
+
+jobs:
+  frontend-tests:
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ github.ref_name }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: PyTest
+        uses: ./.github/workflows/reusable/pytest.yml

.github/workflows/frontend_test.yml → .github/workflows/reusable/frontend_tests.yml

@@ -1,20 +1,17 @@
 name: Frontend Tests
 
 on:
-  push:
+  workflow_call:
     paths:
-      - "src/frontend/**" # Trigger the workflow only when files within srv/frontend change
-      - ".github/workflows/**" # Also trigger if github workflow changes
-  workflow_dispatch:
+      - src/frontend/**
 
 jobs:
   test:
     name: Run Frontend Tests
     runs-on: ubuntu-latest
 
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Setup Node.js
         uses: actions/setup-node@v2

.github/workflows/pytest.yml → .github/workflows/reusable/pytest.yml

@@ -1,18 +1,9 @@
 name: pytest
 
 on:
-  # Run tests on all pushed branches
-  push:
-    branches:
-      - "*"
-  # Run tests on PR, prior to merge to main & development.
-  pull_request:
-    branches:
-      - main
-      - staging
-      - development
-  # Allow manual trigger (workflow_dispatch)
-  workflow_dispatch:
+  workflow_call:
+    paths:
+      - src/backend/**
 
 permissions:
   contents: read

.github/workflows/wiki.yml

@@ -0,0 +1,46 @@
+name: Publish Docs to Wiki
+
+on:
+  push:
+    paths:
+      - docs/**
+    branches: [main]
+  # Allow manual trigger (workflow_dispatch)
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+env:
+  TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  USER_NAME: svcfmtm
+  USER_EMAIL: [email protected]
+  ORG: ${{ github.event.repository.owner.name }}
+  REPO_NAME: ${{ github.event.repository.name }}
+
+jobs:
+  publish_docs_to_wiki:
+    # rclone syncs /docs dir to fmtm.wiki repo
+    name: Publish Docs to Wiki
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Pull content from wiki
+        run: |
+          mkdir tmp_wiki
+          cd tmp_wiki
+          git init
+          git config user.name $USER_NAME
+          git config user.email $USER_EMAIL
+          git pull https://x-access-token:[email protected]/$ORG/$REPO_NAME.wiki.git
+
+      - name: Push content to wiki
+        run: |
+          apt update && apt install -y rsync
+          rsync -av --delete docs/ tmp_wiki/ --exclude .git
+          cd tmp_wiki
+          git add .
+          git commit -m "docs: automated wiki update on push"
+          git push -f --set-upstream https://x-access-token:[email protected]/$ORG/$REPO_NAME.wiki.git master

.gitignore

@@ -75,3 +75,6 @@ src/frontend/node_modules
 **/pdm.toml
 **/.pdm-python
 **/__pypackages__
+
+# mkdocs
+site

INSTALL.md

@@ -229,5 +229,3 @@ This is the easiest way to manage multiple frontends at once.
    - **Map:** <http://127.0.0.1:8081>
 
 That's it, you have successfully set up the frontend!!
-
-> NOTE: For a detailed guide, installation tips and other installation methods, check out the [docs folder](./docs)

README.md

@@ -4,15 +4,15 @@
 
 **Production Workflows**
 
-| PyTest                                                                                                                                                      | Build & Deploy                                                                                                                                                                            | Wiki Docs                                                                                                                                                             |
-| ----------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [![pytest](https://github.com/hotosm/fmtm/actions/workflows/pytest.yml/badge.svg?branch=main)](https://github.com/hotosm/fmtm/actions/workflows/pytest.yml) | [![Build and Deploy](https://github.com/hotosm/fmtm/actions/workflows/build_and_deploy.yml/badge.svg?branch=main)](https://github.com/hotosm/fmtm/actions/workflows/build_and_deploy.yml) | [![Publish Docs to Wiki](https://github.com/hotosm/fmtm/actions/workflows/wiki.yml/badge.svg?branch=main)](https://github.com/hotosm/fmtm/actions/workflows/wiki.yml) |
+| Build & Deploy                                                                                                                                                                            | Docs                                                                                                                                                          | Wiki                                                                                                                                                                  | CI Img                                                                                                                                                                        | ODK Imgs                                                                                                                                                                    |
+| ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [![Build and Deploy](https://github.com/hotosm/fmtm/actions/workflows/build_and_deploy.yml/badge.svg?branch=main)](https://github.com/hotosm/fmtm/actions/workflows/build_and_deploy.yml) | [![Publish Docs](https://github.com/hotosm/fmtm/actions/workflows/docs.yml/badge.svg?branch=main)](https://github.com/hotosm/fmtm/actions/workflows/docs.yml) | [![Publish Docs to Wiki](https://github.com/hotosm/fmtm/actions/workflows/wiki.yml/badge.svg?branch=main)](https://github.com/hotosm/fmtm/actions/workflows/wiki.yml) | [![Build CI Img](https://github.com/hotosm/fmtm/actions/workflows/ci_img_build.yml/badge.svg?branch=main)](https://github.com/hotosm/fmtm/actions/workflows/ci_img_build.yml) | [![Build ODK Images](https://github.com/hotosm/fmtm/actions/workflows/odk_image_build.yml/badge.svg)](https://github.com/hotosm/fmtm/actions/workflows/odk_image_build.yml) |
 
 **Development Workflows**
 
-| PyTest                                                                                                                                                             | Build & Deploy                                                                                                                                                                                   |
-| ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| [![pytest](https://github.com/hotosm/fmtm/actions/workflows/pytest.yml/badge.svg?branch=development)](https://github.com/hotosm/fmtm/actions/workflows/pytest.yml) | [![Build and Deploy](https://github.com/hotosm/fmtm/actions/workflows/build_and_deploy.yml/badge.svg?branch=development)](https://github.com/hotosm/fmtm/actions/workflows/build_and_deploy.yml) |
+| Build & Deploy                                                                                                                                                                                   | CI Img                                                                                                                                                            |
+| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [![Build and Deploy](https://github.com/hotosm/fmtm/actions/workflows/build_and_deploy.yml/badge.svg?branch=development)](https://github.com/hotosm/fmtm/actions/workflows/build_and_deploy.yml) | [![Build CI Img](https://github.com/hotosm/fmtm/actions/workflows/ci_img_build.yml/badge.svg)](https://github.com/hotosm/fmtm/actions/workflows/ci_img_build.yml) |
 
 # Field Mapping Tasking Manager (FMTM)
 

docker-compose.noodk.yml

@@ -25,6 +25,7 @@ volumes:
 
 networks:
   fmtm-dev:
+    name: fmtm-dev
 
 services:
   fmtm-db:

docker-compose.yml

@@ -26,6 +26,7 @@ volumes:
 
 networks:
   fmtm-dev:
+    name: fmtm-dev
 
 services:
   fmtm-db:
@@ -51,6 +52,10 @@ services:
       args:
         APP_VERSION: debug
     container_name: fmtm_api
+    # Uncomment these to debug with a terminal debugger like pdb
+    # Then `docker attach fmtm_api` to debug
+    # stdin_open: true
+    # tty: true
     volumes:
       - fmtm_logs:/opt/logs
       - fmtm_images:/opt/app/images

docs/_Sidebar.md

@@ -0,0 +1,27 @@
+# [Home](https://github.com/hotosm/fmtm/wiki)
+
+[About](https://github.com/hotosm/fmtm/wiki/About)
+
+[Code of Conduct](https://github.com/hotosm/fmtm/wiki/CODE_OF_CONDUCT)
+
+[User Manual for FMTM](https://github.com/hotosm/fmtm/wiki/User-Manual-For-Project-Managers)
+
+[Contribution](https://github.com/hotosm/fmtm/wiki/CONTRIBUTING)
+
+[FAQ](https://github.com/hotosm/fmtm/wiki/FAQ)
+
+## For Developers
+
+[1. Getting Started](https://github.com/hotosm/fmtm/wiki/dev/Setup)
+
+[2. Backend](https://github.com/hotosm/fmtm/wiki/dev/Backend)
+
+[3. Frontend](https://github.com/hotosm/fmtm/wiki/Dev/Frontend)
+
+[4. Database Tips](https://github.com/hotosm/fmtm/wiki/dev/Database-Tips)
+
+[5. Docker Tips](https://github.com/hotosm/fmtm/wiki/dev/Docker-Tips)
+
+[6. Production Deployment](https://github.com/hotosm/fmtm/wiki/dev/Production)
+
+[7. Troubleshooting](https://github.com/hotosm/fmtm/wiki/dev/Troubleshooting)

docs/css/extra.css

@@ -0,0 +1,5 @@
+:root {
+  --md-primary-fg-color: #d73f3f;
+  --md-primary-fg-color--light: #e27575;
+  --md-primary-fg-color--dark: #c22929;
+}

docs/dev/Backend.md

@@ -160,7 +160,7 @@ Creating a new release during development may not always be feasible.
 ```bash
 docker compose \
   -f docker-compose.yml \
-  -f docker-compose.josm.yml \
+  -f josm/docker-compose.yml \
   up -d
 ```
 

docs/dev/Troubleshooting.md

@@ -0,0 +1,48 @@
+# Troubleshooting 🆘
+
+## Running FMTM standalone
+
+- Although it's easiest to use Docker, sometimes it may no be feasible, or not preferred.
+- We use a tool called PDM to manage dependencies.
+- PDM can run in two modes: venv and PEP582 (`__pypackages__`).
+- Be careful when running FMTM you are not accidentally pulling in your system packages.
+
+### Tips
+
+- If a directory `__pypackages__` exists, delete it and attempt to
+  `pdm install`
+  again.
+- If the `__pypackages__` directory returns, then force using venv instead
+  `pdm config python.use_venv true`
+  and remove the directory again.
+- Troubleshoot the packages PDM sees with:
+  `pdm run pip list`
+- Check a package can be imported in the PDM-based Python environment:
+
+```bash
+pdm run python
+import fastapi
+```
+
+If you receive errors such as:
+
+```bash
+pydantic.error_wrappers.ValidationError: 3 validation errors for Settings
+OSM_URL
+  field required (type=value_error.missing)
+OSM_SCOPE
+  field required (type=value_error.missing)
+OSM_LOGIN_REDIRECT_URI
+  field required (type=value_error.missing)
+```
+
+Then you need to set the env variables on your system.
+
+If you would rather not do this,
+an alternative can be to feed them into the pdm command:
+
+```bash
+FRONTEND_MAIN_URL="" FRONTEND_MAP_URL="" \
+OSM_CLIENT_ID="" OSM_CLIENT_SECRET="" OSM_SECRET_KEY="" \
+pdm run uvicorn app.main:api --host 0.0.0.0 --port 8000
+```

docker-compose.josm.yml → josm/docker-compose.yml

@@ -24,7 +24,7 @@ services:
   josm:
     image: "ghcr.io/hotosm/fmtm/josm:latest"
     build:
-      context: josm
+      context: .
     container_name: josm
     environment:
       - DISPLAY=josm-novnc:0.0
@@ -40,7 +40,7 @@ services:
   josm-novnc:
     image: "ghcr.io/hotosm/fmtm/josm-novnc:latest"
     build:
-      context: josm/novnc
+      context: novnc
     container_name: josm_novnc
     environment:
       - DISPLAY_WIDTH=1280

mkdocs.yml

@@ -73,5 +73,6 @@ nav:
       - Production Deployment: dev/Production.md
       - Docker Tips: dev/Docker-Tips.md
       - Database Tips: dev/Database-Tips.md
+      - Troubleshooting: dev/Troubleshooting.md
   - API: swagger/index.html
   - Class Hierarchy: apidocs/html/inherits.html

src/backend/app/auth/auth_routes.py

@@ -22,6 +22,8 @@
 from fastapi import APIRouter, Depends, HTTPException, Request
 from fastapi.responses import JSONResponse
 from sqlalchemy.orm import Session
+from ..config import settings
+
 
 from ..db import database
 from ..db.db_models import DbUser
@@ -65,7 +67,9 @@ def callback(request: Request, osm_auth=Depends(init_osm_auth)):
     -------
     - access_token (string)
     """
-    access_token = osm_auth.callback(str(request.url))
+    print("Call back api requested", request.url)
+
+    access_token = osm_auth.callback(str(request.url).replace('http',settings.URL_SCHEME))
     log.debug(f"Access token returned: {access_token}")
     return JSONResponse(content={"access_token": access_token}, status_code=200)
 

src/backend/app/config.py

@@ -110,6 +110,7 @@ def assemble_db_connection(cls, v: Optional[str], info: FieldValidationInfo) ->
     OSM_SCOPE: str = "read_prefs"
     OSM_LOGIN_REDIRECT_URI: str = "http://127.0.0.1:8080/osmauth/"
 
+    UNDERPASS_API_URL: str = "https://raw-data-api0.hotosm.org/v1"
     SENTRY_DSN: Optional[str] = None
 
     model_config = SettingsConfigDict(

src/backend/app/main.py

@@ -141,9 +141,10 @@ def get_logger():
             serialize=True,
             rotation="00:00",
             retention="10 days",
-            filter=lambda record: record["extra"]["task"] == "create_project",
+            filter=lambda record: record["extra"].get("task") == "create_project",
         )
 
+
 api = get_application()
 
 

src/backend/app/projects/project_crud.py

@@ -581,7 +581,7 @@ def get_osm_extracts(boundary: str):
     else:
         query["geometry"] = json_boundary
 
-    base_url = "https://raw-data-api0.hotosm.org/v1"
+    base_url = settings.UNDERPASS_API_URL
     query_url = f"{base_url}/snapshot/"
     headers = {"accept": "application/json", "Content-Type": "application/json"}
 
@@ -1394,7 +1394,7 @@ def generate_appuser_files(
                 project_log.info(f"Extracting Data from OSM")
 
                 # OSM Extracts for whole project
-                pg = PostgresClient("https://raw-data-api0.hotosm.org/v1", "underpass")
+                pg = PostgresClient(settings.UNDERPASS_API_URL, "underpass")
                 # This file will store osm extracts
                 outfile = f"/tmp/{prefix}_{xform_title}.geojson"
 
@@ -2019,7 +2019,7 @@ async def update_project_form(
     db.commit()
 
     # OSM Extracts for whole project
-    pg = PostgresClient("https://raw-data-api0.hotosm.org/v1", "underpass")
+    pg = PostgresClient(settings.UNDERPASS_API_URL, "underpass")
     outfile = (
         f"/tmp/{project_title}_{category}.geojson"  # This file will store osm extracts
     )