Merge remote-tracking branch 'upstream/development' into fix-xlsforms

.github/workflows/docs.yml

@@ -41,7 +41,9 @@ jobs:
   build_openapi_json:
     name: Build OpenAPI
     runs-on: ubuntu-latest
-    container: ghcr.io/hotosm/fmtm/backend:ci-main
+    container:
+      image: ghcr.io/hotosm/fmtm/backend:ci-main
+      options: --user root
 
     steps:
       - name: Checkout repository
@@ -60,13 +62,15 @@ jobs:
 
       - name: Build OpenAPi JSON
         run: |
-          cd src/backend
-          python ../../scripts/gen_openapi_json.py -o ../../docs/openapi.json
+          chmod -R 777 .
+          gosu appuser python scripts/gen_openapi_json.py -o docs/openapi.json
 
   publish_docs:
     name: Publish Docs
     runs-on: ubuntu-latest
-    container: ghcr.io/hotosm/fmtm/backend:ci-main
+    container:
+      image: ghcr.io/hotosm/fmtm/backend:ci-main
+      options: --user root
     needs: [build_doxygen, build_openapi_json]
 
     steps:
@@ -84,7 +88,12 @@ jobs:
           restore-keys: |
             doc-build-
 
+      - name: Install Git
+        run: |
+          apt-get update
+          apt-get install -y git --no-install-recommends
+
       - name: Publish
         run: |
-          cd src/backend
-          mkdocs gh-deploy --config-file=../../mkdocs.yml --force
+          chmod -R 777 .
+          gosu appuser mkdocs gh-deploy --force

INSTALL.md

@@ -1,4 +1,4 @@
-> NOTE: This is an installation guide to quickly get the fmtm app up and running. For a detailed guide on how to install the fmtm app using different methods and contributing, checkout the [docs](./docs)
+> NOTE: This is an installation guide to quickly get the fmtm app up and running. For a detailed guide on how to install the fmtm app using different methods and contributing, checkout the [dev docs](https://hotosm.github.io/fmtm/dev/Setup/)
 
 # Table of Contents
 

README.md

@@ -1,4 +1,4 @@
-![](images/hot_logo.png)
+![](https://github.com/hotosm/fmtm/blob/main/images/hot_logo.png?raw=true)
 
 [![All Contributors](https://img.shields.io/github/all-contributors/hotosm/fmtm?color=ee8449&style=flat-square)](#contributors-)
 

docker-compose.yml

@@ -38,7 +38,7 @@ services:
       - POSTGRES_PASSWORD=${FMTM_DB_PASSWORD:-fmtm}
       - POSTGRES_DB=${FMTM_DB_NAME:-fmtm}
     ports:
-      - "5433:5432"
+      - "5438:5432"
     networks:
       - fmtm-dev
     restart: unless-stopped

docs/About.md

@@ -1,4 +1,4 @@
-![](images/hot_logo.png)
+![](https://github.com/hotosm/fmtm/blob/main/images/hot_logo.png?raw=true)
 
 # Field Mapping Tasking Manager (FMTM)
 

docs/dev/Backend.md

@@ -139,16 +139,7 @@ Example launch.json config for vscode:
 
 Creating a new release during development may not always be feasible.
 
-**Via Dockerfile**
-
-- The debug stages in the backend Dockerfile install the latest osm-fieldwork repo `main` branch.
-- To re-build, run: `docker compose build api --no-cache`.
-
-> Note: this is useful to debug functionality not yet released in a stable version on PyPi.
-
-**Via Bind-Mount**
-
-- Alternatively, a development version of osm-fieldwork can be mounted into the FMTM container.
+- A development version of osm-fieldwork can be mounted into the FMTM container via bind mount.
 - Clone the osm-fieldwork repo to the same root directory as FMTM.
 - Uncomment the line in docker-compose.yml
 

docs/swagger/swagger-initializer.js

@@ -3,7 +3,7 @@ window.onload = function () {
 
   // the following lines will be replaced by docker/configurator, when it runs in a docker-container
   window.ui = SwaggerUIBundle({
-    url: "https://hotosm.gitlab-pages.wsl.ch/fmtm/openapi.json",
+    url: "https://hotosm.github.io/fmtm/openapi.json",
     dom_id: "#swagger-ui",
     deepLinking: true,
     presets: [SwaggerUIBundle.presets.apis, SwaggerUIStandalonePreset],

src/backend/Dockerfile

@@ -79,6 +79,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1 \
     PYTHONFAULTHANDLER=1 \
     PATH="/home/appuser/.local/bin:$PATH" \
+    PYTHONPATH="/opt" \
     PYTHON_LIB="/home/appuser/.local/lib/python$PYTHON_IMG_TAG/site-packages" \
     SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \
     REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \

src/backend/app/central/central_crud.py

@@ -283,10 +283,11 @@ def delete_odk_xform(
     return result
 
 
-def list_odk_xforms(project_id: int, odk_central: project_schemas.ODKCentral = None):
+# def list_odk_xforms(project_id: int, odk_central: project_schemas.ODKCentral = None):
+def list_odk_xforms(project_id: int, odk_central: project_schemas.ODKCentral = None, metadata:bool = False):
     """List all XForms in an ODK Central project."""
     project = get_odk_project(odk_central)
-    xforms = project.listForms(project_id)
+    xforms = project.listForms(project_id, metadata)
     # FIXME: make sure it's a valid project id
     return xforms
 

src/backend/app/config.py

@@ -20,7 +20,7 @@
 from functools import lru_cache
 from typing import Any, Optional, Union
 
-from pydantic import AnyUrl, Extra, FieldValidationInfo, PostgresDsn, field_validator
+from pydantic import Extra, FieldValidationInfo, PostgresDsn, field_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
@@ -35,13 +35,13 @@ class Settings(BaseSettings):
     FRONTEND_MAIN_URL: Optional[str]
     FRONTEND_MAP_URL: Optional[str]
 
-    EXTRA_CORS_ORIGINS: Optional[Union[str, list[AnyUrl]]] = []
+    EXTRA_CORS_ORIGINS: Optional[Union[str, list[str]]] = []
 
     @field_validator("EXTRA_CORS_ORIGINS", mode="before")
     @classmethod
     def assemble_cors_origins(
         cls,
-        val: Union[str, list[AnyUrl]],
+        val: Union[str, list[str]],
         info: FieldValidationInfo,
     ) -> Union[list[str], str]:
         """Build and validate CORS origins list.
@@ -53,8 +53,8 @@ def assemble_cors_origins(
 
         # Build default origins from env vars
         url_scheme = info.data.get("URL_SCHEME")
-        main_url = info.data.get("URL_SCHEME")
-        map_url = info.data.get("URL_SCHEME")
+        main_url = info.data.get("FRONTEND_MAIN_URL")
+        map_url = info.data.get("FRONTEND_MAP_URL")
         if url_scheme and main_url and map_url:
             default_origins = [
                 f"{url_scheme}://{main_url}",
@@ -99,16 +99,16 @@ def assemble_db_connection(cls, v: Optional[str], info: FieldValidationInfo) ->
         # Convert Url type to string
         return str(pg_url)
 
-    ODK_CENTRAL_URL: Optional[AnyUrl]
-    ODK_CENTRAL_USER: Optional[str]
-    ODK_CENTRAL_PASSWD: Optional[str]
+    ODK_CENTRAL_URL: Optional[str] = ""
+    ODK_CENTRAL_USER: Optional[str] = ""
+    ODK_CENTRAL_PASSWD: Optional[str] = ""
 
     OSM_CLIENT_ID: str
     OSM_CLIENT_SECRET: str
     OSM_SECRET_KEY: str
-    OSM_URL: AnyUrl = "https://www.openstreetmap.org"
+    OSM_URL: str = "https://www.openstreetmap.org"
     OSM_SCOPE: str = "read_prefs"
-    OSM_LOGIN_REDIRECT_URI: AnyUrl = "http://127.0.0.1:8080/osmauth/"
+    OSM_LOGIN_REDIRECT_URI: str = "http://127.0.0.1:8080/osmauth/"
 
     SENTRY_DSN: Optional[str] = None
 

src/backend/app/projects/project_crud.py

@@ -701,15 +701,15 @@ def process_polygon(db:Session, project_id:uuid.UUID, boundary_data:str, no_of_b
         db.commit()
     else:
         # Remove the polygons outside of the project AOI using a parameterized query
-        query = f"""
+        query = text(f"""
                     DELETE FROM ways_poly
                     WHERE NOT ST_Within(ST_Centroid(ways_poly.geom), (SELECT geom FROM project_aoi WHERE project_id = '{project_id}'));
-                """
+                """)
         result = db.execute(query)
         db.commit()
     with open('app/db/split_algorithm.sql', 'r') as sql_file:
         query = sql_file.read()
-    result = db.execute(query, params={'num_buildings': no_of_buildings})
+    result = db.execute(text(query), params={'num_buildings': no_of_buildings})
     result = result.fetchall()
     db.query(db_models.DbBuildings).delete()
     db.query(db_models.DbOsmLines).delete()
@@ -1549,7 +1549,7 @@ def get_task_geometry(db: Session, project_id: int):
 async def get_project_features_geojson(db:Session, project_id:int):
 
     # Get the geojson of those features for this task.
-    query = f"""SELECT jsonb_build_object(
+    query = text(f"""SELECT jsonb_build_object(
                 'type', 'FeatureCollection',
                 'features', jsonb_agg(feature)
                 )
@@ -1563,7 +1563,7 @@ async def get_project_features_geojson(db:Session, project_id:int):
                 FROM features
                 WHERE project_id={project_id}
                 ) features;
-            """
+            """)
 
     result = db.execute(query)
     features = result.fetchone()[0]
@@ -2088,20 +2088,20 @@ async def update_project_form(
         # Get the features for this task.
         # Postgis query to filter task inside this task outline and of this project
         # Update those features and set task_id
-        query = f"""UPDATE features
+        query = text(f"""UPDATE features
                     SET task_id={task}
                     WHERE id in (
                     
                     SELECT id
                     FROM features
                     WHERE project_id={project_id} and ST_Intersects(geometry, '{task_obj.outline}'::Geometry)
 
-                    )"""
+                    )""")
 
         result = db.execute(query)
 
         # Get the geojson of those features for this task.
-        query = f"""SELECT jsonb_build_object(
+        query = text(f"""SELECT jsonb_build_object(
                     'type', 'FeatureCollection',
                     'features', jsonb_agg(feature)
                     )
@@ -2114,7 +2114,7 @@ async def update_project_form(
                     ) AS feature
                     FROM features
                     WHERE project_id={project_id} and task_id={task}
-                    ) features;"""
+                    ) features;""")
 
         result = db.execute(query)
         features = result.fetchone()[0]
@@ -2154,7 +2154,7 @@ async def update_odk_credentials(
 
 async def get_extracted_data_from_db(db: Session, project_id: int, outfile: str):
     """Get the geojson of those features for this project."""
-    query = f"""SELECT jsonb_build_object(
+    query = text(f"""SELECT jsonb_build_object(
                 'type', 'FeatureCollection',
                 'features', jsonb_agg(feature)
                 )
@@ -2167,7 +2167,7 @@ async def get_extracted_data_from_db(db: Session, project_id: int, outfile: str)
                 ) AS feature
                 FROM features
                 WHERE project_id={project_id}
-                ) features;"""
+                ) features;""")
 
     result = db.execute(query)
     features = result.fetchone()[0]
@@ -2208,7 +2208,7 @@ def get_project_tiles(
         db.commit()
 
         # Project Outline
-        query = f"""SELECT jsonb_build_object(
+        query = text(f"""SELECT jsonb_build_object(
                     'type', 'FeatureCollection',
                     'features', jsonb_agg(feature)
                     )
@@ -2220,7 +2220,7 @@ def get_project_tiles(
                     ) AS feature
                     FROM projects
                     WHERE id={project_id}
-                    ) features;"""
+                    ) features;""")
 
         result = db.execute(query)
         features = result.fetchone()[0]
@@ -2233,7 +2233,7 @@ def get_project_tiles(
             jsonfile.truncate(0)
             dump(features, jsonfile)
 
-        basemap = basemapper.BaseMapper(boundary_file, base, source)
+        basemap = basemapper.BaseMapper(boundary_file, base, source, False)
         outf = basemapper.DataFile(outfile, basemap.getFormat())
         suffix = os.path.splitext(outfile)[1]
         if suffix == ".mbtiles":

src/backend/app/projects/project_routes.py

@@ -1024,6 +1024,7 @@ async def download_task_boundary_osm(
     response = Response(content=content, media_type="application/xml")
     return response
 
+from sqlalchemy.sql import text
 
 @router.get("/centroid/")
 async def project_centroid(
@@ -1040,11 +1041,11 @@ async def project_centroid(
         List[Tuple[int, str]]: A list of tuples containing the task ID and the centroid as a string.
     """
 
-    query = f"""SELECT id, ARRAY_AGG(ARRAY[ST_X(ST_Centroid(outline)), ST_Y(ST_Centroid(outline))]) AS centroid
+    query = text(f"""SELECT id, ARRAY_AGG(ARRAY[ST_X(ST_Centroid(outline)), ST_Y(ST_Centroid(outline))]) AS centroid
             FROM projects
             WHERE {f"id={project_id}" if project_id else "1=1"}
-            GROUP BY id;"""
+            GROUP BY id;""")
 
     result = db.execute(query)
-    result = result.fetchall()
-    return result
+    result_dict_list = [{"id": row[0], "centroid": row[1]} for row in result.fetchall()]
+    return result_dict_list

src/backend/app/projects/project_schemas.py

@@ -50,33 +50,33 @@ class BETAProjectUpload(BaseModel):
     xform_title: Union[str, None]
     odk_central: ODKCentral
     hashtags: Union[List[str], None]
-    organisation_id: Optional[int]
+    organisation_id: Optional[int] = None
     # city: str
     # country: str
 
 
 class Feature(BaseModel):
     id: int
     project_id: int
-    task_id: Optional[int]
-    geometry: Optional[Feature]
+    task_id: Optional[int] = None
+    geometry: Optional[Feature] = None
 
 
 class ProjectSummary(BaseModel):
     id: int = -1
     priority: ProjectPriority = ProjectPriority.MEDIUM
     priority_str: str = priority.name
-    title: Optional[str]
-    location_str: Optional[str]
-    description: Optional[str]
-    total_tasks: Optional[int]
-    tasks_mapped: Optional[int]
-    num_contributors: Optional[int]
-    tasks_validated: Optional[int]
-    tasks_bad: Optional[int]
-    hashtags: Optional[List[str]]
-    organisation_id: Optional[int]
-    organisation_logo: Optional[str]
+    title: Optional[str] = None
+    location_str: Optional[str] = None
+    description: Optional[str] = None
+    total_tasks: Optional[int] = None
+    tasks_mapped: Optional[int] = None
+    num_contributors: Optional[int] = None
+    tasks_validated: Optional[int] = None
+    tasks_bad: Optional[int] = None
+    hashtags: Optional[List[str]] = None
+    organisation_id: Optional[int] = None
+    organisation_logo: Optional[str] = None
 
 
 class ProjectBase(BaseModel):
@@ -88,9 +88,9 @@ class ProjectBase(BaseModel):
     # location_str: str
     # outline_geojson: Optional[Feature]
     project_tasks: Optional[List[tasks_schemas.Task]]
-    xform_title: Optional[str]
-    hashtags: Optional[List[str]]
-    organisation_id: Optional[int]
+    xform_title: Optional[str] = None
+    hashtags: Optional[List[str]] = None
+    organisation_id: Optional[int] = None
 
 
 class ProjectOut(ProjectBase):

src/backend/app/submission/submission_crud.py

@@ -394,8 +394,6 @@ def extract_files(zip_file_path):
             zip_file.extractall(extract_dir)
             return [os.path.join(extract_dir, f) for f in zip_file.namelist()]
 
-    # Set up logging configuration
-    logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(threadName)s] %(message)s")
 
     with concurrent.futures.ThreadPoolExecutor() as executor:
         task_list = [x.id for x in project_tasks]

src/backend/app/tasks/tasks_crud.py

@@ -43,7 +43,7 @@
 
 
 async def get_task_count_in_project(db: Session, project_id: int):
-    query = f"""select count(*) from tasks where project_id = {project_id}"""
+    query = text(f"""select count(*) from tasks where project_id = {project_id}""")
     result = db.execute(query)
     return result.fetchone()[0]