Feature - Project Details Mapper UI

.env.example

@@ -4,7 +4,7 @@
 ODK_CENTRAL_VERSION=v2023.2.1
 ODK_CENTRAL_URL=https://central-proxy
 [email protected]
-ODK_CENTRAL_PASSWD=fmtm
+ODK_CENTRAL_PASSWD=testuserpassword
 
 ### FMTM ###
 # DEBUG=True

.github/workflows/build_and_deploy.yml

@@ -164,6 +164,6 @@ jobs:
       - name: Deploy
         run: |
           docker compose --file docker-compose.deploy.yml pull
-          docker compose --file docker-compose.deploy.yml up --detach
+          docker compose --file docker-compose.deploy.yml up --detach --remove-orphans
         env:
           DOCKER_HOST: ${{ vars.DOCKER_HOST }}

.github/workflows/r-pytest.yml

@@ -76,7 +76,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - name: Run pytest as appuser
+      - name: Run pytest
         run: |
           cd src/backend
-          gosu appuser pytest
+          pytest

INSTALL.md

@@ -194,9 +194,9 @@ Make sure to replace `<your-username>` with your GitHub username.
 
 1. You will need to [Install Docker](https://docs.docker.com/engine/install/) and ensure that it is running on your local machine.
 2. From the command line: navigate to the top level directory of the FMTM project.
-3. From the command line run: `docker compose build ui-main`
+3. From the command line run: `docker compose build ui`
    This is essential, as the development container for the frontend is different to production.
-4. Once everything is built, from the command line run: `docker compose up -d ui-main`
+4. Once everything is built, from the command line run: `docker compose up -d ui`
 
 5. If everything goes well you should now be able to **navigate to the project in your browser:** <http://127.0.0.1:8080>
 

docker-compose.deploy.yml

@@ -114,10 +114,11 @@ services:
       - .env
     networks:
       - fmtm-net
+    entrypoint: ["/migrate-entrypoint.sh"]
     command: ["alembic", "upgrade", "head"]
-    restart: "no"
+    restart: "on-failure:3"
 
-  ui-main:
+  ui:
     image: "ghcr.io/hotosm/fmtm/frontend:${FRONTEND_MAIN_VERSION}-${GIT_BRANCH}"
     build:
       context: src/frontend
@@ -126,7 +127,7 @@ services:
         APP_VERSION: ${FRONTEND_MAIN_VERSION}
         API_URL: ${URL_SCHEME}://${API_URL}
         FRONTEND_MAIN_URL: ${URL_SCHEME}://${FRONTEND_MAIN_URL}
-    container_name: fmtm_main
+    container_name: fmtm
     depends_on:
       - api
       - traefik

docker-compose.noodk.yml

@@ -76,10 +76,11 @@ services:
       - .env
     networks:
       - fmtm-net
+    entrypoint: ["/migrate-entrypoint.sh"]
     command: ["alembic", "upgrade", "head"]
-    restart: "no"
+    restart: "on-failure:3"
 
-  ui-main:
+  ui:
     image: "ghcr.io/hotosm/fmtm/frontend:debug"
     build:
       context: src/frontend
@@ -88,7 +89,7 @@ services:
         APP_VERSION: ${FRONTEND_MAIN_VERSION}
         API_URL: ${URL_SCHEME}://${API_URL}
         FRONTEND_MAIN_URL: ${URL_SCHEME}://${FRONTEND_MAIN_URL}
-    container_name: fmtm_main
+    container_name: fmtm
     depends_on:
       - api
     volumes:

docker-compose.yml

@@ -83,19 +83,19 @@ services:
       - .env
     networks:
       - fmtm-dev
+    entrypoint: ["/migrate-entrypoint.sh"]
     command: ["alembic", "upgrade", "head"]
-    restart: "no"
+    restart: "on-failure:3"
 
-  ui-main:
+  ui:
     image: "ghcr.io/hotosm/fmtm/frontend:debug"
     build:
       context: src/frontend
       dockerfile: debug.dockerfile
       args:
-        APP_NAME: main
         API_URL: ${URL_SCHEME}://${API_URL}
         FRONTEND_MAIN_URL: ${URL_SCHEME}://${FRONTEND_MAIN_URL}
-    container_name: fmtm_main
+    container_name: fmtm
     depends_on:
       - api
     volumes:
@@ -104,6 +104,9 @@ services:
     environment:
       - API_URL=${URL_SCHEME}://${API_URL}
       - FRONTEND_MAIN_URL=${URL_SCHEME}://${FRONTEND_MAIN_URL}
+      - ODK_CENTRAL_URL=${ODK_CENTRAL_URL}
+      - ODK_CENTRAL_USER=${ODK_CENTRAL_USER}
+      - ODK_CENTRAL_PASSWD=${ODK_CENTRAL_PASSWD}
     ports:
       - "8080:8080"
     networks:

docs/dev/Backend.md

@@ -88,6 +88,24 @@ The API should now be accessible at: <http://127.0.0.1:8000/docs>
 
 To add authentication to an endpoint, import `login_required` from `auth` module, you can use it as a decorator or use fastapi `Depends(login_required)` on endpoints.
 
+### Database Migration
+
+#### Creating Migration Files
+
+- Exec into the API container: `docker compose exec api bash`.
+- Run the command to generate migrations: `alembic revision`.
+- The migration file should be generated under `src/backend/migrations/versions`.
+- Commit the file to the repo.
+
+#### Applying Migrations
+
+- Should occur automatically as part of the docker compose stack (migration service).
+- To run manually:
+
+```bash
+alembic upgrade head
+```
+
 ## Backend Debugging
 
 - The `docker-compose.yml` builds FMTM using the `debug-with-odk` target in the Dockerfile.

docs/dev/Frontend.md

@@ -12,9 +12,9 @@ For details on how to run the API first, please see: [DEV 2. Backend](https://gi
 
 1. You will need to [Install Docker](https://docs.docker.com/engine/install/) and ensure that it is running on your local machine.
 2. From the command line: navigate to the top level directory of the FMTM project.
-3. From the command line run: `docker compose build ui-main`
+3. From the command line run: `docker compose build ui`
    This is essential, as the development container for the frontend is different to production.
-4. Once everything is built, from the command line run: `docker compose up -d ui-main`
+4. Once everything is built, from the command line run: `docker compose up -d ui`
 
 5. If everything goes well you should now be able to **navigate to the project in your browser:** <http://127.0.0.1:8080>
 

src/backend/.dockerignore

@@ -5,7 +5,8 @@
 !**/*.py
 !app
 !tests
-!container-entrypoint.sh
+!app-entrypoint.sh
+!migrate-entrypoint.sh
 !pyproject.toml
 !pdm.lock
 !migrations

src/backend/Dockerfile

@@ -98,8 +98,8 @@ RUN set -ex \
         "libproj25" \
         "libgeos-c1v5" \
     && rm -rf /var/lib/apt/lists/*
-COPY container-entrypoint.sh /
-ENTRYPOINT ["/container-entrypoint.sh"]
+COPY *-entrypoint.sh /
+ENTRYPOINT ["/app-entrypoint.sh"]
 # Copy Python deps from build to runtime
 COPY --from=build \
     /root/.local \
@@ -113,7 +113,7 @@ COPY alembic.ini /opt/
 RUN useradd -r -u 1001 -m -c "hotosm account" -d /home/appuser -s /bin/false appuser \
     && mkdir -p /opt/logs /opt/tiles \
     && chown -R appuser:appuser /opt /home/appuser \
-    && chmod +x /container-entrypoint.sh
+    && chmod +x /app-entrypoint.sh /migrate-entrypoint.sh
 # Add volumes for persistence
 VOLUME /opt/logs
 VOLUME /opt/tiles

src/backend/app/central/central_crud.py

@@ -382,7 +382,32 @@ async def test_form_validity(xform_content: str, form_type: str):
             f.write(xform_content)
 
         xls2xform_convert(xlsform_path=xlsform_path, xform_path=outfile, validate=False)
-        return {"message": "Your form is valid"}
+
+        namespaces = {
+            "h": "http://www.w3.org/1999/xhtml",
+            "odk": "http://www.opendatakit.org/xforms",
+            "xforms": "http://www.w3.org/2002/xforms",
+        }
+
+        import xml.etree.ElementTree as ET
+
+        with open(outfile, "r") as xml:
+            data = xml.read()
+
+        root = ET.fromstring(data)
+        instances = root.findall(".//xforms:instance[@src]", namespaces)
+
+        geojson_list = []
+        for inst in instances:
+            try:
+                if "src" in inst.attrib:
+                    if (inst.attrib["src"].split("."))[1] == "geojson":
+                        parts = (inst.attrib["src"].split("."))[0].split("/")
+                        geojson_name = parts[-1]
+                        geojson_list.append(geojson_name)
+            except Exception:
+                continue
+        return {"required media": geojson_list, "message": "Your form is valid"}
     except Exception as e:
         return JSONResponse(
             content={"message": "Your form is invalid", "possible_reason": str(e)},

src/backend/app/db/db_models.py

@@ -67,7 +67,7 @@ class DbUser(Base):
 
     id = Column(BigInteger, primary_key=True, index=True)
     username = Column(String, unique=True)
-    role = Column(Enum(UserRole), default=UserRole.MAPPER, nullable=False)
+    role = Column(Enum(UserRole), default=UserRole.MAPPER)
 
     name = Column(String)
     city = Column(String)
@@ -77,11 +77,12 @@ class DbUser(Base):
     is_expert = Column(Boolean, default=False)
 
     mapping_level = Column(
-        Enum(MappingLevel), default=MappingLevel.BEGINNER, nullable=False
+        Enum(MappingLevel),
+        default=MappingLevel.BEGINNER,
     )
-    tasks_mapped = Column(Integer, default=0, nullable=False)
-    tasks_validated = Column(Integer, default=0, nullable=False)
-    tasks_invalidated = Column(Integer, default=0, nullable=False)
+    tasks_mapped = Column(Integer, default=0)
+    tasks_validated = Column(Integer, default=0)
+    tasks_invalidated = Column(Integer, default=0)
     projects_mapped = Column(ARRAY(Integer))
 
     # mentions_notifications = Column(Boolean, default=True, nullable=False)
@@ -396,7 +397,10 @@ class DbProject(Base):
 
     # PROJECT CREATION
     author_id = Column(
-        BigInteger, ForeignKey("users.id", name="fk_users"), nullable=False
+        BigInteger,
+        ForeignKey("users.id", name="fk_users"),
+        nullable=False,
+        server_default="20386219",
     )
     author = relationship(DbUser)
     created = Column(DateTime, default=timestamp, nullable=False)

src/backend/app/projects/project_crud.py

@@ -646,44 +646,63 @@ async def split_into_tasks(
     all_results = []
     boundary_data = []
     result = []
+
     if outline["type"] == "FeatureCollection":
+        log.debug("Project boundary GeoJSON = FeatureCollection")
         boundary_data.extend(feature["geometry"] for feature in outline["features"])
         result.extend(
-            process_polygon(db, project_id, data, no_of_buildings, has_data_extracts)
+            split_polygon_into_tasks(
+                db, project_id, data, no_of_buildings, has_data_extracts
+            )
             for data in boundary_data
         )
+
         for inner_list in result:
-            all_results.extend(iter(inner_list))
+            if inner_list:
+                all_results.extend(iter(inner_list))
 
     elif outline["type"] == "GeometryCollection":
+        log.debug("Project boundary GeoJSON = GeometryCollection")
         geometries = outline["geometries"]
         boundary_data.extend(iter(geometries))
         result.extend(
-            process_polygon(db, project_id, data, no_of_buildings, has_data_extracts)
+            split_polygon_into_tasks(
+                db, project_id, data, no_of_buildings, has_data_extracts
+            )
             for data in boundary_data
         )
         for inner_list in result:
-            all_results.extend(iter(inner_list))
+            if inner_list:
+                all_results.extend(iter(inner_list))
 
     elif outline["type"] == "Feature":
+        log.debug("Project boundary GeoJSON = Feature")
         boundary_data = outline["geometry"]
-        result = process_polygon(
+        result = split_polygon_into_tasks(
             db, project_id, boundary_data, no_of_buildings, has_data_extracts
         )
         all_results.extend(iter(result))
-    else:
+
+    elif outline["type"] == "Polygon":
+        log.debug("Project boundary GeoJSON = Polygon")
         boundary_data = outline
-        result = process_polygon(
+        result = split_polygon_into_tasks(
             db, project_id, boundary_data, no_of_buildings, has_data_extracts
         )
         all_results.extend(result)
+
+    else:
+        log.error(
+            "Project boundary not one of: Polygon, Feature, GeometryCollection,"
+            " FeatureCollection. Task splitting failed."
+        )
     return {
         "type": "FeatureCollection",
         "features": all_results,
     }
 
 
-def process_polygon(
+def split_polygon_into_tasks(
     db: Session,
     project_id: uuid.UUID,
     boundary_data: str,
@@ -727,16 +746,30 @@ def process_polygon(
         )
         result = db.execute(query)
         db.commit()
+
+    # TODO replace with fmtm_splitter algo
     with open("app/db/split_algorithm.sql", "r") as sql_file:
         query = sql_file.read()
+    log.debug(f"STARTED project {project_id} task splitting")
     result = db.execute(text(query), params={"num_buildings": no_of_buildings})
     result = result.fetchall()
     db.query(db_models.DbBuildings).delete()
     db.query(db_models.DbOsmLines).delete()
     db.query(db_models.DbProjectAOI).delete()
     db.commit()
+    log.debug(f"COMPLETE project {project_id} task splitting")
 
-    return result[0][0]["features"]
+    features = result[0][0]["features"]
+    if not features:
+        log.warning(
+            f"Project {project_id}: no tasks returned from splitting algorithm. "
+            f"Params: 'num_buildings': {no_of_buildings}"
+        )
+        return []
+
+    features = json.loads(features)
+    log.debug(f"Project {project_id} split into {len(features)} tasks")
+    return features
 
 
 # def update_project_boundary(

src/backend/migrate-entrypoint.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -eo pipefail
+
+while !</dev/tcp/${FMTM_DB_HOST:-fmtm-db}/5432;
+do
+    sleep 1;
+done;
+
+# Check all vars present
+if [ -z "${FMTM_DB_HOST}" ]; then
+    echo "Environment variable FMTM_DB_HOST is not set."
+    exit 1
+fi
+if [ -z "${FMTM_DB_USER}" ]; then
+    echo "Environment variable FMTM_DB_USER is not set."
+    exit 1
+fi
+if [ -z "${FMTM_DB_PASSWORD}" ]; then
+    echo "Environment variable FMTM_DB_PASSWORD is not set."
+    exit 1
+fi
+if [ -z "${FMTM_DB_NAME}" ]; then
+    echo "Environment variable FMTM_DB_NAME is not set."
+    exit 1
+fi
+
+echo "Crerating default svcfmtm user"
+# Create default svc user
+psql "postgresql://${FMTM_DB_USER}:${FMTM_DB_PASSWORD}@${FMTM_DB_HOST}/${FMTM_DB_NAME}" \
+    -c "INSERT INTO users (id, username, role, mapping_level, tasks_mapped, tasks_validated, tasks_invalidated) \
+        VALUES (20386219, 'svcfmtm', 'MAPPER', 'BEGINNER', 0, 0, 0) \
+        ON CONFLICT (id) DO NOTHING;"
+echo "User creation complete"
+
+exec "$@"