Merge branch 'development' of github.com:hotosm/fmtm into feat/multi-…

…feature-selection

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
   # Lint / autoformat: Python code
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: "v0.6.5"
+    rev: "v0.6.7"
     hooks:
       # Run the linter
       - id: ruff
@@ -71,7 +71,7 @@ repos:
 
   # Lint & Autoformat: SQL
   - repo: https://github.com/sqlfluff/sqlfluff
-    rev: 3.1.1
+    rev: 3.2.0
     hooks:
       - id: sqlfluff-lint
         files: ^src/backend/migrations/(?:.*/)*.*$

docs/dev/Production.md

@@ -209,3 +209,39 @@ pg_restore --verbose -U fmtm -d fmtm
 # Run the entire docker compose stack
 docker compose -f docker-compose.$GIT_BRANCH.yml up -d
 ```
+
+## Help! FMTM Prod Is Broken 😨
+
+### Debugging
+
+- Log into the production server, fmtm.hotosm.org and view the container logs:
+
+  ```bash
+  docker logs fmtm-main-api-1
+  docker logs fmtm-main-api-2
+  docker logs fmtm-main-api-3
+  docker logs fmtm-main-api-4
+  ```
+
+  > Note there are four replica containers running, and any one of them
+  > could have handled the request. You should check them all.
+
+  They often provide useful traceback information, including timestamps.
+
+- View error reports on Sentry: <https://humanitarian-openstreetmap-tea.sentry.io>
+  You can get very detailed tracebacks here, even with the SQL executed on
+  the database amongst other things.
+
+- Reproduce the error on your local machine!
+
+### Making a hotfix
+
+- Sometimes fixes just can't wait to go through the development -->
+  staging --> production cycle. We need the fix now!
+
+- In this case, a `hotfix` can be made directly to the `main` branch:
+  - Create a branch `hotfix/something-i-fixed`, add your code and test
+    it works locally.
+  - Push your branch, then create a PR against the `main` branch in Github.
+  - Merge in the PR and wait for the deployment.
+  - Later the code can be pulled back into develop / staging.

src/backend/app/auth/auth_routes.py

@@ -250,6 +250,14 @@ async def refresh_token(
     request: Request, user_data: AuthUser = Depends(login_required)
 ):
     """Uses the refresh token to generate a new access token."""
+    if settings.DEBUG:
+        return JSONResponse(
+            status_code=HTTPStatus.OK,
+            content={
+                "token": "debugtoken",
+                **user_data.model_dump(),
+            },
+        )
     try:
         refresh_token = extract_refresh_token_from_cookie(request)
         if not refresh_token:

src/backend/app/central/central_crud.py

@@ -24,7 +24,6 @@
 from typing import Optional, Union
 
 import geojson
-from defusedxml import ElementTree
 from fastapi import HTTPException
 from loguru import logger as log
 from osm_fieldwork.OdkCentral import OdkAppUser, OdkForm, OdkProject
@@ -190,16 +189,15 @@ def create_odk_xform(
     odk_id: int,
     xform_data: BytesIO,
     odk_credentials: project_schemas.ODKCentralDecrypted,
-) -> str:
+) -> None:
     """Create an XForm on a remote ODK Central server.
 
     Args:
         odk_id (str): Project ID for ODK Central.
         xform_data (BytesIO): XForm data to set.
         odk_credentials (ODKCentralDecrypted): Creds for ODK Central.
 
-    Returns:
-        form_name (str): ODK Central form name for the API.
+    Returns: None
     """
     try:
         xform = get_odk_form(odk_credentials)
@@ -209,25 +207,7 @@ def create_odk_xform(
             status_code=500, detail={"message": "Connection failed to odk central"}
         ) from e
 
-    xform_id = xform.createForm(odk_id, xform_data, publish=True)
-    if not xform_id:
-        namespaces = {
-            "h": "http://www.w3.org/1999/xhtml",
-            "odk": "http://www.opendatakit.org/xforms",
-            "xforms": "http://www.w3.org/2002/xforms",
-        }
-        # Get the form id from the XML
-        root = ElementTree.fromstring(xform_data.getvalue())
-        xml_data = root.findall(".//xforms:data[@id]", namespaces)
-        extracted_name = "Not Found"
-        for dt in xml_data:
-            extracted_name = dt.get("id")
-        msg = f"Failed to create form on ODK Central: ({extracted_name})"
-        log.error(msg)
-        raise HTTPException(
-            status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=msg
-        ) from None
-    return xform_id
+    xform.createForm(odk_id, xform_data, publish=True)
 
 
 def delete_odk_xform(
@@ -323,7 +303,10 @@ async def read_and_test_xform(input_data: BytesIO) -> None:
         BytesIO: the converted XML representation of the XForm.
     """
     try:
-        log.debug("Parsing XLSForm --> XML data")
+        log.debug(
+            f"Parsing XLSForm --> XML data: input type {type(input_data)} | "
+            f"data length {input_data.getbuffer().nbytes}"
+        )
         # NOTE pyxform.xls2xform.convert returns a ConvertResult object
         return BytesIO(xform_convert(input_data).xform.encode("utf-8"))
     except Exception as e:
@@ -338,16 +321,14 @@ async def append_fields_to_user_xlsform(
     xlsform: BytesIO,
     form_category: str = "buildings",
     additional_entities: list[str] = None,
-    task_count: int = None,
     existing_id: str = None,
-) -> BytesIO:
+) -> tuple[str, BytesIO]:
     """Helper to return the intermediate XLSForm prior to convert."""
     log.debug("Appending mandatory FMTM fields to XLSForm")
     return await append_mandatory_fields(
         xlsform,
         form_category=form_category,
         additional_entities=additional_entities,
-        task_count=task_count,
         existing_id=existing_id,
     )
 
@@ -356,15 +337,13 @@ async def validate_and_update_user_xlsform(
     xlsform: BytesIO,
     form_category: str = "buildings",
     additional_entities: list[str] = None,
-    task_count: int = None,
     existing_id: str = None,
 ) -> BytesIO:
     """Wrapper to append mandatory fields and validate user uploaded XLSForm."""
-    updated_file_bytes = await append_fields_to_user_xlsform(
+    xform_id, updated_file_bytes = await append_fields_to_user_xlsform(
         xlsform,
         form_category=form_category,
         additional_entities=additional_entities,
-        task_count=task_count,
         existing_id=existing_id,
     )
 
@@ -378,7 +357,6 @@ async def update_project_xform(
     odk_id: int,
     xlsform: BytesIO,
     category: str,
-    task_count: int,
     odk_credentials: project_schemas.ODKCentralDecrypted,
 ) -> None:
     """Update and publish the XForm for a project.
@@ -388,7 +366,6 @@ async def update_project_xform(
         odk_id (int): ODK Central form ID.
         xlsform (UploadFile): XForm data.
         category (str): Category of the XForm.
-        task_count (int): The number of tasks in a project.
         odk_credentials (project_schemas.ODKCentralDecrypted): ODK Central creds.
 
     Returns: None
@@ -899,12 +876,10 @@ async def get_appuser_token(
     xform_id: str,
     project_odk_id: int,
     odk_credentials: project_schemas.ODKCentralDecrypted,
-    db: Session,
 ):
     """Get the app user token for a specific project.
 
     Args:
-        db: The database session to use.
         odk_credentials: ODK credentials for the project.
         project_odk_id: The ODK ID of the project.
         xform_id: The ID of the XForm.

src/backend/app/central/central_routes.py

@@ -79,9 +79,8 @@ async def refresh_appuser_token(
     try:
         odk_credentials = await project_deps.get_odk_credentials(db, project_id)
         project_odk_id = project.odkid
-        db_xform = await project_deps.get_project_xform(db, project_id)
         odk_token = await central_crud.get_appuser_token(
-            db_xform.odk_form_id, project_odk_id, odk_credentials, db
+            project.odk_form_id, project_odk_id, odk_credentials, db
         )
         project.odk_token = odk_token
         db.commit()

src/backend/app/db/db_models.py

@@ -210,23 +210,6 @@ class DbXLSForm(Base):
     xls = cast(bytes, Column(LargeBinary))
 
 
-class DbXForm(Base):
-    """XForms linked per project.
-
-    TODO eventually we will support multiple forms per project.
-    TODO So the category field a stub until then.
-    TODO currently it's maintained under projects.xform_category.
-    """
-
-    __tablename__ = "xforms"
-    id = cast(int, Column(Integer, primary_key=True, autoincrement=True))
-    project_id = cast(
-        int, Column(Integer, ForeignKey("projects.id"), name="project_id", index=True)
-    )
-    odk_form_id = cast(str, Column(String))
-    category = cast(str, Column(String))
-
-
 class DbTaskHistory(Base):
     """Describes the history associated with a task."""
 
@@ -453,10 +436,8 @@ def tasks_bad(self):
 
     # XForm category specified
     xform_category = cast(str, Column(String))
-    # Linked XForms
-    forms = relationship(
-        DbXForm, backref="project_xform_link", cascade="all, delete, delete-orphan"
-    )
+    odk_form_id = cast(str, Column(String))
+    xlsform_content = cast(bytes, Column(LargeBinary))
 
     __table_args__ = (
         Index("idx_geometry", outline, postgresql_using="gist"),
@@ -486,13 +467,6 @@ def tasks_bad(self):
     odk_central_password = cast(str, Column(String))
     odk_token = cast(str, Column(String, nullable=True))
 
-    form_xls = cast(
-        bytes, Column(LargeBinary)
-    )  # XLSForm file if custom xls is uploaded
-    form_config_file = cast(
-        bytes, Column(LargeBinary)
-    )  # Yaml config file if custom xls is uploaded
-
     data_extract_type = cast(
         str, Column(String)
     )  # Type of data extract (Polygon or Centroid)
@@ -559,7 +533,10 @@ class DbSubmissionPhotos(Base):
     __tablename__ = "submission_photos"
 
     id = cast(int, Column(Integer, primary_key=True))
-    project_id = cast(int, Column(Integer))
+    project_id = cast(
+        int, Column(Integer, ForeignKey("projects.id"), name="project_id", index=True)
+    )
+    # Note this is not a DbTask, but an ODK task_id
     task_id = cast(int, Column(Integer))
     submission_id = cast(str, Column(String))
     s3_path = cast(str, Column(String))

src/backend/app/projects/project_crud.py

@@ -834,15 +834,13 @@ def flatten_dict(d, parent_key="", sep="_"):
 
 
 async def generate_odk_central_project_content(
-    project: db_models.DbProject,
+    project_odk_id: int,
+    project_odk_form_id: str,
     odk_credentials: project_schemas.ODKCentralDecrypted,
     xlsform: BytesIO,
     task_extract_dict: dict[int, geojson.FeatureCollection],
-    db: Session,
 ) -> str:
     """Populate the project in ODK Central with XForm, Appuser, Permissions."""
-    project_odk_id = project.odkid
-
     # The ODK Dataset (Entity List) must exist prior to main XLSForm
     entities_list = await central_crud.task_geojson_dict_to_entity_values(
         task_extract_dict
@@ -861,33 +859,16 @@ async def generate_odk_central_project_content(
 
     # Upload survey XForm
     log.info("Uploading survey XForm to ODK Central")
-    xform_id = central_crud.create_odk_xform(
+    central_crud.create_odk_xform(
         project_odk_id,
         xform,
         odk_credentials,
     )
 
-    sql = text(
-        """
-        INSERT INTO xforms (
-            project_id, odk_form_id, category
-        )
-        VALUES (
-            :project_id, :xform_id, :category
-        )
-        """
-    )
-    db.execute(
-        sql,
-        {
-            "project_id": project.id,
-            "xform_id": xform_id,
-            "category": project.xform_category,
-        },
-    )
-    db.commit()
     return await central_crud.get_appuser_token(
-        xform_id, project_odk_id, odk_credentials, db
+        project_odk_form_id,
+        project_odk_id,
+        odk_credentials,
     )
 
 
@@ -929,13 +910,15 @@ async def generate_project_files(
 
         # Get ODK Project ID
         project_odk_id = project.odkid
+        project_xlsform = project.xlsform_content
+        project_odk_form_id = project.odk_form_id
 
         encrypted_odk_token = await generate_odk_central_project_content(
-            project,
+            project_odk_id,
+            project_odk_form_id,
             odk_credentials,
-            BytesIO(project.form_xls),
+            BytesIO(project_xlsform),
             task_extract_dict,
-            db,
         )
         log.debug(
             f"Setting odk token for FMTM project ({project_id}) "
@@ -1488,9 +1471,8 @@ async def get_dashboard_detail(
     """Get project details for project dashboard."""
     odk_central = await project_deps.get_odk_credentials(db, project.id)
     xform = central_crud.get_odk_form(odk_central)
-    db_xform = await project_deps.get_project_xform(db, project.id)
 
-    submission_meta_data = xform.getFullDetails(project.odkid, db_xform.odk_form_id)
+    submission_meta_data = xform.getFullDetails(project.odkid, project.odk_form_id)
     project.total_submission = submission_meta_data.get("submissions", 0)
     project.last_active = submission_meta_data.get("lastSubmission")
 

src/backend/app/projects/project_deps.py

@@ -98,28 +98,3 @@ async def get_odk_credentials(db: Session, project_id: int):
         odk_central_user=user,
         odk_central_password=password,
     )
-
-
-async def get_project_xform(db, project_id):
-    """Retrieve the transformation associated with a specific project.
-
-    Args:
-        db: Database connection object.
-        project_id: The ID of the project to retrieve the transformation for.
-
-    Returns:
-        The transformation record associated with the specified project.
-
-    Raises:
-        None
-    """
-    sql = text(
-        """
-        SELECT * FROM xforms
-        WHERE project_id = :project_id;
-    """
-    )
-
-    result = db.execute(sql, {"project_id": project_id})
-    db_xform = result.first()
-    return db_xform