diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d8e672d123..7e8bcf3d4a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: # Lint / autoformat: Python code - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: "v0.6.5" + rev: "v0.6.7" hooks: # Run the linter - id: ruff @@ -71,7 +71,7 @@ repos: # Lint & Autoformat: SQL - repo: https://github.com/sqlfluff/sqlfluff - rev: 3.1.1 + rev: 3.2.0 hooks: - id: sqlfluff-lint files: ^src/backend/migrations/(?:.*/)*.*$ diff --git a/docs/dev/Production.md b/docs/dev/Production.md index ea603fd723..655c09a0b6 100644 --- a/docs/dev/Production.md +++ b/docs/dev/Production.md @@ -209,3 +209,39 @@ pg_restore --verbose -U fmtm -d fmtm # Run the entire docker compose stack docker compose -f docker-compose.$GIT_BRANCH.yml up -d ``` + +## Help! FMTM Prod Is Broken 😨 + +### Debugging + +- Log into the production server, fmtm.hotosm.org and view the container logs: + + ```bash + docker logs fmtm-main-api-1 + docker logs fmtm-main-api-2 + docker logs fmtm-main-api-3 + docker logs fmtm-main-api-4 + ``` + + > Note there are four replica containers running, and any one of them + > could have handled the request. You should check them all. + + They often provide useful traceback information, including timestamps. + +- View error reports on Sentry: + You can get very detailed tracebacks here, even with the SQL executed on + the database amongst other things. + +- Reproduce the error on your local machine! + +### Making a hotfix + +- Sometimes fixes just can't wait to go through the development --> + staging --> production cycle. We need the fix now! + +- In this case, a `hotfix` can be made directly to the `main` branch: + - Create a branch `hotfix/something-i-fixed`, add your code and test + it works locally. + - Push your branch, then create a PR against the `main` branch in Github. + - Merge in the PR and wait for the deployment. + - Later the code can be pulled back into develop / staging. diff --git a/src/backend/app/auth/auth_routes.py b/src/backend/app/auth/auth_routes.py index f5ceccdc41..4fb882fcde 100644 --- a/src/backend/app/auth/auth_routes.py +++ b/src/backend/app/auth/auth_routes.py @@ -250,6 +250,14 @@ async def refresh_token( request: Request, user_data: AuthUser = Depends(login_required) ): """Uses the refresh token to generate a new access token.""" + if settings.DEBUG: + return JSONResponse( + status_code=HTTPStatus.OK, + content={ + "token": "debugtoken", + **user_data.model_dump(), + }, + ) try: refresh_token = extract_refresh_token_from_cookie(request) if not refresh_token: diff --git a/src/backend/app/central/central_crud.py b/src/backend/app/central/central_crud.py index 1428644ae8..22945aa9ed 100644 --- a/src/backend/app/central/central_crud.py +++ b/src/backend/app/central/central_crud.py @@ -24,7 +24,6 @@ from typing import Optional, Union import geojson -from defusedxml import ElementTree from fastapi import HTTPException from loguru import logger as log from osm_fieldwork.OdkCentral import OdkAppUser, OdkForm, OdkProject @@ -190,7 +189,7 @@ def create_odk_xform( odk_id: int, xform_data: BytesIO, odk_credentials: project_schemas.ODKCentralDecrypted, -) -> str: +) -> None: """Create an XForm on a remote ODK Central server. Args: @@ -198,8 +197,7 @@ def create_odk_xform( xform_data (BytesIO): XForm data to set. odk_credentials (ODKCentralDecrypted): Creds for ODK Central. - Returns: - form_name (str): ODK Central form name for the API. + Returns: None """ try: xform = get_odk_form(odk_credentials) @@ -209,25 +207,7 @@ def create_odk_xform( status_code=500, detail={"message": "Connection failed to odk central"} ) from e - xform_id = xform.createForm(odk_id, xform_data, publish=True) - if not xform_id: - namespaces = { - "h": "http://www.w3.org/1999/xhtml", - "odk": "http://www.opendatakit.org/xforms", - "xforms": "http://www.w3.org/2002/xforms", - } - # Get the form id from the XML - root = ElementTree.fromstring(xform_data.getvalue()) - xml_data = root.findall(".//xforms:data[@id]", namespaces) - extracted_name = "Not Found" - for dt in xml_data: - extracted_name = dt.get("id") - msg = f"Failed to create form on ODK Central: ({extracted_name})" - log.error(msg) - raise HTTPException( - status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=msg - ) from None - return xform_id + xform.createForm(odk_id, xform_data, publish=True) def delete_odk_xform( @@ -323,7 +303,10 @@ async def read_and_test_xform(input_data: BytesIO) -> None: BytesIO: the converted XML representation of the XForm. """ try: - log.debug("Parsing XLSForm --> XML data") + log.debug( + f"Parsing XLSForm --> XML data: input type {type(input_data)} | " + f"data length {input_data.getbuffer().nbytes}" + ) # NOTE pyxform.xls2xform.convert returns a ConvertResult object return BytesIO(xform_convert(input_data).xform.encode("utf-8")) except Exception as e: @@ -338,16 +321,14 @@ async def append_fields_to_user_xlsform( xlsform: BytesIO, form_category: str = "buildings", additional_entities: list[str] = None, - task_count: int = None, existing_id: str = None, -) -> BytesIO: +) -> tuple[str, BytesIO]: """Helper to return the intermediate XLSForm prior to convert.""" log.debug("Appending mandatory FMTM fields to XLSForm") return await append_mandatory_fields( xlsform, form_category=form_category, additional_entities=additional_entities, - task_count=task_count, existing_id=existing_id, ) @@ -356,15 +337,13 @@ async def validate_and_update_user_xlsform( xlsform: BytesIO, form_category: str = "buildings", additional_entities: list[str] = None, - task_count: int = None, existing_id: str = None, ) -> BytesIO: """Wrapper to append mandatory fields and validate user uploaded XLSForm.""" - updated_file_bytes = await append_fields_to_user_xlsform( + xform_id, updated_file_bytes = await append_fields_to_user_xlsform( xlsform, form_category=form_category, additional_entities=additional_entities, - task_count=task_count, existing_id=existing_id, ) @@ -378,7 +357,6 @@ async def update_project_xform( odk_id: int, xlsform: BytesIO, category: str, - task_count: int, odk_credentials: project_schemas.ODKCentralDecrypted, ) -> None: """Update and publish the XForm for a project. @@ -388,7 +366,6 @@ async def update_project_xform( odk_id (int): ODK Central form ID. xlsform (UploadFile): XForm data. category (str): Category of the XForm. - task_count (int): The number of tasks in a project. odk_credentials (project_schemas.ODKCentralDecrypted): ODK Central creds. Returns: None @@ -899,12 +876,10 @@ async def get_appuser_token( xform_id: str, project_odk_id: int, odk_credentials: project_schemas.ODKCentralDecrypted, - db: Session, ): """Get the app user token for a specific project. Args: - db: The database session to use. odk_credentials: ODK credentials for the project. project_odk_id: The ODK ID of the project. xform_id: The ID of the XForm. diff --git a/src/backend/app/central/central_routes.py b/src/backend/app/central/central_routes.py index 9dc14c6921..1a425bd953 100644 --- a/src/backend/app/central/central_routes.py +++ b/src/backend/app/central/central_routes.py @@ -79,9 +79,8 @@ async def refresh_appuser_token( try: odk_credentials = await project_deps.get_odk_credentials(db, project_id) project_odk_id = project.odkid - db_xform = await project_deps.get_project_xform(db, project_id) odk_token = await central_crud.get_appuser_token( - db_xform.odk_form_id, project_odk_id, odk_credentials, db + project.odk_form_id, project_odk_id, odk_credentials, db ) project.odk_token = odk_token db.commit() diff --git a/src/backend/app/db/db_models.py b/src/backend/app/db/db_models.py index ad85679943..1737b12e1e 100644 --- a/src/backend/app/db/db_models.py +++ b/src/backend/app/db/db_models.py @@ -210,23 +210,6 @@ class DbXLSForm(Base): xls = cast(bytes, Column(LargeBinary)) -class DbXForm(Base): - """XForms linked per project. - - TODO eventually we will support multiple forms per project. - TODO So the category field a stub until then. - TODO currently it's maintained under projects.xform_category. - """ - - __tablename__ = "xforms" - id = cast(int, Column(Integer, primary_key=True, autoincrement=True)) - project_id = cast( - int, Column(Integer, ForeignKey("projects.id"), name="project_id", index=True) - ) - odk_form_id = cast(str, Column(String)) - category = cast(str, Column(String)) - - class DbTaskHistory(Base): """Describes the history associated with a task.""" @@ -453,10 +436,8 @@ def tasks_bad(self): # XForm category specified xform_category = cast(str, Column(String)) - # Linked XForms - forms = relationship( - DbXForm, backref="project_xform_link", cascade="all, delete, delete-orphan" - ) + odk_form_id = cast(str, Column(String)) + xlsform_content = cast(bytes, Column(LargeBinary)) __table_args__ = ( Index("idx_geometry", outline, postgresql_using="gist"), @@ -486,13 +467,6 @@ def tasks_bad(self): odk_central_password = cast(str, Column(String)) odk_token = cast(str, Column(String, nullable=True)) - form_xls = cast( - bytes, Column(LargeBinary) - ) # XLSForm file if custom xls is uploaded - form_config_file = cast( - bytes, Column(LargeBinary) - ) # Yaml config file if custom xls is uploaded - data_extract_type = cast( str, Column(String) ) # Type of data extract (Polygon or Centroid) @@ -559,7 +533,10 @@ class DbSubmissionPhotos(Base): __tablename__ = "submission_photos" id = cast(int, Column(Integer, primary_key=True)) - project_id = cast(int, Column(Integer)) + project_id = cast( + int, Column(Integer, ForeignKey("projects.id"), name="project_id", index=True) + ) + # Note this is not a DbTask, but an ODK task_id task_id = cast(int, Column(Integer)) submission_id = cast(str, Column(String)) s3_path = cast(str, Column(String)) diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 528bce3bbd..a55c4b32e3 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -834,15 +834,13 @@ def flatten_dict(d, parent_key="", sep="_"): async def generate_odk_central_project_content( - project: db_models.DbProject, + project_odk_id: int, + project_odk_form_id: str, odk_credentials: project_schemas.ODKCentralDecrypted, xlsform: BytesIO, task_extract_dict: dict[int, geojson.FeatureCollection], - db: Session, ) -> str: """Populate the project in ODK Central with XForm, Appuser, Permissions.""" - project_odk_id = project.odkid - # The ODK Dataset (Entity List) must exist prior to main XLSForm entities_list = await central_crud.task_geojson_dict_to_entity_values( task_extract_dict @@ -861,33 +859,16 @@ async def generate_odk_central_project_content( # Upload survey XForm log.info("Uploading survey XForm to ODK Central") - xform_id = central_crud.create_odk_xform( + central_crud.create_odk_xform( project_odk_id, xform, odk_credentials, ) - sql = text( - """ - INSERT INTO xforms ( - project_id, odk_form_id, category - ) - VALUES ( - :project_id, :xform_id, :category - ) - """ - ) - db.execute( - sql, - { - "project_id": project.id, - "xform_id": xform_id, - "category": project.xform_category, - }, - ) - db.commit() return await central_crud.get_appuser_token( - xform_id, project_odk_id, odk_credentials, db + project_odk_form_id, + project_odk_id, + odk_credentials, ) @@ -929,13 +910,15 @@ async def generate_project_files( # Get ODK Project ID project_odk_id = project.odkid + project_xlsform = project.xlsform_content + project_odk_form_id = project.odk_form_id encrypted_odk_token = await generate_odk_central_project_content( - project, + project_odk_id, + project_odk_form_id, odk_credentials, - BytesIO(project.form_xls), + BytesIO(project_xlsform), task_extract_dict, - db, ) log.debug( f"Setting odk token for FMTM project ({project_id}) " @@ -1488,9 +1471,8 @@ async def get_dashboard_detail( """Get project details for project dashboard.""" odk_central = await project_deps.get_odk_credentials(db, project.id) xform = central_crud.get_odk_form(odk_central) - db_xform = await project_deps.get_project_xform(db, project.id) - submission_meta_data = xform.getFullDetails(project.odkid, db_xform.odk_form_id) + submission_meta_data = xform.getFullDetails(project.odkid, project.odk_form_id) project.total_submission = submission_meta_data.get("submissions", 0) project.last_active = submission_meta_data.get("lastSubmission") diff --git a/src/backend/app/projects/project_deps.py b/src/backend/app/projects/project_deps.py index 8b62d0cb19..93cb09b7a4 100644 --- a/src/backend/app/projects/project_deps.py +++ b/src/backend/app/projects/project_deps.py @@ -98,28 +98,3 @@ async def get_odk_credentials(db: Session, project_id: int): odk_central_user=user, odk_central_password=password, ) - - -async def get_project_xform(db, project_id): - """Retrieve the transformation associated with a specific project. - - Args: - db: Database connection object. - project_id: The ID of the project to retrieve the transformation for. - - Returns: - The transformation record associated with the specified project. - - Raises: - None - """ - sql = text( - """ - SELECT * FROM xforms - WHERE project_id = :project_id; - """ - ) - - result = db.execute(sql, {"project_id": project_id}) - db_xform = result.first() - return db_xform diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index 81fbb66bc5..1618119393 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -666,26 +666,31 @@ async def validate_form( ): """Basic validity check for uploaded XLSForm. - Does not append all addition values to make this a valid FMTM form for mapping. + Parses the form using ODK pyxform to check that it is valid. + + If the `debug` param is used, the form is returned for inspection. + NOTE that this debug form has additional fields appended and should + not be used for FMTM project creation. """ if debug: - updated_form = await central_crud.append_fields_to_user_xlsform( + xform_id, updated_form = await central_crud.append_fields_to_user_xlsform( xlsform, - task_count=1, # NOTE this must be included to append task_filter choices ) return StreamingResponse( updated_form, media_type=( "application/vnd.openxmlformats-" "officedocument.spreadsheetml.sheet" ), - headers={"Content-Disposition": "attachment; filename=updated_form.xlsx"}, + headers={"Content-Disposition": f"attachment; filename={xform_id}.xlsx"}, ) else: await central_crud.validate_and_update_user_xlsform( xlsform, - task_count=1, # NOTE this must be included to append task_filter choices ) - return Response(status_code=HTTPStatus.OK) + return JSONResponse( + status_code=HTTPStatus.OK, + content={"message": "Your form is valid"}, + ) @router.post("/{project_id}/generate-project-data") @@ -728,7 +733,6 @@ async def generate_files( project = project_user_dict.get("project") project_id = project.id form_category = project.xform_category - task_count = len(project.tasks) log.debug(f"Generating additional files for project: {project.id}") @@ -739,7 +743,6 @@ async def generate_files( await central_crud.validate_and_update_user_xlsform( xlsform=xlsform_upload, form_category=form_category, - task_count=task_count, additional_entities=additional_entities, ) xlsform = xlsform_upload @@ -752,14 +755,20 @@ async def generate_files( with open(xlsform_path, "rb") as f: xlsform = BytesIO(f.read()) - project_xlsform = await central_crud.append_fields_to_user_xlsform( + xform_id, project_xlsform = await central_crud.append_fields_to_user_xlsform( xlsform=xlsform, form_category=form_category, - task_count=task_count, additional_entities=additional_entities, ) # Write XLS form content to db - project.form_xls = project_xlsform.getvalue() + xlsform_bytes = project_xlsform.getvalue() + if not xlsform_bytes: + raise HTTPException( + status_code=HTTPStatus.UNPROCESSABLE_ENTITY, + detail="There was an error with the XLSForm!", + ) + project.odk_form_id = xform_id + project.xlsform_content = xlsform_bytes db.commit() # Create task in db and return uuid @@ -979,7 +988,7 @@ async def download_form( "Content-Disposition": f"attachment; filename={project.id}_xlsform.xlsx", "Content-Type": "application/media", } - return Response(content=project.form_xls, headers=headers) + return Response(content=project.xlsform_content, headers=headers) @router.post("/update-form") @@ -1019,7 +1028,7 @@ async def update_project_form( ) # Commit changes to db - project.form_xls = xlsform.getvalue() + project.xlsform_content = xlsform.getvalue() db.commit() return project diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index 670e5574fb..b8628a76c5 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -317,7 +317,7 @@ class ProjectBase(BaseModel): """Base project model.""" outline: Any = Field(exclude=True) - forms: Any = Field(exclude=True) + odk_form_id: Optional[str] = Field(exclude=True) id: int odkid: int @@ -360,10 +360,13 @@ def organisation_logo(self) -> Optional[str]: @computed_field @property def xform_id(self) -> Optional[str]: - """Compute the XForm ID from the linked DbXForm.""" - if not self.forms: + """Generate from odk_form_id. + + TODO this could be refactored out in future. + """ + if not self.odk_form_id: return None - return self.forms[0].odk_form_id + return self.odk_form_id class ProjectWithTasks(ProjectBase): diff --git a/src/backend/app/submissions/submission_crud.py b/src/backend/app/submissions/submission_crud.py index 1f681ab7d3..7d59807545 100644 --- a/src/backend/app/submissions/submission_crud.py +++ b/src/backend/app/submissions/submission_crud.py @@ -38,7 +38,6 @@ from app.central.central_crud import ( get_odk_form, - get_odk_project, list_odk_xforms, ) from app.config import settings @@ -46,7 +45,6 @@ from app.models.enums import HTTPStatus from app.projects import project_crud, project_deps from app.s3 import add_obj_to_bucket, get_obj_from_bucket -from app.tasks import tasks_crud # async def convert_json_to_osm(file_path): # """Wrapper for osm-fieldwork json2osm.""" @@ -144,8 +142,7 @@ async def gather_all_submission_csvs(db: Session, project: db_models.DbProject): odk_credentials = await project_deps.get_odk_credentials(db, project.id) xform = get_odk_form(odk_credentials) - db_xform = await project_deps.get_project_xform(db, project.id) - file = xform.getSubmissionMedia(odkid, db_xform.odk_form_id) + file = xform.getSubmissionMedia(odkid, project.odk_form_id) return file.content @@ -290,28 +287,6 @@ def update_submission_in_s3( update_bg_task_sync(db, background_task_id, 2, str(e)) # 2 is FAILED -def get_all_submissions_json(db: Session, project_id): - """Get all submissions for a project in JSON format.""" - get_project_sync = async_to_sync(project_crud.get_project) - project_info = get_project_sync(db, project_id) - - # ODK Credentials - odk_sync = async_to_sync(project_deps.get_odk_credentials) - odk_credentials = odk_sync(db, project_id) - project = get_odk_project(odk_credentials) - - get_task_id_list_sync = async_to_sync(tasks_crud.get_task_id_list) - task_list = get_task_id_list_sync(db, project_id) - - # FIXME use db_xform - xform_list = [ - f"{project_info.project_name_prefix}_task_{task}" for task in task_list - ] - # FIXME use separate func - submissions = project.getAllSubmissions(project_info.odkid, xform_list) - return submissions - - async def download_submission_in_json(db: Session, project: db_models.DbProject): """Download submission data from ODK Central.""" project_name = project.project_name_prefix @@ -329,7 +304,10 @@ async def download_submission_in_json(db: Session, project: db_models.DbProject) async def get_submission_points(db: Session, project_id: int, task_id: Optional[int]): - """Get submission points for a project.""" + """Get submission points for a project. + + FIXME refactor to pass through project object via auth. + """ project_info = await project_crud.get_project_by_id(db, project_id) if not project_info: raise HTTPException(status_code=404, detail="Project not found") @@ -337,9 +315,8 @@ async def get_submission_points(db: Session, project_id: int, task_id: Optional[ odk_id = project_info.odkid odk_credentials = await project_deps.get_odk_credentials(db, project_id) xform = get_odk_form(odk_credentials) - db_xform = await project_deps.get_project_xform(db, project_id) - response_file = xform.getSubmissionMedia(odk_id, db_xform.odk_form_id) + response_file = xform.getSubmissionMedia(odk_id, project_info.odk_form_id) response_file_bytes = response_file.content try: @@ -382,8 +359,7 @@ async def get_submission_count_of_a_project(db: Session, project: db_models.DbPr # Get ODK Form with odk credentials from the project. xform = get_odk_form(odk_credentials) - db_xform = await project_deps.get_project_xform(db, project.id) - data = xform.listSubmissions(project.odkid, db_xform.odk_form_id, {}) + data = xform.listSubmissions(project.odkid, project.odk_form_id, {}) return len(data["value"]) @@ -466,40 +442,10 @@ async def get_submission_by_project( ValueError: If the submission file cannot be found. """ - db_xform = await project_deps.get_project_xform(db, project.id) odk_central = await project_deps.get_odk_credentials(db, project.id) xform = get_odk_form(odk_central) - return xform.listSubmissions(project.odkid, db_xform.odk_form_id, filters) - - -# FIXME this is not needed now it can be directly filtered from submission table -# async def get_submission_by_task( -# project: db_models.DbProject, -# task_id: int, -# filters: dict, -# db: Session, -# ): -# """Get submissions and count by task. - -# Args: -# project: The project instance. -# task_id: The ID of the task. -# filters: A dictionary of filters. -# db: The database session. - -# Returns: -# Tuple: A tuple containing the list of submissions and the count. -# """ -# odk_credentials = await project_deps.get_odk_credentials(db, project.id) - -# xform = get_odk_form(odk_credentials) -# db_xform = await project_deps.get_project_xform(db, project.id) -# data = xform.listSubmissions(project.odkid, db_xform.odk_form_id, filters) -# submissions = data.get("value", []) -# count = data.get("@odata.count", 0) - -# return submissions, count + return xform.listSubmissions(project.odkid, project.odk_form_id, filters) async def get_submission_detail( @@ -519,10 +465,18 @@ async def get_submission_detail( """ odk_credentials = await project_deps.get_odk_credentials(db, project.id) odk_form = get_odk_form(odk_credentials) - db_xform = await project_deps.get_project_xform(db, project.id) - submission = json.loads( - odk_form.getSubmissions(project.odkid, db_xform.odk_form_id, submission_id) + + project_submissions = odk_form.getSubmissions( + project.odkid, project.odk_form_id, submission_id ) + if not project_submissions: + log.warning("Failed to download submissions due to unknown error") + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail="Failed to download submissions", + ) + + submission = json.loads(project_submissions) return submission.get("value", [])[0] @@ -587,7 +541,6 @@ async def upload_attachment_to_s3( """ try: project = await project_deps.get_project_by_id(db, project_id) - db_xform = await project_deps.get_project_xform(db, project_id) odk_central = await project_deps.get_odk_credentials(db, project_id) xform = get_odk_form(odk_central) s3_bucket = settings.S3_BUCKET_NAME @@ -631,7 +584,7 @@ async def upload_attachment_to_s3( attachment = xform.getSubmissionPhoto( project.odkid, str(instance_id), - db_xform.odk_form_id, + project.odk_form_id, str(filename), ) if attachment: diff --git a/src/backend/app/submissions/submission_routes.py b/src/backend/app/submissions/submission_routes.py index a40d917ef1..4cb2ea5c01 100644 --- a/src/backend/app/submissions/submission_routes.py +++ b/src/backend/app/submissions/submission_routes.py @@ -329,9 +329,7 @@ async def get_submission_form_fields( project = project_user.get("project") odk_credentials = await project_deps.get_odk_credentials(db, project.id) odk_form = central_crud.get_odk_form(odk_credentials) - db_xform = await project_deps.get_project_xform(db, project.id) - - return odk_form.formFields(project.odkid, db_xform.odk_form_id) + return odk_form.formFields(project.odkid, project.odk_form_id) @router.get("/submission_table") @@ -414,20 +412,6 @@ async def submission_table( return response -@router.get("/{submission_id}") -async def submission_detail( - submission_id: str, - db: Session = Depends(database.get_db), - project_user: ProjectUserDict = Depends(mapper), -) -> dict: - """This api returns the submission detail of individual submission.""" - project = project_user.get("project") - submission_detail = await submission_crud.get_submission_detail( - submission_id, project, db - ) - return submission_detail - - @router.post("/update_review_state") async def update_review_state( instance_id: str, @@ -440,11 +424,10 @@ async def update_review_state( project = current_user.get("project") odk_creds = await project_deps.get_odk_credentials(db, project.id) odk_project = central_crud.get_odk_project(odk_creds) - db_xform = await project_deps.get_project_xform(db, project.id) response = odk_project.updateReviewState( project.odkid, - db_xform.odk_form_id, + project.odk_form_id, instance_id, {"reviewState": review_state}, ) @@ -530,6 +513,20 @@ async def conflate_geojson( ) from e +@router.get("/{submission_id}") +async def submission_detail( + submission_id: str, + db: Session = Depends(database.get_db), + project_user: ProjectUserDict = Depends(mapper), +) -> dict: + """This api returns the submission detail of individual submission.""" + project = project_user.get("project") + submission_detail = await submission_crud.get_submission_detail( + submission_id, project, db + ) + return submission_detail + + @router.get("/{submission_id}/photos") async def submission_photo( submission_id: str, diff --git a/src/backend/app/tasks/task_deps.py b/src/backend/app/tasks/task_deps.py index 4ef2d8ab35..7df1dc735c 100644 --- a/src/backend/app/tasks/task_deps.py +++ b/src/backend/app/tasks/task_deps.py @@ -18,41 +18,15 @@ """Task dependencies for use in Depends.""" -from typing import Union - from fastapi import Depends from fastapi.exceptions import HTTPException from sqlalchemy.orm import Session from app.db.database import get_db -from app.db.db_models import DbProject, DbTask +from app.db.db_models import DbTask from app.models.enums import HTTPStatus -async def get_xform_name( - project: Union[int, DbProject], - task_id: int, - db: Session = Depends(get_db), -) -> str: - """Get a project xform name.""" - if isinstance(project, int): - db_project = db.query(DbProject).filter(DbProject.id == project).first() - if not db_project: - raise HTTPException( - status_code=HTTPStatus.NOT_FOUND, - detail=f"Project with ID ({project}) does not exist", - ) - else: - db_project = project - - project_name = db_project.project_name_prefix - # TODO in the future we may possibly support multiple forms per project. - # TODO to facilitate this we need to add the _{category} suffix and track. - # TODO this in the new xforms.category field/table. - form_name = project_name - return form_name - - async def get_task_by_id( project_id: int, task_id: int, diff --git a/src/backend/migrations/007-remove-xform-table.sql b/src/backend/migrations/007-remove-xform-table.sql new file mode 100644 index 0000000000..e3b8f95c2e --- /dev/null +++ b/src/backend/migrations/007-remove-xform-table.sql @@ -0,0 +1,63 @@ +-- ## Migration to: +-- * Rename projects.form_xls --> projects.xlsform_content +-- * Remove projects.form_config_file until required +-- * Add missed foreign keys to submission_photos +-- * Remove public.xforms table, moving odk_form_id to public.projects +-- Decided to remove the XForms table as projects likely always have a +-- 1:1 relationship with xforms (spwoodcock) + +-- Start a transaction +BEGIN; + +-- Add foreign keys to submission_photos table, if they don't exist +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM information_schema.table_constraints + WHERE constraint_name = 'fk_project_id' + AND table_name = 'submission_photos' + ) THEN + ALTER TABLE ONLY public.submission_photos + ADD CONSTRAINT fk_project_id FOREIGN KEY (project_id) + REFERENCES public.projects (id); + END IF; +END $$; + +-- Update public.projects table +ALTER TABLE public.projects +-- Remove form_config_file column +DROP COLUMN IF EXISTS form_config_file, +-- Add odk_form_id if not exists +ADD COLUMN IF NOT EXISTS odk_form_id VARCHAR; +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'projects' AND column_name = 'form_xls') THEN + ALTER TABLE public.projects + RENAME COLUMN form_xls TO xlsform_content; -- Rename form_xls to xlsform_content + END IF; +END $$; + +-- Migrate odk_form_id data from xforms to projects +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'xforms' + AND column_name = 'odk_form_id' + ) THEN + -- Perform data migration if odk_form_id exists in xforms + UPDATE public.projects p + SET odk_form_id = x.odk_form_id + FROM public.xforms x + WHERE x.project_id = p.id + AND p.odk_form_id IS NULL; -- Avoid overwriting existing values + END IF; +END $$; + +-- Drop public.xforms table if it exists +DROP TABLE IF EXISTS public.xforms; +DROP SEQUENCE IF EXISTS public.xforms_id_seq; + +-- Commit the transaction +COMMIT; diff --git a/src/backend/migrations/005-rename-public-beta.sql b/src/backend/migrations/008-rename-public-beta.sql similarity index 100% rename from src/backend/migrations/005-rename-public-beta.sql rename to src/backend/migrations/008-rename-public-beta.sql diff --git a/src/backend/migrations/archived/2024-02-24/003-project-roles.sql b/src/backend/migrations/archived/2024-02-24/003-project-roles.sql index 75313f553b..213bb07128 100644 --- a/src/backend/migrations/archived/2024-02-24/003-project-roles.sql +++ b/src/backend/migrations/archived/2024-02-24/003-project-roles.sql @@ -21,10 +21,10 @@ END $$; ALTER TYPE public.projectrole OWNER TO fmtm; ALTER TABLE public.user_roles -ALTER COLUMN "role" TYPE VARCHAR(24); +ALTER COLUMN role TYPE VARCHAR(24); ALTER TABLE public.user_roles -ALTER COLUMN "role" TYPE public.projectrole USING role::public.projectrole; +ALTER COLUMN role TYPE public.projectrole USING role::public.projectrole; -- Commit the transaction COMMIT; diff --git a/src/backend/migrations/archived/2024-02-24/revert/003-project-roles.sql b/src/backend/migrations/archived/2024-02-24/revert/003-project-roles.sql index 72e6ad5d84..930bc17dbb 100644 --- a/src/backend/migrations/archived/2024-02-24/revert/003-project-roles.sql +++ b/src/backend/migrations/archived/2024-02-24/revert/003-project-roles.sql @@ -2,9 +2,9 @@ BEGIN; -- Revert user_roles table changes -ALTER TABLE public.user_roles ALTER COLUMN "role" TYPE VARCHAR(24); +ALTER TABLE public.user_roles ALTER COLUMN role TYPE VARCHAR(24); -ALTER TABLE public.user_roles ALTER COLUMN "role" TYPE public.userrole +ALTER TABLE public.user_roles ALTER COLUMN role TYPE public.userrole USING role::public.userrole; -- Drop the public.projectrole enum diff --git a/src/backend/migrations/init/fmtm_base_schema.sql b/src/backend/migrations/init/fmtm_base_schema.sql index 38416af3d0..839b7cd5f7 100644 --- a/src/backend/migrations/init/fmtm_base_schema.sql +++ b/src/backend/migrations/init/fmtm_base_schema.sql @@ -155,11 +155,11 @@ SET default_table_access_method = heap; -- Tables -CREATE TABLE IF NOT EXISTS public."_migrations" ( +CREATE TABLE IF NOT EXISTS public._migrations ( date_executed TIMESTAMP, script_name TEXT ); -ALTER TABLE public."_migrations" OWNER TO fmtm; +ALTER TABLE public._migrations OWNER TO fmtm; CREATE TABLE public.background_tasks ( @@ -193,11 +193,11 @@ ALTER TABLE public.mbtiles_path_id_seq OWNER TO fmtm; ALTER SEQUENCE public.mbtiles_path_id_seq OWNED BY public.mbtiles_path.id; -CREATE TABLE public."_migrations" ( +CREATE TABLE public._migrations ( script_name text, date_executed timestamp without time zone ); -ALTER TABLE public."_migrations" OWNER TO fmtm; +ALTER TABLE public._migrations OWNER TO fmtm; CREATE TABLE public.organisation_managers ( @@ -260,6 +260,8 @@ CREATE TABLE public.projects ( status public.projectstatus NOT NULL DEFAULT 'DRAFT', total_tasks integer, xform_category character varying, + xlsform_content bytea, + odk_form_id character varying, visibility public.projectvisibility NOT NULL DEFAULT 'PUBLIC', mapper_level public.mappinglevel NOT NULL DEFAULT 'INTERMEDIATE', priority public.projectpriority DEFAULT 'MEDIUM', @@ -278,8 +280,6 @@ CREATE TABLE public.projects ( odk_central_user character varying, odk_central_password character varying, odk_token character varying, - form_xls bytea, - form_config_file bytea, data_extract_type character varying, data_extract_url character varying, task_split_type public.tasksplittype, @@ -390,22 +390,25 @@ CACHE 1; ALTER TABLE public.xlsforms_id_seq OWNER TO fmtm; ALTER SEQUENCE public.xlsforms_id_seq OWNED BY public.xlsforms.id; -CREATE TABLE public.xforms ( +CREATE TABLE public.submission_photos ( id integer NOT NULL, - project_id integer, - odk_form_id character varying, - category character varying + project_id integer NOT NULL, + -- Note this is not public.tasks, but an ODK task_id + task_id integer NOT NULL, + submission_id character varying NOT NULL, + s3_path character varying NOT NULL ); -ALTER TABLE public.xforms OWNER TO fmtm; -CREATE SEQUENCE public.xforms_id_seq +ALTER TABLE public.submission_photos OWNER TO fmtm; +CREATE SEQUENCE public.submission_photos_id_seq AS integer START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; -ALTER TABLE public.xforms_id_seq OWNER TO fmtm; -ALTER SEQUENCE public.xforms_id_seq OWNED BY public.xforms.id; +ALTER TABLE public.submission_photos_id_seq OWNER TO fmtm; +ALTER SEQUENCE public.submission_photos_id_seq +OWNED BY public.submission_photos.id; -- nextval for primary keys (autoincrement) @@ -427,15 +430,15 @@ ALTER TABLE ONLY public.tasks ALTER COLUMN id SET DEFAULT nextval( ALTER TABLE ONLY public.xlsforms ALTER COLUMN id SET DEFAULT nextval( 'public.xlsforms_id_seq'::regclass ); -ALTER TABLE ONLY public.xforms ALTER COLUMN id SET DEFAULT nextval( - 'public.xforms_id_seq'::regclass +ALTER TABLE ONLY public.submission_photos ALTER COLUMN id SET DEFAULT nextval( + 'public.submission_photos_id_seq'::regclass ); -- Constraints for primary keys -ALTER TABLE public."_migrations" -ADD CONSTRAINT "_migrations_pkey" PRIMARY KEY (script_name); +ALTER TABLE public._migrations +ADD CONSTRAINT _migrations_pkey PRIMARY KEY (script_name); ALTER TABLE ONLY public.background_tasks ADD CONSTRAINT background_tasks_pkey PRIMARY KEY (id); @@ -482,8 +485,8 @@ ADD CONSTRAINT xlsforms_pkey PRIMARY KEY (id); ALTER TABLE ONLY public.xlsforms ADD CONSTRAINT xlsforms_title_key UNIQUE (title); -ALTER TABLE ONLY public.xforms -ADD CONSTRAINT xforms_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.submission_photos +ADD CONSTRAINT submission_photos_pkey PRIMARY KEY (id); -- Indexing @@ -593,12 +596,11 @@ ADD CONSTRAINT user_roles_user_id_fkey FOREIGN KEY ( user_id ) REFERENCES public.users (id); -ALTER TABLE ONLY public.xforms +ALTER TABLE ONLY public.submission_photos ADD CONSTRAINT fk_project_id FOREIGN KEY ( project_id ) REFERENCES public.projects (id); - -- Finalise REVOKE USAGE ON SCHEMA public FROM public; diff --git a/src/backend/pdm.lock b/src/backend/pdm.lock index 3ec765848e..60a01b8c60 100644 --- a/src/backend/pdm.lock +++ b/src/backend/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "debug", "dev", "docs", "test", "monitoring"] strategy = ["cross_platform"] lock_version = "4.4.1" -content_hash = "sha256:8c65c80f5e570cc8e16aad773965a60059d14e05b51e55dd334de4b4082329bb" +content_hash = "sha256:d4ec67a7c937dd88ba1bf8178ee1f56b8c87bd1f3d43638d4845f4d53d07580c" [[package]] name = "aiohttp" @@ -1579,7 +1579,7 @@ files = [ [[package]] name = "osm-fieldwork" -version = "0.16.4" +version = "0.16.6" requires_python = ">=3.10" summary = "Processing field data from ODK to OpenStreetMap format." dependencies = [ @@ -1605,8 +1605,8 @@ dependencies = [ "xmltodict>=0.13.0", ] files = [ - {file = "osm-fieldwork-0.16.4.tar.gz", hash = "sha256:0285313d3e4bd99df0cccd91b8706b6d3f66ae427bab259250df19b07c51d31b"}, - {file = "osm_fieldwork-0.16.4-py3-none-any.whl", hash = "sha256:595afcf05a0a3fda035e5c2c342b5a5c1bcfa2e21002098f6c670c7e502baf93"}, + {file = "osm-fieldwork-0.16.6.tar.gz", hash = "sha256:452bcc31c8910addb9392ec5506295c8143afb1663c9909ea78e4cc457601a47"}, + {file = "osm_fieldwork-0.16.6-py3-none-any.whl", hash = "sha256:acdbf4c735c9b033d3905b054a84a74c3473ca3f03963313b1db2f8a03660163"}, ] [[package]] diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index 1829caf205..70e780ed98 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -42,10 +42,9 @@ dependencies = [ "asgiref==3.8.1", "sozipfile==0.3.2", "cryptography>=42.0.8", - "defusedxml>=0.7.1", "pyjwt>=2.8.0", "async-lru>=2.0.4", - "osm-fieldwork>=0.16.4", + "osm-fieldwork>=0.16.6", "osm-login-python==2.0.0", "osm-rawdata==0.3.2", "fmtm-splitter==1.3.1", @@ -148,7 +147,7 @@ update_changelog_on_bump = true [tool.sqlfluff.core] dialect = "postgres" large_file_skip_byte_limit = 30000 # Required to process fmtm_base_schema.sql -exclude_rules = "CP05" # Avoid capitalisation of enums +exclude_rules = "CP05, RF02" # Avoid capitalisation of enums [tool.codespell] skip = "contrib/*.py,*languages_and_countries.py,*pnpm-lock.yaml,*CHANGELOG.md" diff --git a/src/frontend/src/components/DialogTaskActions.tsx b/src/frontend/src/components/DialogTaskActions.tsx index cac25bb9d2..a9692f986d 100755 --- a/src/frontend/src/components/DialogTaskActions.tsx +++ b/src/frontend/src/components/DialogTaskActions.tsx @@ -232,7 +232,7 @@ export default function Dialog({ taskId, feature }: dialogPropType) { ); if (isMobile) { - document.location.href = `odkcollect://form/${projectInfo.xform_id}?task_filter=${taskId}`; + document.location.href = `odkcollect://form/${projectInfo.xform_id}`; } else { dispatch( CommonActions.SetSnackBar({ diff --git a/src/frontend/src/views/SubmissionDetails.tsx b/src/frontend/src/views/SubmissionDetails.tsx index 680ddb2aa9..0e4a233d0e 100644 --- a/src/frontend/src/views/SubmissionDetails.tsx +++ b/src/frontend/src/views/SubmissionDetails.tsx @@ -94,11 +94,7 @@ const SubmissionDetails = () => { const projectDashboardLoading = useAppSelector((state) => state.project.projectDashboardLoading); const submissionDetails = useAppSelector((state) => state.submission.submissionDetails); const submissionDetailsLoading = useAppSelector((state) => state.submission.submissionDetailsLoading); - const taskId = submissionDetails?.task_id - ? submissionDetails?.task_id - : submissionDetails?.task_filter - ? submissionDetails?.task_filter - : '-'; + const taskId = submissionDetails?.task_id ? submissionDetails?.task_id : '-'; const { start, end, today, deviceid, ...restSubmissionDetails } = submissionDetails || {}; const dateDeviceDetails = { start, end, today, deviceid }; @@ -158,7 +154,7 @@ const SubmissionDetails = () => { const newFeaturePoint = { type: 'Feature', geometry: { - ...restSubmissionDetails?.new_feature_point, + ...restSubmissionDetails?.new_feature, }, properties: {}, }; @@ -250,7 +246,7 @@ const SubmissionDetails = () => { featureGeojson={ submissionDetailsLoading ? {} - : restSubmissionDetails?.new_feature === 'yes' + : restSubmissionDetails?.new_feature ? newFeaturePoint : coordinatesArray ? geojsonFeature