diff --git a/src/backend/app/auth/roles.py b/src/backend/app/auth/roles.py index 84180d6e3f..ff5c9c923f 100644 --- a/src/backend/app/auth/roles.py +++ b/src/backend/app/auth/roles.py @@ -252,7 +252,7 @@ async def wrap_check_access( if not db_user: raise HTTPException( status_code=HTTPStatus.FORBIDDEN, - detail="User is not a project manager", + detail="User do not have permission to access the project.", ) return { @@ -321,17 +321,29 @@ async def mapper( project: DbProject = Depends(get_project_by_id), db: Session = Depends(get_db), user_data: AuthUser = Depends(login_required), -) -> AuthUser: +) -> ProjectUserDict: """A mapper for a specific project.""" # If project is public, skip permission check if project.visibility == ProjectVisibility.PUBLIC: - return user_data + user_id = user_data.id + sql = text("SELECT * FROM users WHERE id = :user_id;") + result = db.execute(sql, {"user_id": user_id}) + db_user = result.first() + + if not db_user: + raise HTTPException( + status_code=HTTPStatus.NOT_FOUND, + detail=f"User ({user_id}) does not exist in database", + ) + + return { + "user": DbUser(**db_user._asdict()), + "project": project, + } - await wrap_check_access( + return await wrap_check_access( project, db, user_data, ProjectRole.MAPPER, ) - - return user_data diff --git a/src/backend/app/central/central_crud.py b/src/backend/app/central/central_crud.py index 85f3061e10..6d1ed4670f 100644 --- a/src/backend/app/central/central_crud.py +++ b/src/backend/app/central/central_crud.py @@ -22,7 +22,7 @@ import os import uuid from io import BytesIO, StringIO -from typing import Optional +from typing import Optional, Union from xml.etree.ElementTree import Element, SubElement import geojson @@ -624,8 +624,8 @@ def flatten_json(data: dict, target: dict): async def convert_odk_submission_json_to_geojson( - input_json: BytesIO, -) -> BytesIO: + input_json: Union[BytesIO, list], +) -> geojson.FeatureCollection: """Convert ODK submission JSON file to GeoJSON. Used for loading into QGIS. @@ -636,7 +636,10 @@ async def convert_odk_submission_json_to_geojson( Returns: geojson (BytesIO): GeoJSON format ODK submission. """ - submission_json = json.loads(input_json.getvalue()) + if isinstance(input_json, list): + submission_json = input_json + else: + submission_json = json.loads(input_json.getvalue()) if not submission_json: raise HTTPException( @@ -660,9 +663,7 @@ async def convert_odk_submission_json_to_geojson( feature = geojson.Feature(geometry=geojson_geom, properties=data) all_features.append(feature) - featcol = geojson.FeatureCollection(features=all_features) - - return BytesIO(json.dumps(featcol).encode("utf-8")) + return geojson.FeatureCollection(features=all_features) async def get_entities_geojson( diff --git a/src/backend/app/db/postgis_utils.py b/src/backend/app/db/postgis_utils.py index af2d9cc706..6f23e437ee 100644 --- a/src/backend/app/db/postgis_utils.py +++ b/src/backend/app/db/postgis_utils.py @@ -21,6 +21,7 @@ import logging from asyncio import gather from datetime import datetime, timezone +from io import BytesIO from random import getrandbits from typing import Optional, Union @@ -32,15 +33,20 @@ from geoalchemy2.shape import from_shape, to_shape from geojson_pydantic import Feature, MultiPolygon, Polygon from geojson_pydantic import FeatureCollection as FeatCol +from osm_fieldwork.data_models import data_models_path +from osm_rawdata.postgres import PostgresClient from shapely.geometry import mapping, shape +from shapely.geometry.base import BaseGeometry from shapely.ops import unary_union from sqlalchemy import text from sqlalchemy.exc import ProgrammingError from sqlalchemy.orm import Session from app.config import settings +from app.models.enums import XLSFormType log = logging.getLogger(__name__) +API_URL = settings.RAW_DATA_API_URL def timestamp(): @@ -785,3 +791,119 @@ def parse_featcol(features: Union[Feature, FeatCol, MultiPolygon, Polygon]): elif isinstance(features, Feature): feat_col = geojson.FeatureCollection([feat_col]) return feat_col + + +def get_osm_geometries(form_category, geometry): + """Request a snapshot based on the provided geometry. + + Args: + form_category(str): feature category type (eg: buildings). + geometry (str): The geometry data in JSON format. + + Returns: + dict: The JSON response containing the snapshot data. + """ + config_filename = XLSFormType(form_category).name + data_model = f"{data_models_path}/{config_filename}.yaml" + + with open(data_model, "rb") as data_model_yaml: + extract_config = BytesIO(data_model_yaml.read()) + + pg = PostgresClient( + "underpass", + extract_config, + auth_token=settings.RAW_DATA_API_AUTH_TOKEN + if settings.RAW_DATA_API_AUTH_TOKEN + else None, + ) + return pg.execQuery( + geometry, + extra_params={ + "outputType": "geojson", + "bind_zip": True, + "useStWithin": False, + }, + ) + + +def geometries_almost_equal( + geom1: BaseGeometry, geom2: BaseGeometry, tolerance: float = 1e-6 +) -> bool: + """Determine if two geometries are almost equal within a tolerance. + + Args: + geom1 (BaseGeometry): First geometry. + geom2 (BaseGeometry): Second geometry. + tolerance (float): Tolerance level for almost equality. + + Returns: + bool: True if geometries are almost equal else False. + """ + return geom1.equals_exact(geom2, tolerance) + + +def check_partial_overlap(geom1: BaseGeometry, geom2: BaseGeometry) -> bool: + """Determine if two geometries have a partial overlap. + + Args: + geom1 (BaseGeometry): First geometry. + geom2 (BaseGeometry): Second geometry. + + Returns: + bool: True if geometries have a partial overlap, else False. + """ + intersection = geom1.intersection(geom2) + return not intersection.is_empty and ( + 0 < intersection.area < geom1.area and 0 < intersection.area < geom2.area + ) + + +def conflate_features( + input_features: list, osm_features: list, remove_conflated=False, tolerance=1e-6 +): + """Conflate input features with OSM features to identify overlaps. + + Args: + input_features (list): A list of input features with geometries. + osm_features (list): A list of OSM features with geometries. + remove_conflated (bool): Flag to remove conflated features. + tolerance (float): Tolerance level for almost equality. + + Returns: + list: A list of features after conflation with OSM features. + """ + osm_geometries = [shape(feature["geometry"]) for feature in osm_features] + return_features = [] + + for input_feature in input_features: + input_geometry = shape(input_feature["geometry"]) + is_duplicate = False + is_partial_overlap = False + + for osm_feature, osm_geometry in zip( + osm_features, osm_geometries, strict=False + ): + if geometries_almost_equal(input_geometry, osm_geometry, tolerance): + is_duplicate = True + input_feature["properties"].update(osm_feature["properties"]) + break + + if check_partial_overlap(input_geometry, osm_geometry): + is_partial_overlap = True + new_feature = { + "type": "Feature", + "geometry": mapping(osm_feature["geometry"]), + "properties": osm_feature["properties"], + } + return_features.append(new_feature) + break + + input_feature["properties"]["is_duplicate"] = is_duplicate + input_feature["properties"]["is_partial_overlap"] = is_partial_overlap + + if (is_duplicate or is_partial_overlap) and remove_conflated is True: + continue + + return_features.append(input_feature) + + return return_features diff --git a/src/backend/app/helpers/helper_routes.py b/src/backend/app/helpers/helper_routes.py index b4dc907f49..62c83fd6c0 100644 --- a/src/backend/app/helpers/helper_routes.py +++ b/src/backend/app/helpers/helper_routes.py @@ -225,9 +225,10 @@ async def convert_odk_submission_json_to_geojson_wrapper( contents = await json_file.read() submission_geojson = await convert_odk_submission_json_to_geojson(BytesIO(contents)) + submission_data = BytesIO(json.dumps(submission_geojson).encode("utf-8")) headers = {"Content-Disposition": f"attachment; filename={filename.stem}.geojson"} - return Response(submission_geojson.getvalue(), headers=headers) + return Response(submission_data.getvalue(), headers=headers) @router.get("/view-raw-data-api-token") diff --git a/src/backend/app/submissions/submission_crud.py b/src/backend/app/submissions/submission_crud.py index ff00db3407..5bbf812fff 100644 --- a/src/backend/app/submissions/submission_crud.py +++ b/src/backend/app/submissions/submission_crud.py @@ -35,7 +35,11 @@ # from osm_fieldwork.json2osm import json2osm from sqlalchemy.orm import Session -from app.central.central_crud import get_odk_form, get_odk_project, list_odk_xforms +from app.central.central_crud import ( + get_odk_form, + get_odk_project, + list_odk_xforms, +) from app.config import settings from app.db import db_models from app.models.enums import HTTPStatus @@ -524,3 +528,45 @@ async def get_submission_detail( odk_form.getSubmissions(project.odkid, db_xform.odk_form_id, submission_id) ) return submission.get("value", [])[0] + + +# FIXME might not needed +# async def get_submission_geojson( +# project_id: int, +# db: Session, +# ): +# """Retrieve GeoJSON data for a submission associated with a project. + +# Args: +# project_id (int): The ID of the project. +# db (Session): The database session. + +# Returns: +# FeatCol: A GeoJSON FeatCol containing the submission features. +# """ +# data = await get_submission_by_project(project_id, {}, db) +# submission_json = data.get("value", []) + +# if not submission_json: +# raise HTTPException( +# status_code=HTTPStatus.UNPROCESSABLE_ENTITY, +# detail="Loading JSON submission failed", +# ) + +# all_features = [] +# for submission in submission_json: +# keys_to_remove = ["meta", "__id", "__system"] +# for key in keys_to_remove: +# submission.pop(key) + +# data = {} +# flatten_json(submission, data) + +# geojson_geom = await postgis_utils.javarosa_to_geojson_geom( +# data.pop("xlocation", {}), geom_type="Polygon" +# ) + +# feature = geojson.Feature(geometry=geojson_geom, properties=data) +# all_features.append(feature) + +# return geojson.FeatureCollection(features=all_features) diff --git a/src/backend/app/submissions/submission_routes.py b/src/backend/app/submissions/submission_routes.py index a69f9838f9..d8c46c6e4c 100644 --- a/src/backend/app/submissions/submission_routes.py +++ b/src/backend/app/submissions/submission_routes.py @@ -32,9 +32,10 @@ from app.auth.roles import mapper, project_manager from app.central import central_crud from app.db import database, db_models, postgis_utils -from app.models.enums import HTTPStatus, ReviewStateEnum +from app.models.enums import ReviewStateEnum from app.projects import project_crud, project_deps from app.submissions import submission_crud, submission_schemas +from app.tasks.task_deps import get_task_by_id router = APIRouter( prefix="/submission", @@ -558,32 +559,56 @@ async def download_submission_geojson( data = await submission_crud.get_submission_by_project(project_id, {}, db) submission_json = data.get("value", []) - if not submission_json: - raise HTTPException( - status_code=HTTPStatus.UNPROCESSABLE_ENTITY, - detail="Loading JSON submission failed", - ) + submission_geojson = await central_crud.convert_odk_submission_json_to_geojson( + submission_json + ) + submission_data = BytesIO(json.dumps(submission_geojson).encode("utf-8")) + filename = project.project_name_prefix - all_features = [] - for submission in submission_json: - keys_to_remove = ["meta", "__id", "__system"] - for key in keys_to_remove: - submission.pop(key) + headers = {"Content-Disposition": f"attachment; filename={filename}.geojson"} - data = {} - central_crud.flatten_json(submission, data) + return Response(submission_data.getvalue(), headers=headers) - geojson_geom = await postgis_utils.javarosa_to_geojson_geom( - data.pop("xlocation", {}), geom_type="Polygon" - ) - feature = geojson.Feature(geometry=geojson_geom, properties=data) - all_features.append(feature) +@router.get("/conflate_submission_geojson/") +async def conflate_geojson( + task_id: int, + current_user: dict = Depends(mapper), # FIXME change this validator + remove_conflated=False, + db: Session = Depends(database.get_db), +): + """Conflates the input GeoJSON with OpenStreetMap data. - featcol = geojson.FeatureCollection(features=all_features) - submission_geojson = BytesIO(json.dumps(featcol).encode("utf-8")) - filename = project.project_name_prefix + Args: + task_id(int): task index of project. + current_user(dict): Check if user is mapper. + remove_conflated(bool): returns geojson which are not overlapped with osm data. + db (Session): The database session. - headers = {"Content-Disposition": f"attachment; filename={filename}.geojson"} + Returns: + str: Updated GeoJSON string with conflated features. + """ + try: + project = current_user["project"] + db_task = await get_task_by_id(project.id, task_id, db) + task_aoi = postgis_utils.geometry_to_geojson(db_task.outline) + task_geojson = geojson.dumps(task_aoi, indent=2) + + data = await submission_crud.get_submission_by_project(project.id, {}, db) + submission_json = data.get("value", []) - return Response(submission_geojson.getvalue(), headers=headers) + submission_geojson = await central_crud.convert_odk_submission_json_to_geojson( + submission_json + ) + form_category = project.xform_category + input_features = submission_geojson["features"] + osm_features = postgis_utils.get_osm_geometries(form_category, task_geojson) + submission_geojson["features"] = postgis_utils.conflate_features( + input_features, osm_features.get("features", []), remove_conflated + ) + + return submission_geojson + except Exception as e: + raise HTTPException( + status_code=500, detail=f"Failed to process conflation: {str(e)}" + ) from e diff --git a/src/backend/app/tasks/task_deps.py b/src/backend/app/tasks/task_deps.py index d3160699bf..4ef2d8ab35 100644 --- a/src/backend/app/tasks/task_deps.py +++ b/src/backend/app/tasks/task_deps.py @@ -25,7 +25,7 @@ from sqlalchemy.orm import Session from app.db.database import get_db -from app.db.db_models import DbProject +from app.db.db_models import DbProject, DbTask from app.models.enums import HTTPStatus @@ -34,7 +34,7 @@ async def get_xform_name( task_id: int, db: Session = Depends(get_db), ) -> str: - """Get a single project by id.""" + """Get a project xform name.""" if isinstance(project, int): db_project = db.query(DbProject).filter(DbProject.id == project).first() if not db_project: @@ -51,3 +51,22 @@ async def get_xform_name( # TODO this in the new xforms.category field/table. form_name = project_name return form_name + + +async def get_task_by_id( + project_id: int, + task_id: int, + db: Session = Depends(get_db), +): + """Get a single task by task index.""" + if ( + db_task := db.query(DbTask) + .filter(DbTask.project_task_index == task_id, DbTask.project_id == project_id) + .first() + ): + return db_task + else: + raise HTTPException( + status_code=HTTPStatus.NOT_FOUND, + detail=f"Task with ID {task_id} does not exist", + )