From d05d4c3512fb7d8d3174ab2514d3a8bb58437dc0 Mon Sep 17 00:00:00 2001 From: Sujan Adhikari <109404840+Sujanadh@users.noreply.github.com> Date: Thu, 2 Jan 2025 10:03:24 +0545 Subject: [PATCH] create a table to store bad and new geoms (#2046) * feat: create geometrylog table to store bad and new geoms * feat: add geometrylog migration in fmtm base schema --- src/backend/app/db/enums.py | 7 +++ src/backend/app/db/models.py | 61 +++++++++++++++++++ src/backend/app/projects/project_routes.py | 24 ++++++++ src/backend/app/projects/project_schemas.py | 29 +++++++-- .../migrations/002-create-geometry-log.sql | 21 +++++++ .../migrations/init/fmtm_base_schema.sql | 14 +++++ .../revert/002-create-geometry-log.sql | 11 ++++ 7 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 src/backend/migrations/002-create-geometry-log.sql create mode 100644 src/backend/migrations/revert/002-create-geometry-log.sql diff --git a/src/backend/app/db/enums.py b/src/backend/app/db/enums.py index 9b8ea4d31c..974f2d307a 100644 --- a/src/backend/app/db/enums.py +++ b/src/backend/app/db/enums.py @@ -251,3 +251,10 @@ class XLSFormType(StrEnum, Enum): # religious = "religious" # landusage = "landusage" # waterways = "waterways" + + +class GeomStatus(StrEnum, Enum): + """Geometry status.""" + + NEW = "NEW" + BAD = "BAD" diff --git a/src/backend/app/db/models.py b/src/backend/app/db/models.py index 24c990dba1..134894985b 100644 --- a/src/backend/app/db/models.py +++ b/src/backend/app/db/models.py @@ -43,6 +43,7 @@ BackgroundTaskStatus, CommunityType, EntityState, + GeomStatus, HTTPStatus, MappingLevel, MappingState, @@ -70,6 +71,7 @@ BackgroundTaskUpdate, BasemapIn, BasemapUpdate, + GeometryLogIn, ProjectIn, ProjectUpdate, ) @@ -1741,3 +1743,62 @@ def slugify(name: Optional[str]) -> Optional[str]: # Remove consecutive hyphens slug = sub(r"[-\s]+", "-", slug) return slug + + +class DbGeometryLog(BaseModel): + """Table geometry log.""" + + geom: dict + status: GeomStatus + project_id: Optional[int] + task_id: Optional[int] + + @classmethod + async def create( + cls, + db: Connection, + geometry_log_in: "GeometryLogIn", + ) -> Self: + """Creates a new geometry log with its status.""" + model_dump = dump_and_check_model(geometry_log_in) + columns = [] + value_placeholders = [] + + for key in model_dump.keys(): + columns.append(key) + if key == "geom": + value_placeholders.append(f"ST_GeomFromGeoJSON(%({key})s)") + # Must be string json for db input + model_dump[key] = json.dumps(model_dump[key]) + else: + value_placeholders.append(f"%({key})s") + async with db.cursor(row_factory=class_row(cls)) as cur: + await cur.execute( + f""" + INSERT INTO geometrylog + ({", ".join(columns)}) + VALUES + ({", ".join(value_placeholders)}) + RETURNING + *, + ST_AsGeoJSON(geom)::jsonb AS geom; + """, + model_dump, + ) + new_geomlog = await cur.fetchone() + return new_geomlog + + @classmethod + async def delete( + cls, + db: Connection, + id: int, + ) -> bool: + """Delete a geometry.""" + async with db.cursor() as cur: + await cur.execute( + """ + DELETE FROM geometrylog WHERE id = %(id)s; + """, + {"id": id}, + ) diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index e21b225826..159b558fb0 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -61,6 +61,7 @@ from app.db.models import ( DbBackgroundTask, DbBasemap, + DbGeometryLog, DbOdkEntities, DbProject, DbTask, @@ -1254,3 +1255,26 @@ async def download_task_boundaries( } return Response(content=out, headers=headers) + + +@router.post("/{project_id}/geometries") +async def create_geom_log( + geom_log: project_schemas.GeometryLogIn, + db: Annotated[Connection, Depends(db_conn)], +): + """Creates a new entry in the geometries log table. + + Returns: + geometries (DbGeometryLog): The created geometries log entry. + + Raises: + HTTPException: If the geometries log creation fails. + """ + geometries = await DbGeometryLog.create(db, geom_log) + if not geometries: + raise HTTPException( + status_code=HTTPStatus.CONFLICT, + detail="geometries log creation failed.", + ) + + return geometries diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index 7323ad38f6..a739615330 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -33,10 +33,7 @@ from app.central.central_schemas import ODKCentralDecrypted, ODKCentralIn from app.config import decrypt_value, encrypt_value, settings -from app.db.enums import ( - BackgroundTaskStatus, - ProjectPriority, -) +from app.db.enums import BackgroundTaskStatus, GeomStatus, ProjectPriority from app.db.models import DbBackgroundTask, DbBasemap, DbProject, slugify from app.db.postgis_utils import ( geojson_to_featcol, @@ -46,6 +43,30 @@ ) +class GeometryLogIn(BaseModel): + """Geometry log insert.""" + + status: GeomStatus + geom: dict + project_id: Optional[int] + task_id: Optional[int] + + @field_validator("geom", mode="before") + @classmethod + def parse_input_geometry( + cls, + value: FeatureCollection | Feature | MultiPolygon | Polygon, + ) -> Optional[dict]: + """Parse any format geojson into featurecollection. + + Return geometry only. + """ + if value is None: + return None + featcol = geojson_to_featcol(value) + return featcol.get("features")[0].get("geometry") + + class ProjectInBase(DbProject): """Base model for project insert / update (validators).""" diff --git a/src/backend/migrations/002-create-geometry-log.sql b/src/backend/migrations/002-create-geometry-log.sql new file mode 100644 index 0000000000..63cf426341 --- /dev/null +++ b/src/backend/migrations/002-create-geometry-log.sql @@ -0,0 +1,21 @@ +-- ## Migration to create a table to store new and bad geometries. + +-- Start a transaction + +BEGIN; + +CREATE TABLE geometrylog ( + id SERIAL PRIMARY KEY, + geom GEOMETRY NOT NULL, + status geomstatus, + project_id int, + task_id int +); + +ALTER TABLE geometrylog OWNER TO fmtm; + +-- Indexes for efficient querying +CREATE INDEX idx_geometrylog ON geometrylog USING gist (geom); + +-- Commit the transaction +COMMIT; diff --git a/src/backend/migrations/init/fmtm_base_schema.sql b/src/backend/migrations/init/fmtm_base_schema.sql index 4015d61cb0..27bb2daaf1 100644 --- a/src/backend/migrations/init/fmtm_base_schema.sql +++ b/src/backend/migrations/init/fmtm_base_schema.sql @@ -386,6 +386,15 @@ ALTER TABLE public.submission_photos_id_seq OWNER TO fmtm; ALTER SEQUENCE public.submission_photos_id_seq OWNED BY public.submission_photos.id; +CREATE TABLE geometrylog ( + id SERIAL PRIMARY KEY, + geom GEOMETRY NOT NULL, + status geomstatus, + project_id int, + task_id int +); +ALTER TABLE geometrylog OWNER TO fmtm; + -- nextval for primary keys (autoincrement) ALTER TABLE ONLY public.organisations ALTER COLUMN id SET DEFAULT nextval( @@ -458,6 +467,9 @@ ADD CONSTRAINT xlsforms_title_key UNIQUE (title); ALTER TABLE ONLY public.submission_photos ADD CONSTRAINT submission_photos_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.idx_geometrylog +ADD CONSTRAINT geometrylog_pkey PRIMARY KEY (id); + -- Indexing CREATE INDEX idx_projects_outline ON public.projects USING gist (outline); @@ -504,6 +516,8 @@ CREATE INDEX idx_entities_task_id ON public.odk_entities USING btree ( entity_id, task_id ); +CREATE INDEX idx_geometrylog +ON geometrylog USING gist (geom); -- Foreign keys diff --git a/src/backend/migrations/revert/002-create-geometry-log.sql b/src/backend/migrations/revert/002-create-geometry-log.sql new file mode 100644 index 0000000000..d9298cb02f --- /dev/null +++ b/src/backend/migrations/revert/002-create-geometry-log.sql @@ -0,0 +1,11 @@ +-- ## Migration to delete geometrylog table. + +-- Start a transaction + +BEGIN; + +DROP TABLE IF EXISTS geometrylog; + +-- Commit the transaction + +COMMIT;