fix: mapper frontend task display and event POSTs (#1842)

* build: add primary key constraint on odk_entities.entity_id

* feat: working update of entity statuses in fmtm db

* docs: comment during proj create

* fix: task display and event creation in mapper frontend

* build: use correct port mapping for electric

docker-compose.yml

@@ -282,7 +282,7 @@ services:
       # AUTH_JWT_KEY: ${ENCRYPTION_KEY}
       # AUTH_JWT_AUD: ${FMTM_DOMAIN}
     ports:
-      - "7055:7055"
+      - "7055:3000"
     networks:
       - fmtm-net
     restart: "unless-stopped"

src/backend/app/db/models.py

@@ -41,6 +41,7 @@
 from app.db.enums import (
     BackgroundTaskStatus,
     CommunityType,
+    EntityState,
     HTTPStatus,
     MappingLevel,
     MappingState,
@@ -842,7 +843,7 @@ async def create(
 
         Args:
             db (Connection): The database connection.
-            project_id (int): The organisation ID.
+            project_id (int): The project ID.
             tasks (geojson.FeatureCollection): FeatureCollection of task areas.
 
         Returns:
@@ -1296,6 +1297,79 @@ async def delete(cls, db: Connection, project_id: int) -> bool:
             )
 
 
+class DbOdkEntities(BaseModel):
+    """Table odk_entities.
+
+    Mirror tracking the status of Entities in ODK.
+    """
+
+    entity_id: UUID
+    status: EntityState
+    project_id: int
+    task_id: int
+
+    @classmethod
+    async def upsert(
+        cls,
+        db: Connection,
+        project_id: int,
+        entities: list[Self],
+    ) -> bool:
+        """Update or insert Entity data, with statuses.
+
+        Args:
+            db (Connection): The database connection.
+            project_id (int): The project ID.
+            entities (list[Self]): List of DbOdkEntities objects.
+
+        Returns:
+            bool: Success or failure.
+        """
+        log.info(
+            f"Updating FMTM database Entities for project {project_id} "
+            f"with ({len(entities)}) features"
+        )
+
+        sql = """
+            INSERT INTO public.odk_entities
+                (entity_id, status, project_id, task_id)
+            VALUES
+        """
+
+        # Prepare data for bulk insert
+        values = []
+        data = {}
+        for index, entity in enumerate(entities):
+            entity_index = f"entity_{index}"
+            values.append(
+                f"(%({entity_index}_entity_id)s, "
+                f"%({entity_index}_status)s, "
+                f"%({entity_index}_project_id)s, "
+                f"%({entity_index}_task_id)s)"
+            )
+            data[f"{entity_index}_entity_id"] = entity["id"]
+            data[f"{entity_index}_status"] = EntityState(int(entity["status"])).name
+            data[f"{entity_index}_project_id"] = project_id
+            task_id = entity["task_id"]
+            data[f"{entity_index}_task_id"] = int(task_id) if task_id else None
+
+        sql += (
+            ", ".join(values)
+            + """
+            ON CONFLICT (entity_id) DO UPDATE SET
+                status = EXCLUDED.status,
+                task_id = EXCLUDED.task_id
+            RETURNING True;
+        """
+        )
+
+        async with db.cursor() as cur:
+            await cur.execute(sql, data)
+            result = await cur.fetchall()
+
+        return bool(result)
+
+
 class DbBackgroundTask(BaseModel):
     """Table background_tasks.
 

src/backend/app/projects/project_crud.py

@@ -532,8 +532,8 @@ async def generate_project_files(
 
     # Split extract by task area
     log.debug("Splitting data extract per task area")
-    # TODO in future this splitting could be removed as the task_id is
-    # no longer used in the XLSForm
+    # TODO in future this splitting could be removed if the task_id is
+    # no longer used in the XLSForm for the map filter
     task_extract_dict = await split_geojson_by_task_areas(
         db, feature_collection, project_id
     )

src/backend/app/projects/project_routes.py

@@ -57,7 +57,14 @@
     ProjectRole,
     XLSFormType,
 )
-from app.db.models import DbBackgroundTask, DbBasemap, DbProject, DbTask, DbUserRole
+from app.db.models import (
+    DbBackgroundTask,
+    DbBasemap,
+    DbOdkEntities,
+    DbProject,
+    DbTask,
+    DbUserRole,
+)
 from app.db.postgis_utils import (
     check_crs,
     featcol_keep_single_geom_type,
@@ -174,14 +181,20 @@ async def get_odk_entities_geojson(
     response_model=list[central_schemas.EntityMappingStatus],
 )
 async def get_odk_entities_mapping_statuses(
+    project_id: int,
     project: Annotated[DbProject, Depends(project_deps.get_project)],
     db: Annotated[Connection, Depends(db_conn)],
 ):
     """Get the ODK entities mapping statuses, i.e. in progress or complete."""
-    return await central_crud.get_entities_data(
+    entities = await central_crud.get_entities_data(
         project.odk_credentials,
         project.odkid,
     )
+    # First update the Entity statuses in the db
+    # FIXME this is a hack and in the long run should be replaced
+    # https://github.com/hotosm/fmtm/issues/1841
+    await DbOdkEntities.upsert(db, project_id, entities)
+    return entities
 
 
 @router.get(

src/backend/migrations/009-entity-table.sql

@@ -0,0 +1,36 @@
+-- ## Migration to:
+-- * Add a new table that syncs the ODK Entity status to FMTM.
+-- * Add a primary key for entity_id field.
+-- * Add two indexes on entity_id + project_id / task_id
+
+
+-- Start a transaction
+BEGIN;
+
+CREATE TABLE IF NOT EXISTS public.odk_entities (
+    entity_id UUID NOT NULL,
+    status entitystate NOT NULL,
+    project_id integer NOT NULL,
+    task_id integer
+);
+ALTER TABLE public.odk_entities OWNER TO fmtm;
+
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'odk_entities_pkey') THEN
+        ALTER TABLE ONLY public.odk_entities
+        ADD CONSTRAINT odk_entities_pkey PRIMARY KEY (entity_id);
+    END IF;
+END $$;
+
+CREATE INDEX IF NOT EXISTS idx_entities_project_id
+ON public.odk_entities USING btree (
+    entity_id, project_id
+);
+CREATE INDEX IF NOT EXISTS idx_entities_task_id
+ON public.odk_entities USING btree (
+    entity_id, task_id
+);
+
+-- Commit the transaction
+COMMIT;

src/backend/migrations/init/fmtm_base_schema.sql

@@ -160,7 +160,7 @@ SET default_table_access_method = heap;
 
 -- Tables
 
-CREATE TABLE IF NOT EXISTS public._migrations (
+CREATE TABLE public._migrations (
     script_name text,
     date_executed timestamp with time zone
 );
@@ -339,6 +339,14 @@ CREATE TABLE public.users (
 );
 ALTER TABLE public.users OWNER TO fmtm;
 
+CREATE TABLE public.odk_entities (
+    entity_id UUID NOT NULL,
+    status entitystate NOT NULL,
+    project_id integer NOT NULL,
+    task_id integer
+);
+ALTER TABLE public.odk_entities OWNER TO fmtm;
+
 CREATE TABLE public.xlsforms (
     id integer NOT NULL,
     title character varying,
@@ -435,6 +443,9 @@ ADD CONSTRAINT users_pkey PRIMARY KEY (id);
 ALTER TABLE ONLY public.users
 ADD CONSTRAINT users_username_key UNIQUE (username);
 
+ALTER TABLE ONLY public.odk_entities
+ADD CONSTRAINT odk_entities_pkey PRIMARY KEY (entity_id);
+
 ALTER TABLE ONLY public.xlsforms
 ADD CONSTRAINT xlsforms_pkey PRIMARY KEY (id);
 
@@ -447,16 +458,16 @@ ADD CONSTRAINT submission_photos_pkey PRIMARY KEY (id);
 -- Indexing
 
 CREATE INDEX idx_projects_outline ON public.projects USING gist (outline);
-CREATE INDEX IF NOT EXISTS idx_projects_mapper_level
+CREATE INDEX idx_projects_mapper_level
 ON public.projects USING btree (
     mapper_level
 );
-CREATE INDEX IF NOT EXISTS idx_projects_organisation_id
+CREATE INDEX idx_projects_organisation_id
 ON public.projects USING btree (
     organisation_id
 );
 CREATE INDEX idx_tasks_outline ON public.tasks USING gist (outline);
-CREATE INDEX IF NOT EXISTS idx_tasks_composite
+CREATE INDEX idx_tasks_composite
 ON public.tasks USING btree (
     id, project_id
 );
@@ -466,26 +477,34 @@ CREATE INDEX idx_user_roles ON public.user_roles USING btree (
 CREATE INDEX idx_org_managers ON public.organisation_managers USING btree (
     user_id, organisation_id
 );
-CREATE INDEX IF NOT EXISTS idx_task_event_composite
+CREATE INDEX idx_task_event_composite
 ON public.task_events USING btree (
     task_id, project_id
 );
-CREATE INDEX IF NOT EXISTS idx_task_event_project_user
+CREATE INDEX idx_task_event_project_user
 ON public.task_events USING btree (
     user_id, project_id
 );
-CREATE INDEX IF NOT EXISTS idx_task_event_project_id
+CREATE INDEX idx_task_event_project_id
 ON public.task_events USING btree (
     task_id, project_id
 );
-CREATE INDEX IF NOT EXISTS idx_task_event_user_id
+CREATE INDEX idx_task_event_user_id
 ON public.task_events USING btree (
     task_id, user_id
 );
-CREATE INDEX IF NOT EXISTS idx_task_history_date
+CREATE INDEX idx_task_history_date
 ON public.task_history USING btree (
     task_id, created_at
 );
+CREATE INDEX idx_entities_project_id
+ON public.odk_entities USING btree (
+    entity_id, project_id
+);
+CREATE INDEX idx_entities_task_id
+ON public.odk_entities USING btree (
+    entity_id, task_id
+);
 
 -- Foreign keys
 

src/mapper/src/lib/components/page/layer-switcher.svelte

@@ -1,5 +1,5 @@
 <script>
-	import { clickOutside } from '../../../utilFunctions/clickOutside.ts';
+	import { clickOutside } from '$lib/utils/clickOutside.ts';
 
 	let isOpen = false;
 </script>

src/mapper/src/lib/components/page/legend.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
-	import { clickOutside } from '../../../utilFunctions/clickOutside.ts';
-	import LockImg from '../../../assets/images/black-lock.png';
-	import '../../../styles/page.css';
+	import { clickOutside } from '$lib/utils/clickOutside.ts';
+	import LockImg from '$assets/images/black-lock.png';
+	import '$styles/page.css';
 
 	type taskStatusesType = { status: string; color?: string; icon?: string };