Update migration idempotence, add org email field, extra project fiel…

…ds (#2072)

* build: update migration idempotence, add org email field, project fields

* feat(backend): add new project + org fields to response models

* build: tweak migrations to work when applied on fresh instance

src/backend/app/db/enums.py

@@ -234,6 +234,14 @@ class GeometryType(StrEnum, Enum):
     Point = "Point"
 
 
+class DbGeomType(StrEnum, Enum):
+    """Enum in the database, all geom types are in caps."""
+
+    POINT = "POINT"
+    POLYGON = "POLYGON"
+    LINESTRING = "LINESTRING"
+
+
 class XLSFormType(StrEnum, Enum):
     """XLSForm categories bundled by default.
 

src/backend/app/db/models.py

@@ -34,7 +34,7 @@
 from psycopg import Connection
 from psycopg.errors import UniqueViolation
 from psycopg.rows import class_row
-from pydantic import AwareDatetime, BaseModel, Field, ValidationInfo
+from pydantic import AwareDatetime, BaseModel, Field, PositiveInt, ValidationInfo
 from pydantic.functional_validators import field_validator
 
 from app.central.central_schemas import ODKCentralDecrypted
@@ -43,6 +43,7 @@
 from app.db.enums import (
     BackgroundTaskStatus,
     CommunityType,
+    DbGeomType,
     EntityState,
     GeomStatus,
     HTTPStatus,
@@ -330,6 +331,7 @@ class DbOrganisation(BaseModel):
     type: Optional[OrganisationType] = None
     approved: Optional[bool] = None
     created_by: Optional[int] = None  # this is not foreign key linked intentionally
+    associated_email: Optional[str] = None
     odk_central_url: Optional[str] = None
     odk_central_user: Optional[str] = None
     odk_central_password: Optional[str] = None
@@ -967,6 +969,9 @@ class DbProject(BaseModel):
     data_extract_url: Optional[str] = None
     task_split_dimension: Optional[int] = None
     task_num_buildings: Optional[int] = None
+    new_geom_type: Optional[DbGeomType] = None
+    geo_restrict_distance_meters: Optional[PositiveInt] = None
+    geo_restrict_force_error: Optional[bool] = None
     hashtags: Optional[list[str]] = None
     due_date: Optional[AwareDatetime] = None
     updated_at: Optional[AwareDatetime] = None

src/backend/app/organisations/organisation_crud.py

@@ -66,6 +66,7 @@ async def init_admin_org(db: Connection) -> None:
         name="HOTOSM",
         description="Humanitarian OpenStreetMap Team.",
         url="https://hotosm.org",
+        associated_email="[email protected]",
         odk_central_url=settings.ODK_CENTRAL_URL,
         odk_central_user=settings.ODK_CENTRAL_USER,
         odk_central_password=settings.ODK_CENTRAL_PASSWD.get_secret_value()

src/backend/app/organisations/organisation_schemas.py

@@ -107,4 +107,5 @@ class OrganisationOut(BaseModel):
     description: Optional[str]
     slug: Optional[str]
     url: Optional[str]
+    associated_email: Optional[str]
     odk_central_url: Optional[str]

src/backend/migrations/002-create-geometry-log.sql

@@ -4,13 +4,14 @@
 
 BEGIN;
 
-CREATE TYPE public.geomstatus AS ENUM (
-    'BAD',
-    'NEW'
-);
+DO $$ BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomstatus') THEN
+        CREATE TYPE public.geomstatus AS ENUM ('BAD', 'NEW');
+    END IF;
+END $$;
 ALTER TYPE public.geomstatus OWNER TO fmtm;
 
-CREATE TABLE public.geometrylog (
+CREATE TABLE IF NOT EXISTS public.geometrylog (
     id SERIAL PRIMARY KEY,
     geom GEOMETRY NOT NULL,
     status geomstatus,
@@ -20,7 +21,12 @@ CREATE TABLE public.geometrylog (
 ALTER TABLE public.geometrylog OWNER TO fmtm;
 
 -- Indexes for efficient querying
-CREATE INDEX idx_geometrylog ON geometrylog USING gist (geom);
+DO $$
+BEGIN
+    IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'geometrylog' AND column_name = 'geom') THEN
+        CREATE INDEX IF NOT EXISTS idx_geometrylog ON geometrylog USING gist (geom);
+    END IF;
+END $$;
 
 -- Commit the transaction
 COMMIT;

src/backend/migrations/003-jsonb-geom-geometry-log.sql

@@ -3,12 +3,25 @@
 -- Start a transaction
 
 BEGIN;
+
 -- drop existing indexes
 DROP INDEX IF EXISTS idx_geometrylog;
 
 -- Change the 'geom' column to jsonb type
-ALTER TABLE geometrylog
-ALTER COLUMN geom TYPE jsonb USING geom::jsonb;
+DO $$
+BEGIN
+    IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'geometrylog' AND column_name = 'geom') THEN
+        ALTER TABLE geometrylog
+        ALTER COLUMN geom TYPE jsonb USING geom::jsonb;
+    END IF;
+END $$;
+DO $$
+BEGIN
+    IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'geometrylog' AND column_name = 'geojson') THEN
+        ALTER TABLE geometrylog
+        ALTER COLUMN geojson TYPE JSONB USING geojson::jsonb;
+    END IF;
+END $$;
 
 -- Alter the 'id' column to UUID
 -- set the default value using gen_random_uuid()
@@ -21,7 +34,12 @@ ALTER TABLE geometrylog
 ALTER COLUMN id TYPE UUID USING gen_random_uuid(),
 ALTER COLUMN id SET DEFAULT gen_random_uuid();
 
-CREATE INDEX IF NOT EXISTS idx_geom_gin ON geometrylog USING gist (geom);
+DO $$
+BEGIN
+    IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'geometrylog' AND column_name = 'geom') THEN
+        CREATE INDEX IF NOT EXISTS idx_geom_gin ON geometrylog USING gist (geom);
+    END IF;
+END $$;
 
 -- Commit the transaction
 COMMIT;

src/backend/migrations/004-project-extra-fields.sql

@@ -0,0 +1,54 @@
+-- ## Migration add some extra fields.
+-- * Add geo_restrict_distance_meters and geo_restrict_force_error to projects.
+-- * Add new_geom_type to projects.
+-- * Add associated_email to organisations.
+
+-- Related issues:
+-- https://github.com/hotosm/fmtm/issues/1985#issuecomment-2577342507
+-- https://github.com/hotosm/fmtm/issues/1979
+-- https://github.com/hotosm/fmtm/issues/2066
+
+-- Start a transaction
+
+BEGIN;
+
+DO $$ BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomtype') THEN
+        CREATE TYPE public.geomtype AS ENUM ('POINT', 'POLYLINE', 'POLYGON');
+    END IF;
+END $$;
+ALTER TYPE public.geomstatus OWNER TO fmtm;
+
+ALTER TABLE public.organisations
+ADD COLUMN IF NOT EXISTS associated_email
+character varying;
+
+ALTER TABLE public.projects
+ADD COLUMN IF NOT EXISTS new_geom_type
+public.geomtype
+DEFAULT 'POINT',
+ADD COLUMN IF NOT EXISTS geo_restrict_distance_meters
+int2
+DEFAULT 50
+CHECK (geo_restrict_distance_meters >= 0),
+ADD COLUMN IF NOT EXISTS geo_restrict_force_error
+BOOLEAN
+DEFAULT false;
+
+-- Also update the fields from previous migration
+DROP INDEX IF EXISTS idx_geometrylog;
+DROP INDEX IF EXISTS idx_geom_gin;
+
+DO $$
+BEGIN
+    IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'geometrylog' AND column_name = 'geom') THEN
+        ALTER TABLE public.geometrylog RENAME COLUMN geom TO geojson;
+        ALTER TABLE public.geometrylog ALTER COLUMN geojson TYPE JSONB USING geojson::jsonb;
+    END IF;
+END $$;
+
+CREATE INDEX IF NOT EXISTS idx_geometrylog_geojson
+ON geometrylog USING gin (geojson);
+
+-- Commit the transaction
+COMMIT;

src/backend/migrations/init/fmtm_base_schema.sql

@@ -159,6 +159,13 @@ CREATE TYPE public.geomstatus AS ENUM (
 );
 ALTER TYPE public.geomstatus OWNER TO fmtm;
 
+CREATE TYPE public.geomtype AS ENUM (
+    'POINT',
+    'POLYLINE',
+    'POLYGON'
+);
+ALTER TYPE public.geomtype OWNER TO fmtm;
+
 -- Extra
 
 SET default_tablespace = '';
@@ -217,6 +224,7 @@ CREATE TABLE public.organisations (
     type public.organisationtype DEFAULT 'FREE',
     community_type public.communitytype DEFAULT 'OSM_COMMUNITY',
     created_by integer,
+    associated_email character varying,
     approved BOOLEAN DEFAULT false,
     odk_central_url character varying,
     odk_central_user character varying,
@@ -269,6 +277,11 @@ CREATE TABLE public.projects (
     task_num_buildings smallint,
     hashtags character varying [],
     custom_tms_url character varying,
+    geo_restrict_force_error boolean DEFAULT false,
+    geo_restrict_distance_meters int2 DEFAULT 50 CHECK (
+        geo_restrict_distance_meters >= 0
+    ),
+    new_geom_type public.geomtype DEFAULT 'POINT',
     created_at timestamp with time zone NOT NULL DEFAULT now(),
     updated_at timestamp with time zone DEFAULT now()
 );
@@ -392,8 +405,8 @@ ALTER SEQUENCE public.submission_photos_id_seq
 OWNED BY public.submission_photos.id;
 
 CREATE TABLE geometrylog (
-    id uuid DEFAULT gen_random_uuid() PRIMARY KEY,
-    geom JSONB NOT NULL,
+    id UUID NOT NULL DEFAULT gen_random_uuid(),
+    geojson JSONB NOT NULL,
     status geomstatus,
     project_id int,
     task_id int
@@ -521,8 +534,9 @@ CREATE INDEX idx_entities_task_id
 ON public.odk_entities USING btree (
     entity_id, task_id
 );
-CREATE INDEX idx_geometrylog
-ON geometrylog USING gist (geom);
+CREATE INDEX idx_geometrylog_geojson
+ON geometrylog USING gin (geom);
+
 
 -- Foreign keys
 

src/backend/migrations/revert/004-project-extra-fields.sql

@@ -0,0 +1,17 @@
+-- Start a transaction
+
+BEGIN;
+
+ALTER TABLE public.organisations
+DROP COLUMN IF EXISTS associated_email;
+
+ALTER TABLE public.projects
+DROP CONSTRAINT IF EXISTS projects_geo_restrict_distance_meters_check,
+DROP COLUMN IF EXISTS new_geom_type,
+DROP COLUMN IF EXISTS geo_restrict_distance_meters,
+DROP COLUMN IF EXISTS geo_restrict_force_error;
+
+DROP TYPE IF EXISTS public.geomtype;
+
+-- Commit the transaction
+COMMIT;