Merge pull request #261 from hotosm/feature/allow-user-info-on-exports

Feature : Let user export mapper information on exports

API/raw_data.py

@@ -21,12 +21,14 @@
 """
 # Standard library imports
 import json
+from typing import AsyncGenerator
 
 # Third party imports
+import orjson
 import redis
 from area import area
 from fastapi import APIRouter, Body, Depends, HTTPException, Request
-from fastapi.responses import JSONResponse
+from fastapi.responses import JSONResponse, StreamingResponse
 from fastapi_versioning import version
 
 # Reader imports
@@ -39,6 +41,7 @@
 )
 from src.config import LIMITER as limiter
 from src.config import RATE_LIMIT_PER_MIN as export_rate_limit
+from src.query_builder.builder import raw_currentdata_extraction_query
 from src.validation.models import (
     RawDataCurrentParams,
     RawDataCurrentParamsBase,
@@ -448,6 +451,15 @@ def get_osm_current_snapshot_as_file(
                         ],
                     )
 
+    if user.id == 0 and params.include_user_metadata:
+        raise HTTPException(
+            status_code=403,
+            detail=[
+                {
+                    "msg": "Insufficient Permission for extracting exports with user metadata , Please login first"
+                }
+            ],
+        )
     queue_name = DEFAULT_QUEUE_NAME  # Everything directs to default now
     task = process_raw_data.apply_async(
         args=(params.model_dump(),),
@@ -466,7 +478,7 @@ def get_osm_current_snapshot_as_file(
 
 @router.post("/snapshot/plain/")
 @version(1)
-def get_osm_current_snapshot_as_plain_geojson(
+async def get_osm_current_snapshot_as_plain_geojson(
     request: Request,
     params: RawDataCurrentParamsBase,
     user: AuthUser = Depends(get_optional_user),
@@ -475,25 +487,61 @@ def get_osm_current_snapshot_as_plain_geojson(
 
     Args:
         request (Request): _description_
-        params (RawDataCurrentParamsBase): Same as /snapshot excpet multiple output format options and configurations
+        params (RawDataCurrentParamsBase): Same as /snapshot except multiple output format options and configurations
 
     Returns:
-        Featurecollection: Geojson
+        FeatureCollection: Geojson
     """
+    if user.id == 0 and params.include_user_metadata:
+        raise HTTPException(
+            status_code=403,
+            detail=[
+                {
+                    "msg": "Insufficient Permission for extracting exports with user metadata, Please login first"
+                }
+            ],
+        )
     area_m2 = area(json.loads(params.geometry.model_dump_json()))
+
     area_km2 = area_m2 * 1e-6
-    if area_km2 > 5:
+    if int(area_km2) > 6:
         raise HTTPException(
             status_code=400,
             detail=[
                 {
-                    "msg": f"""Polygon Area {int(area_km2)} Sq.KM is higher than Threshold : 10 Sq.KM"""
+                    "msg": f"""Polygon Area {int(area_km2)} Sq.KM is higher than Threshold : 6 Sq.KM"""
                 }
             ],
         )
+
     params.output_type = "geojson"  # always geojson
-    result = RawData(params).extract_plain_geojson()
-    return result
+
+    async def generate_geojson() -> AsyncGenerator[bytes, None]:
+        # start of featurecollection
+        yield b'{"type": "FeatureCollection", "features": ['
+
+        raw_data = RawData(params)
+        extraction_query = raw_currentdata_extraction_query(params)
+
+        with raw_data.con.cursor(name="fetch_raw_quick") as cursor:
+            cursor.itersize = 500
+            cursor.execute(extraction_query)
+
+            first_feature = True
+            for row in cursor:
+                feature = orjson.loads(row[0])
+                if not first_feature:
+                    # add comma to maintain the struct
+                    yield b","
+                else:
+                    first_feature = False
+                yield orjson.dumps(feature)
+            cursor.close()
+
+        # end of featurecollect
+        yield b"]}"
+
+    return StreamingResponse(generate_geojson(), media_type="application/geo+json")
 
 
 @router.get("/countries/")

src/app.py

@@ -880,21 +880,6 @@ def get_osm_feature(self, osm_id):
         self.cur.close()
         return FeatureCollection(features=features)
 
-    def extract_plain_geojson(self):
-        """Gets geojson for small area Returns plain geojson without binding"""
-        extraction_query = raw_currentdata_extraction_query(self.params)
-        features = []
-
-        with self.con.cursor(
-            name="fetch_raw_quick"
-        ) as cursor:  # using server side cursor
-            cursor.itersize = 500
-            cursor.execute(extraction_query)
-            for row in cursor:
-                features.append(orjson.loads(row[0]))
-            cursor.close()
-        return FeatureCollection(features=features)
-
 
 class S3FileTransfer:
     """Responsible for the file transfer to s3 from API maachine"""
@@ -1913,7 +1898,7 @@ def upload_dataset(self, dump_config_to_s3=False):
                 dataset_info["hdx_upload"] = "SUCCESS"
             except Exception as ex:
                 logging.error(ex)
-                raise ex
+                # raise ex
                 dataset_info["hdx_upload"] = "FAILED"
 
             dataset_info["name"] = self.dataset["name"]

src/query_builder/builder.py

@@ -502,7 +502,6 @@ def raw_currentdata_extraction_query(
     g_id=None,
     c_id=None,
     ogr_export=False,
-    select_all=False,
     country_export=False,
 ):
     """Default function to support current snapshot extraction with all of the feature that export_tool_api has"""
@@ -536,8 +535,8 @@ def raw_currentdata_extraction_query(
     use_geomtype_in_relation = True
 
     # query_table = []
-    if select_all:
-        select_condition = f"""osm_id, tableoid::regclass AS osm_type, version,tags,changeset,timestamp,{'ST_Centroid(geom) as geom' if params.centroid else 'geom'}"""  # FIXme have condition for displaying userinfo after user authentication
+    if params.include_user_metadata:
+        select_condition = f"""osm_id, tableoid::regclass AS osm_type, version,tags,changeset, uid, user, timestamp,{'ST_Centroid(geom) as geom' if params.centroid else 'geom'}"""
     else:
         select_condition = f"""osm_id, tableoid::regclass AS osm_type, version,tags,changeset,timestamp,{'ST_Centroid(geom) as geom' if params.centroid else 'geom'}"""  # this is default attribute that we will deliver to user if user defines his own attribute column then those will be appended with osm_id only
 

src/validation/models.py

@@ -158,6 +158,10 @@ class RawDataCurrentParamsBase(BaseModel, GeometryValidatorMixin):
         default=True,
         description="Exports features which are exactly inside the passed polygons (ST_WITHIN) By default features which are intersected with passed polygon is exported",
     )
+    include_user_metadata: Optional[bool] = Field(
+        default=False,
+        description="Include user metadata on exports , Only available to logged in users",
+    )
     if ENABLE_POLYGON_STATISTICS_ENDPOINTS:
         include_stats: Optional[bool] = Field(
             default=False,
@@ -216,6 +220,7 @@ class RawDataCurrentParams(RawDataCurrentParamsBase):
         default=False,
         description="Wraps all flatgeobuff output to geometrycollection geometry type",
     )
+
     if ALLOW_BIND_ZIP_FILTER:
         bind_zip: Optional[bool] = True
 
@@ -298,22 +303,22 @@ class StatsRequestParams(BaseModel, GeometryValidatorMixin):
         max_length=3,
         example="NPL",
     )
-    geometry: Optional[
-        Union[Polygon, MultiPolygon, Feature, FeatureCollection]
-    ] = Field(
-        default=None,
-        example={
-            "type": "Polygon",
-            "coordinates": [
-                [
-                    [83.96919250488281, 28.194446860487773],
-                    [83.99751663208006, 28.194446860487773],
-                    [83.99751663208006, 28.214869548073377],
-                    [83.96919250488281, 28.214869548073377],
-                    [83.96919250488281, 28.194446860487773],
-                ]
-            ],
-        },
+    geometry: Optional[Union[Polygon, MultiPolygon, Feature, FeatureCollection]] = (
+        Field(
+            default=None,
+            example={
+                "type": "Polygon",
+                "coordinates": [
+                    [
+                        [83.96919250488281, 28.194446860487773],
+                        [83.99751663208006, 28.194446860487773],
+                        [83.99751663208006, 28.214869548073377],
+                        [83.96919250488281, 28.214869548073377],
+                        [83.96919250488281, 28.194446860487773],
+                    ]
+                ],
+            },
+        )
     )
 
     @validator("geometry", pre=True, always=True)
@@ -619,22 +624,22 @@ class DynamicCategoriesModel(CategoriesBase, GeometryValidatorMixin):
         max_length=3,
         example="USA",
     )
-    geometry: Optional[
-        Union[Polygon, MultiPolygon, Feature, FeatureCollection]
-    ] = Field(
-        default=None,
-        example={
-            "type": "Polygon",
-            "coordinates": [
-                [
-                    [83.96919250488281, 28.194446860487773],
-                    [83.99751663208006, 28.194446860487773],
-                    [83.99751663208006, 28.214869548073377],
-                    [83.96919250488281, 28.214869548073377],
-                    [83.96919250488281, 28.194446860487773],
-                ]
-            ],
-        },
+    geometry: Optional[Union[Polygon, MultiPolygon, Feature, FeatureCollection]] = (
+        Field(
+            default=None,
+            example={
+                "type": "Polygon",
+                "coordinates": [
+                    [
+                        [83.96919250488281, 28.194446860487773],
+                        [83.99751663208006, 28.194446860487773],
+                        [83.99751663208006, 28.214869548073377],
+                        [83.96919250488281, 28.214869548073377],
+                        [83.96919250488281, 28.194446860487773],
+                    ]
+                ],
+            },
+        )
     )
 
     @validator("geometry", pre=True, always=True)

tests/test_API.py

@@ -718,6 +718,84 @@ def test_snapshot_bind_zip():
     wait_for_task_completion(track_link)
 
 
+def test_snapshot_bind_zip():
+    headers = {"access-token": access_token}
+    payload = {
+        "geometry": {
+            "type": "Polygon",
+            "coordinates": [
+                [
+                    [83.96919250488281, 28.194446860487773],
+                    [83.99751663208006, 28.194446860487773],
+                    [83.99751663208006, 28.214869548073377],
+                    [83.96919250488281, 28.214869548073377],
+                    [83.96919250488281, 28.194446860487773],
+                ]
+            ],
+        },
+        "bindZip": False,
+    }
+
+    response = client.post("/v1/snapshot/", json=payload, headers=headers)
+
+    assert response.status_code == 200
+    res = response.json()
+    track_link = res["track_link"]
+    wait_for_task_completion(track_link)
+
+
+## Test snapshot include user metadata
+
+
+def test_snapshot_with_user_meatadata():
+    headers = {"access-token": access_token}
+    payload = {
+        "geometry": {
+            "type": "Polygon",
+            "coordinates": [
+                [
+                    [83.96919250488281, 28.194446860487773],
+                    [83.99751663208006, 28.194446860487773],
+                    [83.99751663208006, 28.214869548073377],
+                    [83.96919250488281, 28.214869548073377],
+                    [83.96919250488281, 28.194446860487773],
+                ]
+            ],
+        },
+        "includeUserMetadata": True,
+    }
+
+    response = client.post("/v1/snapshot/", json=payload, headers=headers)
+
+    assert response.status_code == 200
+    res = response.json()
+    track_link = res["track_link"]
+    wait_for_task_completion(track_link)
+
+
+def test_snapshot_with_user_meatadata_without_login():
+    # headers = {"access-token": access_token}
+    payload = {
+        "geometry": {
+            "type": "Polygon",
+            "coordinates": [
+                [
+                    [83.96919250488281, 28.194446860487773],
+                    [83.99751663208006, 28.194446860487773],
+                    [83.99751663208006, 28.214869548073377],
+                    [83.96919250488281, 28.214869548073377],
+                    [83.96919250488281, 28.194446860487773],
+                ]
+            ],
+        },
+        "includeUserMetadata": True,
+    }
+
+    response = client.post("/v1/snapshot/", json=payload)
+
+    assert response.status_code == 403
+
+
 ## Snapshot Plain