diff --git a/API/raw_data.py b/API/raw_data.py index 78752b54..eaa8f3cf 100644 --- a/API/raw_data.py +++ b/API/raw_data.py @@ -21,12 +21,14 @@ """ # Standard library imports import json +from typing import AsyncGenerator # Third party imports +import orjson import redis from area import area from fastapi import APIRouter, Body, Depends, HTTPException, Request -from fastapi.responses import JSONResponse +from fastapi.responses import JSONResponse, StreamingResponse from fastapi_versioning import version # Reader imports @@ -39,6 +41,7 @@ ) from src.config import LIMITER as limiter from src.config import RATE_LIMIT_PER_MIN as export_rate_limit +from src.query_builder.builder import raw_currentdata_extraction_query from src.validation.models import ( RawDataCurrentParams, RawDataCurrentParamsBase, @@ -448,6 +451,15 @@ def get_osm_current_snapshot_as_file( ], ) + if user.id == 0 and params.include_user_metadata: + raise HTTPException( + status_code=403, + detail=[ + { + "msg": "Insufficient Permission for extracting exports with user metadata , Please login first" + } + ], + ) queue_name = DEFAULT_QUEUE_NAME # Everything directs to default now task = process_raw_data.apply_async( args=(params.model_dump(),), @@ -466,7 +478,7 @@ def get_osm_current_snapshot_as_file( @router.post("/snapshot/plain/") @version(1) -def get_osm_current_snapshot_as_plain_geojson( +async def get_osm_current_snapshot_as_plain_geojson( request: Request, params: RawDataCurrentParamsBase, user: AuthUser = Depends(get_optional_user), @@ -475,25 +487,61 @@ def get_osm_current_snapshot_as_plain_geojson( Args: request (Request): _description_ - params (RawDataCurrentParamsBase): Same as /snapshot excpet multiple output format options and configurations + params (RawDataCurrentParamsBase): Same as /snapshot except multiple output format options and configurations Returns: - Featurecollection: Geojson + FeatureCollection: Geojson """ + if user.id == 0 and params.include_user_metadata: + raise HTTPException( + status_code=403, + detail=[ + { + "msg": "Insufficient Permission for extracting exports with user metadata, Please login first" + } + ], + ) area_m2 = area(json.loads(params.geometry.model_dump_json())) + area_km2 = area_m2 * 1e-6 - if area_km2 > 5: + if int(area_km2) > 6: raise HTTPException( status_code=400, detail=[ { - "msg": f"""Polygon Area {int(area_km2)} Sq.KM is higher than Threshold : 10 Sq.KM""" + "msg": f"""Polygon Area {int(area_km2)} Sq.KM is higher than Threshold : 6 Sq.KM""" } ], ) + params.output_type = "geojson" # always geojson - result = RawData(params).extract_plain_geojson() - return result + + async def generate_geojson() -> AsyncGenerator[bytes, None]: + # start of featurecollection + yield b'{"type": "FeatureCollection", "features": [' + + raw_data = RawData(params) + extraction_query = raw_currentdata_extraction_query(params) + + with raw_data.con.cursor(name="fetch_raw_quick") as cursor: + cursor.itersize = 500 + cursor.execute(extraction_query) + + first_feature = True + for row in cursor: + feature = orjson.loads(row[0]) + if not first_feature: + # add comma to maintain the struct + yield b"," + else: + first_feature = False + yield orjson.dumps(feature) + cursor.close() + + # end of featurecollect + yield b"]}" + + return StreamingResponse(generate_geojson(), media_type="application/geo+json") @router.get("/countries/") diff --git a/src/app.py b/src/app.py index 2788efb2..ece19b49 100644 --- a/src/app.py +++ b/src/app.py @@ -880,21 +880,6 @@ def get_osm_feature(self, osm_id): self.cur.close() return FeatureCollection(features=features) - def extract_plain_geojson(self): - """Gets geojson for small area Returns plain geojson without binding""" - extraction_query = raw_currentdata_extraction_query(self.params) - features = [] - - with self.con.cursor( - name="fetch_raw_quick" - ) as cursor: # using server side cursor - cursor.itersize = 500 - cursor.execute(extraction_query) - for row in cursor: - features.append(orjson.loads(row[0])) - cursor.close() - return FeatureCollection(features=features) - class S3FileTransfer: """Responsible for the file transfer to s3 from API maachine""" @@ -1913,7 +1898,7 @@ def upload_dataset(self, dump_config_to_s3=False): dataset_info["hdx_upload"] = "SUCCESS" except Exception as ex: logging.error(ex) - raise ex + # raise ex dataset_info["hdx_upload"] = "FAILED" dataset_info["name"] = self.dataset["name"] diff --git a/src/query_builder/builder.py b/src/query_builder/builder.py index 90b85eff..e4db165c 100644 --- a/src/query_builder/builder.py +++ b/src/query_builder/builder.py @@ -502,7 +502,6 @@ def raw_currentdata_extraction_query( g_id=None, c_id=None, ogr_export=False, - select_all=False, country_export=False, ): """Default function to support current snapshot extraction with all of the feature that export_tool_api has""" @@ -536,8 +535,8 @@ def raw_currentdata_extraction_query( use_geomtype_in_relation = True # query_table = [] - if select_all: - select_condition = f"""osm_id, tableoid::regclass AS osm_type, version,tags,changeset,timestamp,{'ST_Centroid(geom) as geom' if params.centroid else 'geom'}""" # FIXme have condition for displaying userinfo after user authentication + if params.include_user_metadata: + select_condition = f"""osm_id, tableoid::regclass AS osm_type, version,tags,changeset, uid, user, timestamp,{'ST_Centroid(geom) as geom' if params.centroid else 'geom'}""" else: select_condition = f"""osm_id, tableoid::regclass AS osm_type, version,tags,changeset,timestamp,{'ST_Centroid(geom) as geom' if params.centroid else 'geom'}""" # this is default attribute that we will deliver to user if user defines his own attribute column then those will be appended with osm_id only diff --git a/src/validation/models.py b/src/validation/models.py index 925a3713..4b757447 100644 --- a/src/validation/models.py +++ b/src/validation/models.py @@ -158,6 +158,10 @@ class RawDataCurrentParamsBase(BaseModel, GeometryValidatorMixin): default=True, description="Exports features which are exactly inside the passed polygons (ST_WITHIN) By default features which are intersected with passed polygon is exported", ) + include_user_metadata: Optional[bool] = Field( + default=False, + description="Include user metadata on exports , Only available to logged in users", + ) if ENABLE_POLYGON_STATISTICS_ENDPOINTS: include_stats: Optional[bool] = Field( default=False, @@ -216,6 +220,7 @@ class RawDataCurrentParams(RawDataCurrentParamsBase): default=False, description="Wraps all flatgeobuff output to geometrycollection geometry type", ) + if ALLOW_BIND_ZIP_FILTER: bind_zip: Optional[bool] = True @@ -298,22 +303,22 @@ class StatsRequestParams(BaseModel, GeometryValidatorMixin): max_length=3, example="NPL", ) - geometry: Optional[ - Union[Polygon, MultiPolygon, Feature, FeatureCollection] - ] = Field( - default=None, - example={ - "type": "Polygon", - "coordinates": [ - [ - [83.96919250488281, 28.194446860487773], - [83.99751663208006, 28.194446860487773], - [83.99751663208006, 28.214869548073377], - [83.96919250488281, 28.214869548073377], - [83.96919250488281, 28.194446860487773], - ] - ], - }, + geometry: Optional[Union[Polygon, MultiPolygon, Feature, FeatureCollection]] = ( + Field( + default=None, + example={ + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + }, + ) ) @validator("geometry", pre=True, always=True) @@ -619,22 +624,22 @@ class DynamicCategoriesModel(CategoriesBase, GeometryValidatorMixin): max_length=3, example="USA", ) - geometry: Optional[ - Union[Polygon, MultiPolygon, Feature, FeatureCollection] - ] = Field( - default=None, - example={ - "type": "Polygon", - "coordinates": [ - [ - [83.96919250488281, 28.194446860487773], - [83.99751663208006, 28.194446860487773], - [83.99751663208006, 28.214869548073377], - [83.96919250488281, 28.214869548073377], - [83.96919250488281, 28.194446860487773], - ] - ], - }, + geometry: Optional[Union[Polygon, MultiPolygon, Feature, FeatureCollection]] = ( + Field( + default=None, + example={ + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + }, + ) ) @validator("geometry", pre=True, always=True) diff --git a/tests/test_API.py b/tests/test_API.py index 750269e5..2475bca2 100644 --- a/tests/test_API.py +++ b/tests/test_API.py @@ -718,6 +718,84 @@ def test_snapshot_bind_zip(): wait_for_task_completion(track_link) +def test_snapshot_bind_zip(): + headers = {"access-token": access_token} + payload = { + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + }, + "bindZip": False, + } + + response = client.post("/v1/snapshot/", json=payload, headers=headers) + + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +## Test snapshot include user metadata + + +def test_snapshot_with_user_meatadata(): + headers = {"access-token": access_token} + payload = { + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + }, + "includeUserMetadata": True, + } + + response = client.post("/v1/snapshot/", json=payload, headers=headers) + + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_snapshot_with_user_meatadata_without_login(): + # headers = {"access-token": access_token} + payload = { + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + }, + "includeUserMetadata": True, + } + + response = client.post("/v1/snapshot/", json=payload) + + assert response.status_code == 403 + + ## Snapshot Plain