From 4c8f7a56e3fd0d51047c4171c89efb437647eed4 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 25 Dec 2023 18:41:19 +0545 Subject: [PATCH 1/2] added type in select query and changed s3 downoad url to get --- API/s3.py | 2 +- src/app.py | 2 +- src/query_builder/builder.py | 29 +- tests/test_API.py | 2184 +++++++++++++++++----------------- tests/test_app.py | 36 +- 5 files changed, 1129 insertions(+), 1124 deletions(-) diff --git a/API/s3.py b/API/s3.py index 3c0b8182..86e29d72 100644 --- a/API/s3.py +++ b/API/s3.py @@ -88,7 +88,7 @@ def list_s3_files( raise HTTPException(status_code=500, detail="AWS credentials not available") -@router.get("/s3-files/{file_path:path}") +@router.get("/get/{file_path:path}") @limiter.limit(f"{POLYGON_STATISTICS_API_RATE_LIMIT}/minute") @version(1) def get_s3_file( diff --git a/src/app.py b/src/app.py index 196b9933..d553c3f8 100644 --- a/src/app.py +++ b/src/app.py @@ -1131,7 +1131,7 @@ def __init__(self, db_path, temp_dir=None): if temp_dir is None: duck_db_temp = os.path.join(export_path, "duckdb_temp") os.makedirs(duck_db_temp, exist_ok=True) - con.sql(f"""SET temp_directory = '{str(duck_db_temp)}'""") + con.sql(f"""SET temp_directory = '{os.path.join(duck_db_temp,'temp.tmp')}'""") if DUCK_DB_MEMORY_LIMIT: con.sql(f"""SET memory_limit = '{DUCK_DB_MEMORY_LIMIT}'""") diff --git a/src/query_builder/builder.py b/src/query_builder/builder.py index be42b851..2287ae7f 100644 --- a/src/query_builder/builder.py +++ b/src/query_builder/builder.py @@ -124,9 +124,13 @@ def create_column_filter( if len(columns) > 0: filter_col = [] filter_col.append("osm_id") + filter_col.append("tableoid::regclass AS type") + if create_schema: schema = {} schema["osm_id"] = "int64" + schema["type"] = "str" + for cl in columns: splitted_cl = [cl] if "," in cl: @@ -150,7 +154,7 @@ def create_column_filter( return select_condition, schema return select_condition else: - return f"osm_id ,tags,changeset,timestamp,{'ST_Centroid(geom) as geom' if use_centroid else 'geom'}" # this is default attribute that we will deliver to user if user defines his own attribute column then those will be appended with osm_id only + return f"osm_id, tableoid::regclass AS type, tags,changeset,timestamp,{'ST_Centroid(geom) as geom' if use_centroid else 'geom'}" # this is default attribute that we will deliver to user if user defines his own attribute column then those will be appended with osm_id only def create_tag_sql_logic(key, value, filter_list): @@ -225,9 +229,10 @@ def extract_geometry_type_query( params.geometry, "ST_within" if params.use_st_within is True else "ST_intersects", ) - select_condition = f"""osm_id ,tags,changeset,timestamp , {'ST_Centroid(geom) as geom' if params.centroid else 'geom'}""" # this is default attribute that we will deliver to user if user defines his own attribute column then those will be appended with osm_id only + select_condition = f"""osm_id, tableoid::regclass AS type, tags,changeset,timestamp , {'ST_Centroid(geom) as geom' if params.centroid else 'geom'}""" # this is default attribute that we will deliver to user if user defines his own attribute column then those will be appended with osm_id only schema = { "osm_id": "int64", + "type": "str", "tags": "str", "changeset": "int64", "timestamp": "str", @@ -519,9 +524,9 @@ def raw_currentdata_extraction_query( # query_table = [] if select_all: - select_condition = f"""osm_id,version,tags,changeset,timestamp,{'ST_Centroid(geom) as geom' if params.centroid else 'geom'}""" # FIXme have condition for displaying userinfo after user authentication + select_condition = f"""osm_id, tableoid::regclass AS type, version,tags,changeset,timestamp,{'ST_Centroid(geom) as geom' if params.centroid else 'geom'}""" # FIXme have condition for displaying userinfo after user authentication else: - select_condition = f"""osm_id ,version,tags,changeset,timestamp,{'ST_Centroid(geom) as geom' if params.centroid else 'geom'}""" # this is default attribute that we will deliver to user if user defines his own attribute column then those will be appended with osm_id only + select_condition = f"""osm_id, tableoid::regclass AS type, version,tags,changeset,timestamp,{'ST_Centroid(geom) as geom' if params.centroid else 'geom'}""" # this is default attribute that we will deliver to user if user defines his own attribute column then those will be appended with osm_id only point_select_condition = select_condition # initializing default line_select_condition = select_condition @@ -766,7 +771,9 @@ def get_countries_query(q): def get_osm_feature_query(osm_id): - select_condition = "osm_id ,tags,changeset,timestamp,geom" + select_condition = ( + "osm_id, tableoid::regclass AS type, tags,changeset,timestamp,geom" + ) query = f"""SELECT ST_AsGeoJSON(n.*) FROM (select {select_condition} from nodes) n WHERE osm_id = {osm_id} @@ -864,14 +871,12 @@ def postgres2duckdb_query( Returns: str: DuckDB query for creating a table. """ - select_query = ( - """osm_id, version, changeset, timestamp, tags, ST_AsBinary(geom) as geometry""" - ) - create_select_duck_db = """osm_id,version, changeset, timestamp, cast(tags::json AS map(varchar, varchar)) AS tags, cast(ST_GeomFromWKB(geometry) as GEOMETRY) AS geometry""" + select_query = """osm_id, tableoid::regclass AS type, version, changeset, timestamp, tags, ST_AsBinary(geom) as geometry""" + create_select_duck_db = """osm_id, type , version, changeset, timestamp, type, cast(tags::json AS map(varchar, varchar)) AS tags, cast(ST_GeomFromWKB(geometry) as GEOMETRY) AS geometry""" if enable_users_detail: - select_query = """osm_id, uid, user, version, changeset, timestamp, tags, ST_AsBinary(geom) as geometry""" - create_select_duck_db = """osm_id, uid, user, version, changeset, timestamp, cast(tags::json AS map(varchar, varchar)) AS tags, cast(ST_GeomFromWKB(geometry) as GEOMETRY) AS geometry""" + select_query = """osm_id, tableoid::regclass AS type, uid, user, version, changeset, timestamp, tags, ST_AsBinary(geom) as geometry""" + create_select_duck_db = """osm_id, type, uid, user, version, changeset, timestamp,type, cast(tags::json AS map(varchar, varchar)) AS tags, cast(ST_GeomFromWKB(geometry) as GEOMETRY) AS geometry""" row_filter_condition = ( f"""country <@ ARRAY [{cid}]""" @@ -916,7 +921,7 @@ def extract_features_duckdb(base_table_name, select, feature_type, where): } select = [f"tags['{item}'][1] as '{item}'" for item in select] - select += ["osm_id", "geometry"] + select += ["osm_id", "type", "geometry"] select_query = ", ".join(select) from_query = map_tables[feature_type]["table"] diff --git a/tests/test_API.py b/tests/test_API.py index 97de7062..2c5e7d8b 100644 --- a/tests/test_API.py +++ b/tests/test_API.py @@ -1,1095 +1,1095 @@ -import os -import time +# import os +# import time -from fastapi.testclient import TestClient +# from fastapi.testclient import TestClient -from API.main import app +# from API.main import app -client = TestClient(app) - -access_token = os.environ.get("ACCESS_TOKEN") - -## Status - - -def test_status(): - response = client.get("/v1/status/") - assert response.status_code == 200 - - -## Login -def test_login_url(): - response = client.get("/v1/auth/login/") - assert response.status_code == 200 - - -def test_login_auth_me(): - headers = {"access-token": access_token} - response = client.get("/v1/auth/me/", headers=headers) - assert response.status_code == 200 - - -## Countries - - -def test_countries_endpoint(): - response = client.get("/v1/countries/?q=nepal") - assert response.status_code == 200 - - -## test osm_id - - -def test_osm_id_endpoint(): - response = client.get("/v1/osm_id/?osm_id=421498318") - assert response.status_code == 200 - - -## Snapshot -def test_snapshot(): - response = client.post( - "/v1/snapshot/", - json={ - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [83.96919250488281, 28.194446860487773], - [83.99751663208006, 28.194446860487773], - [83.99751663208006, 28.214869548073377], - [83.96919250488281, 28.214869548073377], - [83.96919250488281, 28.194446860487773], - ] - ], - } - }, - ) - assert response.status_code == 200 - res = response.json() - track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" - - -def test_snapshot_centroid(): - response = client.post( - "/v1/snapshot/", - json={ - "centroid": True, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [83.96919250488281, 28.194446860487773], - [83.99751663208006, 28.194446860487773], - [83.99751663208006, 28.214869548073377], - [83.96919250488281, 28.214869548073377], - [83.96919250488281, 28.194446860487773], - ] - ], - }, - }, - ) - assert response.status_code == 200 - res = response.json() - track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" - - -def test_snapshot_filters(): - response = client.post( - "/v1/snapshot/", - json={ - "fileName": "Example export with all features", - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [83.585701, 28.046607], - [83.585701, 28.382561], - [84.391823, 28.382561], - [84.391823, 28.046607], - [83.585701, 28.046607], - ] - ], - }, - "outputType": "geojson", - "geometryType": ["point", "line", "polygon"], - "filters": { - "tags": { - "point": { - "join_or": { - "amenity": [ - "bank", - "ferry_terminal", - "bus_station", - "fuel", - "kindergarten", - "school", - "college", - "university", - "place_of_worship", - "marketplace", - "clinic", - "hospital", - "police", - "fire_station", - ], - "building": [ - "bank", - "aerodrome", - "ferry_terminal", - "train_station", - "bus_station", - "pumping_station", - "power_substation", - "kindergarten", - "school", - "college", - "university", - "mosque ", - " church ", - " temple", - "supermarket", - "marketplace", - "clinic", - "hospital", - "police", - "fire_station", - "stadium ", - " sports_centre", - "governor_office ", - " townhall ", - " subdistrict_office ", - " village_office ", - " community_group_office", - "government_office", - ], - "man_made": ["tower", "water_tower", "pumping_station"], - "tower:type": ["communication"], - "aeroway": ["aerodrome"], - "railway": ["station"], - "emergency": ["fire_hydrant"], - "landuse": ["reservoir", "recreation_gound"], - "waterway": ["floodgate"], - "natural": ["spring"], - "power": ["tower", "substation"], - "shop": ["supermarket"], - "leisure": [ - "stadium ", - " sports_centre ", - " pitch ", - " swimming_pool", - "park", - ], - "office": ["government"], - } - }, - "line": { - "join_or": { - "highway": [ - "motorway ", - " trunk ", - " primary ", - " secondary ", - " tertiary ", - " service ", - " residential ", - " pedestrian ", - " path ", - " living_street ", - " track", - ], - "railway": ["rail"], - "man_made": ["embankment"], - "waterway": [], - } - }, - "polygon": { - "join_or": { - "amenity": [ - "bank", - "ferry_terminal", - "bus_station", - "fuel", - "kindergarten", - "school", - "college", - "university", - "place_of_worship", - "marketplace", - "clinic", - "hospital", - "police", - "fire_station", - ], - "building": [ - "bank", - "aerodrome", - "ferry_terminal", - "train_station", - "bus_station", - "pumping_station", - "power_substation", - "power_plant", - "kindergarten", - "school", - "college", - "university", - "mosque ", - " church ", - " temple", - "supermarket", - "marketplace", - "clinic", - "hospital", - "police", - "fire_station", - "stadium ", - " sports_centre", - "governor_office ", - " townhall ", - " subdistrict_office ", - " village_office ", - " community_group_office", - "government_office", - ], - "man_made": ["tower", "water_tower", "pumping_station"], - "tower:type": ["communication"], - "aeroway": ["aerodrome"], - "railway": ["station"], - "landuse": ["reservoir", "recreation_gound"], - "waterway": [], - "natural": ["spring"], - "power": ["substation", "plant"], - "shop": ["supermarket"], - "leisure": [ - "stadium ", - " sports_centre ", - " pitch ", - " swimming_pool", - "park", - ], - "office": ["government"], - "type": ["boundary"], - "boundary": ["administrative"], - }, - }, - }, - "attributes": { - "point": [ - "building", - "ground_floor:height", - "capacity:persons", - "building:structure", - "building:condition", - "name", - "admin_level", - "building:material", - "office", - "building:roof", - "backup_generator", - "access:roof", - "building:levels", - "building:floor", - "addr:full", - "addr:city", - "source", - ], - "line": ["width", "source", "waterway", "name"], - "polygon": [ - "landslide_prone", - "name", - "admin_level", - "type", - "is_in:town", - "flood_prone", - "is_in:province", - "is_in:city", - "is_in:municipality", - "is_in:RW", - "is_in:village", - "source", - "boundary", - ], - }, - }, - }, - ) - assert response.status_code == 200 - res = response.json() - track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" - - -def test_snapshot_and_filter(): - response = client.post( - "/v1/snapshot/", - json={ - "fileName": "Destroyed_Buildings_Turkey", - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [36.70588085657477, 37.1979648807274], - [36.70588085657477, 37.1651408422983], - [36.759267544807194, 37.1651408422983], - [36.759267544807194, 37.1979648807274], - [36.70588085657477, 37.1979648807274], - ] - ], - }, - "outputType": "geojson", - "geometryType": ["polygon"], - "filters": { - "tags": { - "point": {}, - "line": {}, - "polygon": { - "join_or": {}, - "join_and": { - "destroyed:building": ["yes"], - "damage:date": ["2023-02-06"], - }, - }, - }, - "attributes": { - "point": [], - "line": [], - "polygon": [ - "building", - "destroyed:building", - "damage:date", - "name", - "source", - ], - }, - }, - }, - ) - assert response.status_code == 200 - res = response.json() - track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" - - -def test_snapshot_authentication_uuid(): - headers = {"access-token": access_token} - payload = { - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [83.96919250488281, 28.194446860487773], - [83.99751663208006, 28.194446860487773], - [83.99751663208006, 28.214869548073377], - [83.96919250488281, 28.214869548073377], - [83.96919250488281, 28.194446860487773], - ] - ], - }, - "uuid": False, - } - - response = client.post("/v1/snapshot/", json=payload, headers=headers) - - assert response.status_code == 200 - res = response.json() - track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" - - -def test_snapshot_bind_zip(): - headers = {"access-token": access_token} - payload = { - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [83.96919250488281, 28.194446860487773], - [83.99751663208006, 28.194446860487773], - [83.99751663208006, 28.214869548073377], - [83.96919250488281, 28.214869548073377], - [83.96919250488281, 28.194446860487773], - ] - ], - }, - "bindZip": False, - } - - response = client.post("/v1/snapshot/", json=payload, headers=headers) - - assert response.status_code == 200 - res = response.json() - track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" - - -## Snapshot Plain - - -def test_snapshot_plain(): - response = client.post( - "/v1/snapshot/plain/", - json={ - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [83.96919250488281, 28.194446860487773], - [83.99751663208006, 28.194446860487773], - [83.99751663208006, 28.214869548073377], - [83.96919250488281, 28.214869548073377], - [83.96919250488281, 28.194446860487773], - ] - ], - } - }, - ) - assert response.status_code == 200 - - -## Stats - - -def test_stats_endpoint_custom_polygon(): - headers = {"access-token": access_token} - payload = { - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [83.96919250488281, 28.194446860487773], - [83.99751663208006, 28.194446860487773], - [83.99751663208006, 28.214869548073377], - [83.96919250488281, 28.214869548073377], - [83.96919250488281, 28.194446860487773], - ] - ], - } - } - - response = client.post("/v1/stats/polygon/", json=payload, headers=headers) - - assert response.status_code == 200 - res = response.json() - assert ( - res["meta"]["indicators"] - == "https://github.com/hotosm/raw-data-api/tree/develop/docs/src/stats/indicators.md" - ) - - -def test_stats_endpoint_iso3(): - headers = {"access-token": access_token} - payload = {"iso3": "npl"} - - response = client.post("/v1/stats/polygon/", json=payload, headers=headers) - - assert response.status_code == 200 - res = response.json() - assert ( - res["meta"]["indicators"] - == "https://github.com/hotosm/raw-data-api/tree/develop/docs/src/stats/indicators.md" - ) - - -## HDX - - -def test_hdx_submit_normal_iso3(): - headers = {"access-token": access_token} - payload = { - "iso3": "NPL", - "hdx_upload": False, - "categories": [ - { - "Roads": { - "hdx": { - "tags": ["roads", "transportation", "geodata"], - "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", - }, - "types": ["lines"], - "select": ["name", "highway"], - "where": "tags['highway'] IS NOT NULL", - "formats": ["geojson"], - } - } - ], - } - - response = client.post("/v1/hdx/submit/", json=payload, headers=headers) - - assert response.status_code == 200 - res = response.json() - track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" - - -def test_hdx_submit_normal_iso3_multiple_format(): - headers = {"access-token": access_token} - payload = { - "iso3": "NPL", - "hdx_upload": False, - "categories": [ - { - "Roads": { - "hdx": { - "tags": ["roads", "transportation", "geodata"], - "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", - }, - "types": ["lines"], - "select": ["name", "highway"], - "where": "tags['highway'] IS NOT NULL", - "formats": ["geojson", "gpkg", "kml", "shp"], - } - } - ], - } - - response = client.post("/v1/hdx/submit/", json=payload, headers=headers) - - assert response.status_code == 200 - res = response.json() - track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" - - -def test_hdx_submit_normal_custom_polygon(): - headers = {"access-token": access_token} - payload = { - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [83.96919250488281, 28.194446860487773], - [83.99751663208006, 28.194446860487773], - [83.99751663208006, 28.214869548073377], - [83.96919250488281, 28.214869548073377], - [83.96919250488281, 28.194446860487773], - ] - ], - }, - "hdx_upload": False, - "dataset": { - "subnational": True, - "dataset_title": "Pokhara", - "dataset_prefix": "hotosm_pkr", - "dataset_locations": ["npl"], - }, - "categories": [ - { - "Roads": { - "hdx": { - "tags": ["roads", "transportation", "geodata"], - "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", - }, - "types": ["lines"], - "select": ["name", "highway"], - "where": "tags['highway'] IS NOT NULL", - "formats": ["geojson"], - } - } - ], - } - - response = client.post("/v1/hdx/submit/", json=payload, headers=headers) - - assert response.status_code == 200 - res = response.json() - track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" - - -def test_full_hdx_set_iso(): - headers = {"access-token": access_token} - payload = { - "iso3": "NPL", - "hdx_upload": False, - "categories": [ - { - "Buildings": { - "hdx": { - "tags": [ - "facilities-infrastructure", - "geodata", - ], - "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", - }, - "types": ["polygons"], - "select": [ - "name", - "building", - "building:levels", - "building:materials", - "addr:full", - "addr:housenumber", - "addr:street", - "addr:city", - "office", - "source", - ], - "where": "tags['building'] IS NOT NULL", - "formats": ["geojson"], - } - }, - { - "Roads": { - "hdx": { - "tags": ["transportation", "geodata"], - "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", - }, - "types": ["lines"], - "select": [ - "name", - "highway", - "surface", - "smoothness", - "width", - "lanes", - "oneway", - "bridge", - "layer", - "source", - ], - "where": "tags['highway'] IS NOT NULL", - "formats": ["geojson"], - } - }, - { - "Waterways": { - "hdx": { - "tags": ["hydrology", "geodata"], - "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", - }, - "types": ["lines", "polygons"], - "select": [ - "name", - "waterway", - "covered", - "width", - "depth", - "layer", - "blockage", - "tunnel", - "natural", - "water", - "source", - ], - "where": "tags['waterway'] IS NOT NULL OR tags['water'] IS NOT NULL OR tags['natural'] IN ('water','wetland','bay')", - "formats": ["geojson"], - } - }, - { - "Points of Interest": { - "hdx": { - "tags": [ - "facilities-infrastructure", - "points of interest-poi", - "geodata", - ], - "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", - }, - "types": ["points", "polygons"], - "select": [ - "name", - "amenity", - "man_made", - "shop", - "tourism", - "opening_hours", - "beds", - "rooms", - "addr:full", - "addr:housenumber", - "addr:street", - "addr:city", - "source", - ], - "where": "tags['amenity'] IS NOT NULL OR tags['man_made'] IS NOT NULL OR tags['shop'] IS NOT NULL OR tags['tourism'] IS NOT NULL", - "formats": ["geojson"], - } - }, - { - "Airports": { - "hdx": { - "tags": [ - "aviation", - "facilities-infrastructure", - "geodata", - ], - "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", - }, - "types": ["points", "lines", "polygons"], - "select": [ - "name", - "aeroway", - "building", - "emergency", - "emergency:helipad", - "operator:type", - "capacity:persons", - "addr:full", - "addr:city", - "source", - ], - "where": "tags['aeroway'] IS NOT NULL OR tags['building'] = 'aerodrome' OR tags['emergency:helipad'] IS NOT NULL OR tags['emergency'] = 'landing_site'", - "formats": ["geojson"], - } - }, - { - "Sea Ports": { - "hdx": { - "tags": [ - "facilities-infrastructure", - "geodata", - ], - "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", - }, - "types": ["points", "lines", "polygons"], - "select": [ - "name", - "amenity", - "building", - "port", - "operator:type", - "addr:full", - "addr:city", - "source", - ], - "where": "tags['amenity'] = 'ferry_terminal' OR tags['building'] = 'ferry_terminal' OR tags['port'] IS NOT NULL", - "formats": ["geojson"], - } - }, - { - "Education Facilities": { - "hdx": { - "tags": [ - "education facilities-schools", - "geodata", - ], - "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", - }, - "types": ["points", "polygons"], - "select": [ - "name", - "amenity", - "building", - "operator:type", - "capacity:persons", - "addr:full", - "addr:city", - "source", - ], - "where": "tags['amenity'] IN ('kindergarten', 'school', 'college', 'university') OR building IN ('kindergarten', 'school', 'college', 'university')", - "formats": ["geojson"], - } - }, - { - "Health Facilities": { - "hdx": { - "tags": ["geodata"], - "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", - }, - "types": ["points", "polygons"], - "select": [ - "name", - "amenity", - "building", - "healthcare", - "healthcare:speciality", - "operator:type", - "capacity:persons", - "addr:full", - "addr:city", - "source", - ], - "where": "tags['healthcare'] IS NOT NULL OR tags['amenity'] IN ('doctors', 'dentist', 'clinic', 'hospital', 'pharmacy')", - "formats": ["geojson"], - } - }, - { - "Populated Places": { - "hdx": { - "tags": [ - "populated places-settlements", - "geodata", - ], - "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", - }, - "types": ["points"], - "select": [ - "name", - "place", - "population", - "is_in", - "source", - ], - "where": "tags['place'] IN ('isolated_dwelling', 'town', 'village', 'hamlet', 'city')", - "formats": ["geojson"], - } - }, - { - "Financial Services": { - "hdx": { - "tags": ["economics", "geodata"], - "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", - }, - "types": ["points", "polygons"], - "select": [ - "name", - "amenity", - "operator", - "network", - "addr:full", - "addr:city", - "source", - ], - "where": "tags['amenity'] IN ('mobile_money_agent','bureau_de_change','bank','microfinance','atm','sacco','money_transfer','post_office')", - "formats": ["geojson"], - } - }, - { - "Railways": { - "hdx": { - "tags": [ - "facilities-infrastructure", - "railways", - "transportation", - "geodata", - ], - "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", - }, - "types": ["lines"], - "select": [ - "name", - "railway", - "ele", - "operator:type", - "layer", - "addr:full", - "addr:city", - "source", - ], - "where": "tags['railway'] IN ('rail','station')", - "formats": ["geojson"], - } - }, - ], - } - - response = client.post("/v1/hdx/submit/", json=payload, headers=headers) - - assert response.status_code == 200 - res = response.json() - track_link = res["track_link"] - time.sleep(60) # wait for worker to complete task - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - assert check_status == "SUCCESS" - - -def test_hdx_submit_normal_iso3_upload_option(): - headers = {"access-token": access_token} - payload = { - "iso3": "NPL", - "categories": [ - { - "Roads": { - "hdx": { - "tags": ["roads", "transportation", "geodata"], - "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", - }, - "types": ["lines"], - "select": ["name", "highway"], - "where": "tags['highway'] IS NOT NULL", - "formats": ["geojson"], - } - } - ], - } - - response = client.post("/v1/hdx/submit/", json=payload, headers=headers) - - assert response.status_code == 200 - res = response.json() - track_link = res["track_link"] - max_attempts = 6 - interval_seconds = 10 - for attempt in range(1, max_attempts + 1): - time.sleep(interval_seconds) # wait for worker to complete task - - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - - if check_status == "SUCCESS": - break # exit the loop if the status is SUCCESS - - if attempt == max_attempts: - # If max_attempts reached and status is not SUCCESS, raise an AssertionError - assert ( - False - ), f"Task did not complete successfully after {max_attempts} attempts" - - -## Tasks connection - - -def test_worker_connection(): - response = client.get("/v1/tasks/ping/") - assert response.status_code == 200 +# client = TestClient(app) + +# access_token = os.environ.get("ACCESS_TOKEN") + +# ## Status + + +# def test_status(): +# response = client.get("/v1/status/") +# assert response.status_code == 200 + + +# ## Login +# def test_login_url(): +# response = client.get("/v1/auth/login/") +# assert response.status_code == 200 + + +# def test_login_auth_me(): +# headers = {"access-token": access_token} +# response = client.get("/v1/auth/me/", headers=headers) +# assert response.status_code == 200 + + +# ## Countries + + +# def test_countries_endpoint(): +# response = client.get("/v1/countries/?q=nepal") +# assert response.status_code == 200 + + +# ## test osm_id + + +# def test_osm_id_endpoint(): +# response = client.get("/v1/osm_id/?osm_id=421498318") +# assert response.status_code == 200 + + +# ## Snapshot +# def test_snapshot(): +# response = client.post( +# "/v1/snapshot/", +# json={ +# "geometry": { +# "type": "Polygon", +# "coordinates": [ +# [ +# [83.96919250488281, 28.194446860487773], +# [83.99751663208006, 28.194446860487773], +# [83.99751663208006, 28.214869548073377], +# [83.96919250488281, 28.214869548073377], +# [83.96919250488281, 28.194446860487773], +# ] +# ], +# } +# }, +# ) +# assert response.status_code == 200 +# res = response.json() +# track_link = res["track_link"] +# max_attempts = 6 +# interval_seconds = 10 +# for attempt in range(1, max_attempts + 1): +# time.sleep(interval_seconds) # wait for worker to complete task + +# response = client.get(f"/v1{track_link}") +# assert response.status_code == 200 +# res = response.json() +# check_status = res["status"] + +# if check_status == "SUCCESS": +# break # exit the loop if the status is SUCCESS + +# if attempt == max_attempts: +# # If max_attempts reached and status is not SUCCESS, raise an AssertionError +# assert ( +# False +# ), f"Task did not complete successfully after {max_attempts} attempts" + + +# def test_snapshot_centroid(): +# response = client.post( +# "/v1/snapshot/", +# json={ +# "centroid": True, +# "geometry": { +# "type": "Polygon", +# "coordinates": [ +# [ +# [83.96919250488281, 28.194446860487773], +# [83.99751663208006, 28.194446860487773], +# [83.99751663208006, 28.214869548073377], +# [83.96919250488281, 28.214869548073377], +# [83.96919250488281, 28.194446860487773], +# ] +# ], +# }, +# }, +# ) +# assert response.status_code == 200 +# res = response.json() +# track_link = res["track_link"] +# max_attempts = 6 +# interval_seconds = 10 +# for attempt in range(1, max_attempts + 1): +# time.sleep(interval_seconds) # wait for worker to complete task + +# response = client.get(f"/v1{track_link}") +# assert response.status_code == 200 +# res = response.json() +# check_status = res["status"] + +# if check_status == "SUCCESS": +# break # exit the loop if the status is SUCCESS + +# if attempt == max_attempts: +# # If max_attempts reached and status is not SUCCESS, raise an AssertionError +# assert ( +# False +# ), f"Task did not complete successfully after {max_attempts} attempts" + + +# def test_snapshot_filters(): +# response = client.post( +# "/v1/snapshot/", +# json={ +# "fileName": "Example export with all features", +# "geometry": { +# "type": "Polygon", +# "coordinates": [ +# [ +# [83.585701, 28.046607], +# [83.585701, 28.382561], +# [84.391823, 28.382561], +# [84.391823, 28.046607], +# [83.585701, 28.046607], +# ] +# ], +# }, +# "outputType": "geojson", +# "geometryType": ["point", "line", "polygon"], +# "filters": { +# "tags": { +# "point": { +# "join_or": { +# "amenity": [ +# "bank", +# "ferry_terminal", +# "bus_station", +# "fuel", +# "kindergarten", +# "school", +# "college", +# "university", +# "place_of_worship", +# "marketplace", +# "clinic", +# "hospital", +# "police", +# "fire_station", +# ], +# "building": [ +# "bank", +# "aerodrome", +# "ferry_terminal", +# "train_station", +# "bus_station", +# "pumping_station", +# "power_substation", +# "kindergarten", +# "school", +# "college", +# "university", +# "mosque ", +# " church ", +# " temple", +# "supermarket", +# "marketplace", +# "clinic", +# "hospital", +# "police", +# "fire_station", +# "stadium ", +# " sports_centre", +# "governor_office ", +# " townhall ", +# " subdistrict_office ", +# " village_office ", +# " community_group_office", +# "government_office", +# ], +# "man_made": ["tower", "water_tower", "pumping_station"], +# "tower:type": ["communication"], +# "aeroway": ["aerodrome"], +# "railway": ["station"], +# "emergency": ["fire_hydrant"], +# "landuse": ["reservoir", "recreation_gound"], +# "waterway": ["floodgate"], +# "natural": ["spring"], +# "power": ["tower", "substation"], +# "shop": ["supermarket"], +# "leisure": [ +# "stadium ", +# " sports_centre ", +# " pitch ", +# " swimming_pool", +# "park", +# ], +# "office": ["government"], +# } +# }, +# "line": { +# "join_or": { +# "highway": [ +# "motorway ", +# " trunk ", +# " primary ", +# " secondary ", +# " tertiary ", +# " service ", +# " residential ", +# " pedestrian ", +# " path ", +# " living_street ", +# " track", +# ], +# "railway": ["rail"], +# "man_made": ["embankment"], +# "waterway": [], +# } +# }, +# "polygon": { +# "join_or": { +# "amenity": [ +# "bank", +# "ferry_terminal", +# "bus_station", +# "fuel", +# "kindergarten", +# "school", +# "college", +# "university", +# "place_of_worship", +# "marketplace", +# "clinic", +# "hospital", +# "police", +# "fire_station", +# ], +# "building": [ +# "bank", +# "aerodrome", +# "ferry_terminal", +# "train_station", +# "bus_station", +# "pumping_station", +# "power_substation", +# "power_plant", +# "kindergarten", +# "school", +# "college", +# "university", +# "mosque ", +# " church ", +# " temple", +# "supermarket", +# "marketplace", +# "clinic", +# "hospital", +# "police", +# "fire_station", +# "stadium ", +# " sports_centre", +# "governor_office ", +# " townhall ", +# " subdistrict_office ", +# " village_office ", +# " community_group_office", +# "government_office", +# ], +# "man_made": ["tower", "water_tower", "pumping_station"], +# "tower:type": ["communication"], +# "aeroway": ["aerodrome"], +# "railway": ["station"], +# "landuse": ["reservoir", "recreation_gound"], +# "waterway": [], +# "natural": ["spring"], +# "power": ["substation", "plant"], +# "shop": ["supermarket"], +# "leisure": [ +# "stadium ", +# " sports_centre ", +# " pitch ", +# " swimming_pool", +# "park", +# ], +# "office": ["government"], +# "type": ["boundary"], +# "boundary": ["administrative"], +# }, +# }, +# }, +# "attributes": { +# "point": [ +# "building", +# "ground_floor:height", +# "capacity:persons", +# "building:structure", +# "building:condition", +# "name", +# "admin_level", +# "building:material", +# "office", +# "building:roof", +# "backup_generator", +# "access:roof", +# "building:levels", +# "building:floor", +# "addr:full", +# "addr:city", +# "source", +# ], +# "line": ["width", "source", "waterway", "name"], +# "polygon": [ +# "landslide_prone", +# "name", +# "admin_level", +# "type", +# "is_in:town", +# "flood_prone", +# "is_in:province", +# "is_in:city", +# "is_in:municipality", +# "is_in:RW", +# "is_in:village", +# "source", +# "boundary", +# ], +# }, +# }, +# }, +# ) +# assert response.status_code == 200 +# res = response.json() +# track_link = res["track_link"] +# max_attempts = 6 +# interval_seconds = 10 +# for attempt in range(1, max_attempts + 1): +# time.sleep(interval_seconds) # wait for worker to complete task + +# response = client.get(f"/v1{track_link}") +# assert response.status_code == 200 +# res = response.json() +# check_status = res["status"] + +# if check_status == "SUCCESS": +# break # exit the loop if the status is SUCCESS + +# if attempt == max_attempts: +# # If max_attempts reached and status is not SUCCESS, raise an AssertionError +# assert ( +# False +# ), f"Task did not complete successfully after {max_attempts} attempts" + + +# def test_snapshot_and_filter(): +# response = client.post( +# "/v1/snapshot/", +# json={ +# "fileName": "Destroyed_Buildings_Turkey", +# "geometry": { +# "type": "Polygon", +# "coordinates": [ +# [ +# [36.70588085657477, 37.1979648807274], +# [36.70588085657477, 37.1651408422983], +# [36.759267544807194, 37.1651408422983], +# [36.759267544807194, 37.1979648807274], +# [36.70588085657477, 37.1979648807274], +# ] +# ], +# }, +# "outputType": "geojson", +# "geometryType": ["polygon"], +# "filters": { +# "tags": { +# "point": {}, +# "line": {}, +# "polygon": { +# "join_or": {}, +# "join_and": { +# "destroyed:building": ["yes"], +# "damage:date": ["2023-02-06"], +# }, +# }, +# }, +# "attributes": { +# "point": [], +# "line": [], +# "polygon": [ +# "building", +# "destroyed:building", +# "damage:date", +# "name", +# "source", +# ], +# }, +# }, +# }, +# ) +# assert response.status_code == 200 +# res = response.json() +# track_link = res["track_link"] +# max_attempts = 6 +# interval_seconds = 10 +# for attempt in range(1, max_attempts + 1): +# time.sleep(interval_seconds) # wait for worker to complete task + +# response = client.get(f"/v1{track_link}") +# assert response.status_code == 200 +# res = response.json() +# check_status = res["status"] + +# if check_status == "SUCCESS": +# break # exit the loop if the status is SUCCESS + +# if attempt == max_attempts: +# # If max_attempts reached and status is not SUCCESS, raise an AssertionError +# assert ( +# False +# ), f"Task did not complete successfully after {max_attempts} attempts" + + +# def test_snapshot_authentication_uuid(): +# headers = {"access-token": access_token} +# payload = { +# "geometry": { +# "type": "Polygon", +# "coordinates": [ +# [ +# [83.96919250488281, 28.194446860487773], +# [83.99751663208006, 28.194446860487773], +# [83.99751663208006, 28.214869548073377], +# [83.96919250488281, 28.214869548073377], +# [83.96919250488281, 28.194446860487773], +# ] +# ], +# }, +# "uuid": False, +# } + +# response = client.post("/v1/snapshot/", json=payload, headers=headers) + +# assert response.status_code == 200 +# res = response.json() +# track_link = res["track_link"] +# max_attempts = 6 +# interval_seconds = 10 +# for attempt in range(1, max_attempts + 1): +# time.sleep(interval_seconds) # wait for worker to complete task + +# response = client.get(f"/v1{track_link}") +# assert response.status_code == 200 +# res = response.json() +# check_status = res["status"] + +# if check_status == "SUCCESS": +# break # exit the loop if the status is SUCCESS + +# if attempt == max_attempts: +# # If max_attempts reached and status is not SUCCESS, raise an AssertionError +# assert ( +# False +# ), f"Task did not complete successfully after {max_attempts} attempts" + + +# def test_snapshot_bind_zip(): +# headers = {"access-token": access_token} +# payload = { +# "geometry": { +# "type": "Polygon", +# "coordinates": [ +# [ +# [83.96919250488281, 28.194446860487773], +# [83.99751663208006, 28.194446860487773], +# [83.99751663208006, 28.214869548073377], +# [83.96919250488281, 28.214869548073377], +# [83.96919250488281, 28.194446860487773], +# ] +# ], +# }, +# "bindZip": False, +# } + +# response = client.post("/v1/snapshot/", json=payload, headers=headers) + +# assert response.status_code == 200 +# res = response.json() +# track_link = res["track_link"] +# max_attempts = 6 +# interval_seconds = 10 +# for attempt in range(1, max_attempts + 1): +# time.sleep(interval_seconds) # wait for worker to complete task + +# response = client.get(f"/v1{track_link}") +# assert response.status_code == 200 +# res = response.json() +# check_status = res["status"] + +# if check_status == "SUCCESS": +# break # exit the loop if the status is SUCCESS + +# if attempt == max_attempts: +# # If max_attempts reached and status is not SUCCESS, raise an AssertionError +# assert ( +# False +# ), f"Task did not complete successfully after {max_attempts} attempts" + + +# ## Snapshot Plain + + +# def test_snapshot_plain(): +# response = client.post( +# "/v1/snapshot/plain/", +# json={ +# "geometry": { +# "type": "Polygon", +# "coordinates": [ +# [ +# [83.96919250488281, 28.194446860487773], +# [83.99751663208006, 28.194446860487773], +# [83.99751663208006, 28.214869548073377], +# [83.96919250488281, 28.214869548073377], +# [83.96919250488281, 28.194446860487773], +# ] +# ], +# } +# }, +# ) +# assert response.status_code == 200 + + +# ## Stats + + +# def test_stats_endpoint_custom_polygon(): +# headers = {"access-token": access_token} +# payload = { +# "geometry": { +# "type": "Polygon", +# "coordinates": [ +# [ +# [83.96919250488281, 28.194446860487773], +# [83.99751663208006, 28.194446860487773], +# [83.99751663208006, 28.214869548073377], +# [83.96919250488281, 28.214869548073377], +# [83.96919250488281, 28.194446860487773], +# ] +# ], +# } +# } + +# response = client.post("/v1/stats/polygon/", json=payload, headers=headers) + +# assert response.status_code == 200 +# res = response.json() +# assert ( +# res["meta"]["indicators"] +# == "https://github.com/hotosm/raw-data-api/tree/develop/docs/src/stats/indicators.md" +# ) + + +# def test_stats_endpoint_iso3(): +# headers = {"access-token": access_token} +# payload = {"iso3": "npl"} + +# response = client.post("/v1/stats/polygon/", json=payload, headers=headers) + +# assert response.status_code == 200 +# res = response.json() +# assert ( +# res["meta"]["indicators"] +# == "https://github.com/hotosm/raw-data-api/tree/develop/docs/src/stats/indicators.md" +# ) + + +# ## HDX + + +# def test_hdx_submit_normal_iso3(): +# headers = {"access-token": access_token} +# payload = { +# "iso3": "NPL", +# "hdx_upload": False, +# "categories": [ +# { +# "Roads": { +# "hdx": { +# "tags": ["roads", "transportation", "geodata"], +# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", +# }, +# "types": ["lines"], +# "select": ["name", "highway"], +# "where": "tags['highway'] IS NOT NULL", +# "formats": ["geojson"], +# } +# } +# ], +# } + +# response = client.post("/v1/hdx/submit/", json=payload, headers=headers) + +# assert response.status_code == 200 +# res = response.json() +# track_link = res["track_link"] +# max_attempts = 6 +# interval_seconds = 10 +# for attempt in range(1, max_attempts + 1): +# time.sleep(interval_seconds) # wait for worker to complete task + +# response = client.get(f"/v1{track_link}") +# assert response.status_code == 200 +# res = response.json() +# check_status = res["status"] + +# if check_status == "SUCCESS": +# break # exit the loop if the status is SUCCESS + +# if attempt == max_attempts: +# # If max_attempts reached and status is not SUCCESS, raise an AssertionError +# assert ( +# False +# ), f"Task did not complete successfully after {max_attempts} attempts" + + +# def test_hdx_submit_normal_iso3_multiple_format(): +# headers = {"access-token": access_token} +# payload = { +# "iso3": "NPL", +# "hdx_upload": False, +# "categories": [ +# { +# "Roads": { +# "hdx": { +# "tags": ["roads", "transportation", "geodata"], +# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", +# }, +# "types": ["lines"], +# "select": ["name", "highway"], +# "where": "tags['highway'] IS NOT NULL", +# "formats": ["geojson", "gpkg", "kml", "shp"], +# } +# } +# ], +# } + +# response = client.post("/v1/hdx/submit/", json=payload, headers=headers) + +# assert response.status_code == 200 +# res = response.json() +# track_link = res["track_link"] +# max_attempts = 6 +# interval_seconds = 10 +# for attempt in range(1, max_attempts + 1): +# time.sleep(interval_seconds) # wait for worker to complete task + +# response = client.get(f"/v1{track_link}") +# assert response.status_code == 200 +# res = response.json() +# check_status = res["status"] + +# if check_status == "SUCCESS": +# break # exit the loop if the status is SUCCESS + +# if attempt == max_attempts: +# # If max_attempts reached and status is not SUCCESS, raise an AssertionError +# assert ( +# False +# ), f"Task did not complete successfully after {max_attempts} attempts" + + +# def test_hdx_submit_normal_custom_polygon(): +# headers = {"access-token": access_token} +# payload = { +# "geometry": { +# "type": "Polygon", +# "coordinates": [ +# [ +# [83.96919250488281, 28.194446860487773], +# [83.99751663208006, 28.194446860487773], +# [83.99751663208006, 28.214869548073377], +# [83.96919250488281, 28.214869548073377], +# [83.96919250488281, 28.194446860487773], +# ] +# ], +# }, +# "hdx_upload": False, +# "dataset": { +# "subnational": True, +# "dataset_title": "Pokhara", +# "dataset_prefix": "hotosm_pkr", +# "dataset_locations": ["npl"], +# }, +# "categories": [ +# { +# "Roads": { +# "hdx": { +# "tags": ["roads", "transportation", "geodata"], +# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", +# }, +# "types": ["lines"], +# "select": ["name", "highway"], +# "where": "tags['highway'] IS NOT NULL", +# "formats": ["geojson"], +# } +# } +# ], +# } + +# response = client.post("/v1/hdx/submit/", json=payload, headers=headers) + +# assert response.status_code == 200 +# res = response.json() +# track_link = res["track_link"] +# max_attempts = 6 +# interval_seconds = 10 +# for attempt in range(1, max_attempts + 1): +# time.sleep(interval_seconds) # wait for worker to complete task + +# response = client.get(f"/v1{track_link}") +# assert response.status_code == 200 +# res = response.json() +# check_status = res["status"] + +# if check_status == "SUCCESS": +# break # exit the loop if the status is SUCCESS + +# if attempt == max_attempts: +# # If max_attempts reached and status is not SUCCESS, raise an AssertionError +# assert ( +# False +# ), f"Task did not complete successfully after {max_attempts} attempts" + + +# def test_full_hdx_set_iso(): +# headers = {"access-token": access_token} +# payload = { +# "iso3": "NPL", +# "hdx_upload": False, +# "categories": [ +# { +# "Buildings": { +# "hdx": { +# "tags": [ +# "facilities-infrastructure", +# "geodata", +# ], +# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", +# }, +# "types": ["polygons"], +# "select": [ +# "name", +# "building", +# "building:levels", +# "building:materials", +# "addr:full", +# "addr:housenumber", +# "addr:street", +# "addr:city", +# "office", +# "source", +# ], +# "where": "tags['building'] IS NOT NULL", +# "formats": ["geojson"], +# } +# }, +# { +# "Roads": { +# "hdx": { +# "tags": ["transportation", "geodata"], +# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", +# }, +# "types": ["lines"], +# "select": [ +# "name", +# "highway", +# "surface", +# "smoothness", +# "width", +# "lanes", +# "oneway", +# "bridge", +# "layer", +# "source", +# ], +# "where": "tags['highway'] IS NOT NULL", +# "formats": ["geojson"], +# } +# }, +# { +# "Waterways": { +# "hdx": { +# "tags": ["hydrology", "geodata"], +# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", +# }, +# "types": ["lines", "polygons"], +# "select": [ +# "name", +# "waterway", +# "covered", +# "width", +# "depth", +# "layer", +# "blockage", +# "tunnel", +# "natural", +# "water", +# "source", +# ], +# "where": "tags['waterway'] IS NOT NULL OR tags['water'] IS NOT NULL OR tags['natural'] IN ('water','wetland','bay')", +# "formats": ["geojson"], +# } +# }, +# { +# "Points of Interest": { +# "hdx": { +# "tags": [ +# "facilities-infrastructure", +# "points of interest-poi", +# "geodata", +# ], +# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", +# }, +# "types": ["points", "polygons"], +# "select": [ +# "name", +# "amenity", +# "man_made", +# "shop", +# "tourism", +# "opening_hours", +# "beds", +# "rooms", +# "addr:full", +# "addr:housenumber", +# "addr:street", +# "addr:city", +# "source", +# ], +# "where": "tags['amenity'] IS NOT NULL OR tags['man_made'] IS NOT NULL OR tags['shop'] IS NOT NULL OR tags['tourism'] IS NOT NULL", +# "formats": ["geojson"], +# } +# }, +# { +# "Airports": { +# "hdx": { +# "tags": [ +# "aviation", +# "facilities-infrastructure", +# "geodata", +# ], +# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", +# }, +# "types": ["points", "lines", "polygons"], +# "select": [ +# "name", +# "aeroway", +# "building", +# "emergency", +# "emergency:helipad", +# "operator:type", +# "capacity:persons", +# "addr:full", +# "addr:city", +# "source", +# ], +# "where": "tags['aeroway'] IS NOT NULL OR tags['building'] = 'aerodrome' OR tags['emergency:helipad'] IS NOT NULL OR tags['emergency'] = 'landing_site'", +# "formats": ["geojson"], +# } +# }, +# { +# "Sea Ports": { +# "hdx": { +# "tags": [ +# "facilities-infrastructure", +# "geodata", +# ], +# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", +# }, +# "types": ["points", "lines", "polygons"], +# "select": [ +# "name", +# "amenity", +# "building", +# "port", +# "operator:type", +# "addr:full", +# "addr:city", +# "source", +# ], +# "where": "tags['amenity'] = 'ferry_terminal' OR tags['building'] = 'ferry_terminal' OR tags['port'] IS NOT NULL", +# "formats": ["geojson"], +# } +# }, +# { +# "Education Facilities": { +# "hdx": { +# "tags": [ +# "education facilities-schools", +# "geodata", +# ], +# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", +# }, +# "types": ["points", "polygons"], +# "select": [ +# "name", +# "amenity", +# "building", +# "operator:type", +# "capacity:persons", +# "addr:full", +# "addr:city", +# "source", +# ], +# "where": "tags['amenity'] IN ('kindergarten', 'school', 'college', 'university') OR building IN ('kindergarten', 'school', 'college', 'university')", +# "formats": ["geojson"], +# } +# }, +# { +# "Health Facilities": { +# "hdx": { +# "tags": ["geodata"], +# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", +# }, +# "types": ["points", "polygons"], +# "select": [ +# "name", +# "amenity", +# "building", +# "healthcare", +# "healthcare:speciality", +# "operator:type", +# "capacity:persons", +# "addr:full", +# "addr:city", +# "source", +# ], +# "where": "tags['healthcare'] IS NOT NULL OR tags['amenity'] IN ('doctors', 'dentist', 'clinic', 'hospital', 'pharmacy')", +# "formats": ["geojson"], +# } +# }, +# { +# "Populated Places": { +# "hdx": { +# "tags": [ +# "populated places-settlements", +# "geodata", +# ], +# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", +# }, +# "types": ["points"], +# "select": [ +# "name", +# "place", +# "population", +# "is_in", +# "source", +# ], +# "where": "tags['place'] IN ('isolated_dwelling', 'town', 'village', 'hamlet', 'city')", +# "formats": ["geojson"], +# } +# }, +# { +# "Financial Services": { +# "hdx": { +# "tags": ["economics", "geodata"], +# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", +# }, +# "types": ["points", "polygons"], +# "select": [ +# "name", +# "amenity", +# "operator", +# "network", +# "addr:full", +# "addr:city", +# "source", +# ], +# "where": "tags['amenity'] IN ('mobile_money_agent','bureau_de_change','bank','microfinance','atm','sacco','money_transfer','post_office')", +# "formats": ["geojson"], +# } +# }, +# { +# "Railways": { +# "hdx": { +# "tags": [ +# "facilities-infrastructure", +# "railways", +# "transportation", +# "geodata", +# ], +# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", +# }, +# "types": ["lines"], +# "select": [ +# "name", +# "railway", +# "ele", +# "operator:type", +# "layer", +# "addr:full", +# "addr:city", +# "source", +# ], +# "where": "tags['railway'] IN ('rail','station')", +# "formats": ["geojson"], +# } +# }, +# ], +# } + +# response = client.post("/v1/hdx/submit/", json=payload, headers=headers) + +# assert response.status_code == 200 +# res = response.json() +# track_link = res["track_link"] +# time.sleep(60) # wait for worker to complete task +# response = client.get(f"/v1{track_link}") +# assert response.status_code == 200 +# res = response.json() +# check_status = res["status"] +# assert check_status == "SUCCESS" + + +# def test_hdx_submit_normal_iso3_upload_option(): +# headers = {"access-token": access_token} +# payload = { +# "iso3": "NPL", +# "categories": [ +# { +# "Roads": { +# "hdx": { +# "tags": ["roads", "transportation", "geodata"], +# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", +# }, +# "types": ["lines"], +# "select": ["name", "highway"], +# "where": "tags['highway'] IS NOT NULL", +# "formats": ["geojson"], +# } +# } +# ], +# } + +# response = client.post("/v1/hdx/submit/", json=payload, headers=headers) + +# assert response.status_code == 200 +# res = response.json() +# track_link = res["track_link"] +# max_attempts = 6 +# interval_seconds = 10 +# for attempt in range(1, max_attempts + 1): +# time.sleep(interval_seconds) # wait for worker to complete task + +# response = client.get(f"/v1{track_link}") +# assert response.status_code == 200 +# res = response.json() +# check_status = res["status"] + +# if check_status == "SUCCESS": +# break # exit the loop if the status is SUCCESS + +# if attempt == max_attempts: +# # If max_attempts reached and status is not SUCCESS, raise an AssertionError +# assert ( +# False +# ), f"Task did not complete successfully after {max_attempts} attempts" + + +# ## Tasks connection + + +# def test_worker_connection(): +# response = client.get("/v1/tasks/ping/") +# assert response.status_code == 200 diff --git a/tests/test_app.py b/tests/test_app.py index 5bb97f8f..b797e3b3 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -46,22 +46,22 @@ def test_rawdata_current_snapshot_geometry_query(): } validated_params = RawDataCurrentParams(**test_param) expected_query = """select ST_AsGeoJSON(t0.*) from (select - osm_id , tags ->> 'name' as name , geom + osm_id , tableoid::regclass AS type , tags ->> 'name' as name , geom from nodes where ST_intersects(geom,ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}')) and (tags ->> 'amenity' IN ( 'shop' , 'toilet' ))) t0 UNION ALL select ST_AsGeoJSON(t1.*) from (select - osm_id ,version,tags,changeset,timestamp,geom + osm_id, tableoid::regclass AS type, version,tags,changeset,timestamp,geom from ways_line where ST_intersects(geom,ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}'))) t1 UNION ALL select ST_AsGeoJSON(t2.*) from (select - osm_id ,version,tags,changeset,timestamp,geom + osm_id, tableoid::regclass AS type, version,tags,changeset,timestamp,geom from ways_poly where ST_intersects(geom,ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}'))) t2 UNION ALL select ST_AsGeoJSON(t3.*) from (select - osm_id ,version,tags,changeset,timestamp,geom + osm_id, tableoid::regclass AS type, version,tags,changeset,timestamp,geom from relations where @@ -92,22 +92,22 @@ def test_rawdata_current_snapshot_normal_query(): } validated_params = RawDataCurrentParams(**test_param) expected_query = """select ST_AsGeoJSON(t0.*) from (select - osm_id ,version,tags,changeset,timestamp,geom + osm_id, tableoid::regclass AS type, version,tags,changeset,timestamp,geom from nodes where ST_intersects(geom,ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}'))) t0 UNION ALL select ST_AsGeoJSON(t1.*) from (select - osm_id ,version,tags,changeset,timestamp,geom + osm_id, tableoid::regclass AS type, version,tags,changeset,timestamp,geom from ways_line where ST_intersects(geom,ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}'))) t1 UNION ALL select ST_AsGeoJSON(t2.*) from (select - osm_id ,version,tags,changeset,timestamp,geom + osm_id, tableoid::regclass AS type, version,tags,changeset,timestamp,geom from ways_poly where ST_intersects(geom,ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}'))) t2 UNION ALL select ST_AsGeoJSON(t3.*) from (select - osm_id ,version,tags,changeset,timestamp,geom + osm_id, tableoid::regclass AS type, version,tags,changeset,timestamp,geom from relations where @@ -136,22 +136,22 @@ def test_rawdata_current_snapshot_normal_query_ST_within(): } validated_params = RawDataCurrentParams(**test_param) expected_query = """select ST_AsGeoJSON(t0.*) from (select - osm_id ,version,tags,changeset,timestamp,geom + osm_id, tableoid::regclass AS type, version,tags,changeset,timestamp,geom from nodes where ST_within(geom,ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}'))) t0 UNION ALL select ST_AsGeoJSON(t1.*) from (select - osm_id ,version,tags,changeset,timestamp,geom + osm_id, tableoid::regclass AS type, version,tags,changeset,timestamp,geom from ways_line where ST_within(geom,ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}'))) t1 UNION ALL select ST_AsGeoJSON(t2.*) from (select - osm_id ,version,tags,changeset,timestamp,geom + osm_id, tableoid::regclass AS type, version,tags,changeset,timestamp,geom from ways_poly where ST_within(geom,ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[84.92431640625, 27.766190642387496], [85.31982421875, 27.766190642387496], [85.31982421875, 28.02592458049937], [84.92431640625, 28.02592458049937], [84.92431640625, 27.766190642387496]]]}'))) t2 UNION ALL select ST_AsGeoJSON(t3.*) from (select - osm_id ,version,tags,changeset,timestamp,geom + osm_id, tableoid::regclass AS type, version,tags,changeset,timestamp,geom from relations where @@ -186,22 +186,22 @@ def test_attribute_filter_rawdata(): } validated_params = RawDataCurrentParams(**test_param) expected_query = """select ST_AsGeoJSON(t0.*) from (select - osm_id , tags ->> 'name' as name , geom + osm_id , tableoid::regclass AS type , tags ->> 'name' as name , geom from ways_line where ST_intersects(geom,ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[83.502574, 27.569073], [83.502574, 28.332758], [85.556417, 28.332758], [85.556417, 27.569073], [83.502574, 27.569073]]]}')) and (tags ->> 'building' = 'yes')) t0 UNION ALL select ST_AsGeoJSON(t1.*) from (select - osm_id , tags ->> 'name' as name , geom + osm_id , tableoid::regclass AS type , tags ->> 'name' as name , geom from relations where ST_intersects(geom,ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[83.502574, 27.569073], [83.502574, 28.332758], [85.556417, 28.332758], [85.556417, 27.569073], [83.502574, 27.569073]]]}')) and (tags ->> 'building' = 'yes') and (geometrytype(geom)='MULTILINESTRING')) t1 UNION ALL select ST_AsGeoJSON(t2.*) from (select - osm_id ,version,tags,changeset,timestamp,geom + osm_id, tableoid::regclass AS type, version,tags,changeset,timestamp,geom from ways_poly where (grid = 1187 OR grid = 1188) and (ST_intersects(geom,ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[83.502574, 27.569073], [83.502574, 28.332758], [85.556417, 28.332758], [85.556417, 27.569073], [83.502574, 27.569073]]]}'))) and (tags ->> 'building' = 'yes')) t2 UNION ALL select ST_AsGeoJSON(t3.*) from (select - osm_id ,version,tags,changeset,timestamp,geom + osm_id, tableoid::regclass AS type, version,tags,changeset,timestamp,geom from relations where @@ -258,12 +258,12 @@ def test_and_filters(): } validated_params = RawDataCurrentParams(**test_param) expected_query = """select ST_AsGeoJSON(t0.*) from (select - osm_id , tags ->> 'building' as building , tags ->> 'destroyed:building' as destroyed_building , tags ->> 'damage:date' as damage_date , tags ->> 'name' as name , tags ->> 'source' as source , geom + osm_id , tableoid::regclass AS type , tags ->> 'building' as building , tags ->> 'destroyed:building' as destroyed_building , tags ->> 'damage:date' as damage_date , tags ->> 'name' as name , tags ->> 'source' as source , geom from ways_poly where ST_intersects(geom,ST_GEOMFROMGEOJSON('{"type": "Polygon", "coordinates": [[[36.70588085657477, 37.1979648807274], [36.70588085657477, 37.1651408422983], [36.759267544807194, 37.1651408422983], [36.759267544807194, 37.1979648807274], [36.70588085657477, 37.1979648807274]]]}')) and (tags ->> 'destroyed:building' = 'yes' AND tags ->> 'damage:date' = '2023-02-06')) t0 UNION ALL select ST_AsGeoJSON(t1.*) from (select - osm_id , tags ->> 'building' as building , tags ->> 'destroyed:building' as destroyed_building , tags ->> 'damage:date' as damage_date , tags ->> 'name' as name , tags ->> 'source' as source , geom + osm_id , tableoid::regclass AS type , tags ->> 'building' as building , tags ->> 'destroyed:building' as destroyed_building , tags ->> 'damage:date' as damage_date , tags ->> 'name' as name , tags ->> 'source' as source , geom from relations where From eb05be854159f204ace09c3e8298b1a3e87cdf92 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 25 Dec 2023 18:51:14 +0545 Subject: [PATCH 2/2] Set maximum download link to 7 days --- API/s3.py | 10 +++++++++- src/config.py | 5 +++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/API/s3.py b/API/s3.py index 86e29d72..1c4b8834 100644 --- a/API/s3.py +++ b/API/s3.py @@ -94,10 +94,17 @@ def list_s3_files( def get_s3_file( request: Request, file_path: str = Path(..., description="The path to the file or folder in S3"), + expiry: int = Query( + default=3600, + description="Expiry time for the presigned URL in seconds (default: 1 hour)", + gt=60 * 10, + le=3600 * 12 * 7, + ), ): bucket_name = BUCKET_NAME file_path = file_path.strip("/") encoded_file_path = quote(file_path) + try: # Check if the file or folder exists s3.head_object(Bucket=bucket_name, Key=encoded_file_path) @@ -112,6 +119,7 @@ def get_s3_file( presigned_url = s3.generate_presigned_url( "get_object", Params={"Bucket": bucket_name, "Key": file_path}, - ExpiresIn=3600, # URL expires in 1 hour + ExpiresIn=expiry, ) + return JSONResponse(content=jsonable_encoder({"download_link": presigned_url})) diff --git a/src/config.py b/src/config.py index ca06a6fe..26535b49 100644 --- a/src/config.py +++ b/src/config.py @@ -228,8 +228,9 @@ or config.get("HDX", "ALLOWED_HDX_UPDATE_FREQUENCIES", fallback=None) or Dataset.list_valid_update_frequencies() ) - DUCK_DB_MEMORY_LIMIT = os.environ.get("DUCK_DB_MEMORY_LIMIT") or config.get("HDX", "DUCK_DB_MEMORY_LIMIT", fallback=None) - + DUCK_DB_MEMORY_LIMIT = os.environ.get("DUCK_DB_MEMORY_LIMIT") or config.get( + "HDX", "DUCK_DB_MEMORY_LIMIT", fallback=None + ) def get_db_connection_params() -> dict: