diff --git a/API/main.py b/API/main.py index f887895a..9e6dab6b 100644 --- a/API/main.py +++ b/API/main.py @@ -13,11 +13,13 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +# Standard library imports # Humanitarian OpenStreetmap Team # 1100 13th Street NW Suite 800 Washington, D.C. 20005 # import time +# Third party imports import psycopg2 from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -26,6 +28,7 @@ from slowapi import _rate_limit_exceeded_handler from slowapi.errors import RateLimitExceeded +# Reader imports from src.config import ( ENABLE_CUSTOM_EXPORTS, ENABLE_HDX_EXPORTS, @@ -57,6 +60,7 @@ from .hdx import router as hdx_router if SENTRY_DSN: + # Third party imports import sentry_sdk # only use sentry if it is specified in config blocks @@ -71,6 +75,7 @@ if LOG_LEVEL.lower() == "debug": # This is used for local setup for auth login + # Standard library imports import os os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1" diff --git a/src/app.py b/src/app.py index 8c29ee2f..2788efb2 100644 --- a/src/app.py +++ b/src/app.py @@ -599,7 +599,27 @@ def ogr_export(query, outputtype, working_dir, dump_temp_path, params): params.min_zoom, params.max_zoom ) if params.min_zoom and params.max_zoom - else "-dsco ZOOM_LEVEL_AUTO=YES" + else "MINZOOM=0 MAXZOOM=20" + ), + } + format_options[RawDataOutputType.PMTILES.value] = { + "format": "PMTiles", + "extra": ( + "-dsco MINZOOM={} -dsco MAXZOOM={} ".format( + params.min_zoom, params.max_zoom + ) + if params.min_zoom and params.max_zoom + else "-dsco MINZOOM=0 MAXZOOM=20" + ), + } + format_options[RawDataOutputType.MVT.value] = { + "format": "MVT", + "extra": ( + "-dsco MINZOOM={} -dsco MAXZOOM={} ".format( + params.min_zoom, params.max_zoom + ) + if params.min_zoom and params.max_zoom + else "-dsco MINZOOM=0 MAXZOOM=20 FORMAT=MBTILES" ), } @@ -689,16 +709,6 @@ def get_grid_id(geom, cur): country_export, ) - @staticmethod - def geojson2tiles(geojson_path, tile_path, tile_layer_name): - """Responsible for geojson to tiles""" - cmd = """tippecanoe -zg --projection=EPSG:4326 -o {tile_output_path} -l {tile_layer_name} --force {geojson_input_path}""".format( - tile_output_path=tile_path, - tile_layer_name=tile_layer_name, - geojson_input_path=geojson_path, - ) - run_ogr2ogr_cmd(cmd) - def extract_current_data(self, exportname): """Responsible for Extracting rawdata current snapshot, Initially it creates a geojson file , Generates query , run it with 1000 chunk size and writes it directly to the geojson file and closes the file after dump Args: @@ -731,25 +741,11 @@ def extract_current_data(self, exportname): try: # currently we have only geojson binding function written other than that we have depend on ogr if ENABLE_TILES: - if output_type == RawDataOutputType.PMTILES.value: - geojson_path = os.path.join( - working_dir, - f"{self.params.file_name if self.params.file_name else 'Export'}.geojson", - ) - RawData.query2geojson( - self.con, - raw_currentdata_extraction_query( - self.params, - g_id=grid_id, - c_id=country, - country_export=country_export, - ), - geojson_path, - ) - RawData.geojson2tiles( - geojson_path, dump_temp_file_path, self.params.file_name - ) - if output_type == RawDataOutputType.MBTILES.value: + if output_type in [ + RawDataOutputType.PMTILES.value, + RawDataOutputType.MBTILES.value, + RawDataOutputType.MVT.value, + ]: RawData.ogr_export( query=raw_currentdata_extraction_query( self.params, @@ -762,7 +758,7 @@ def extract_current_data(self, exportname): dump_temp_path=dump_temp_file_path, working_dir=working_dir, params=self.params, - ) # uses ogr export to export + ) if output_type == RawDataOutputType.GEOJSON.value: RawData.query2geojson( diff --git a/src/query_builder/builder.py b/src/query_builder/builder.py index 211199ea..90b85eff 100644 --- a/src/query_builder/builder.py +++ b/src/query_builder/builder.py @@ -140,18 +140,22 @@ def create_column_filter( schema = {} schema["osm_id"] = "int64" schema["type"] = "str" - - for cl in columns: - splitted_cl = [cl] - if "," in cl: - splitted_cl = cl.split(",") - for cl in splitted_cl: - if cl != "": - filter_col.append( - f"""tags ->> '{cl.strip()}' as {remove_spaces(cl.strip())}""" - ) - if create_schema: - schema[remove_spaces(cl.strip())] = "str" + if "*" in columns: + filter_col.append("tags") + if create_schema: + schema["tags"] = "str" + else: + for cl in columns: + splitted_cl = [cl] + if "," in cl: + splitted_cl = cl.split(",") + for cl in splitted_cl: + if cl != "": + filter_col.append( + f"""tags ->> '{cl.strip()}' as {remove_spaces(cl.strip())}""" + ) + if create_schema: + schema[remove_spaces(cl.strip())] = "str" if output_type == "csv": # if it is csv geom logic is different filter_col.append("ST_X(ST_Centroid(geom)) as longitude") filter_col.append("ST_Y(ST_Centroid(geom)) as latitude") @@ -973,7 +977,10 @@ def extract_features_custom_exports( }, } if USE_DUCK_DB_FOR_CUSTOM_EXPORTS is True: - select = [f"""tags['{item}'][1] as "{item}" """ for item in select] + if "*" in select: + select = [f"""tags::json as tags """] + else: + select = [f"""tags['{item}'][1] as "{item}" """ for item in select] select += ["osm_id", "osm_type", "geom"] select_query = ", ".join(select) else: diff --git a/src/validation/models.py b/src/validation/models.py index 9f759535..e5aba21f 100644 --- a/src/validation/models.py +++ b/src/validation/models.py @@ -65,6 +65,7 @@ class RawDataOutputType(Enum): if ENABLE_TILES: MBTILES = "mbtiles" PMTILES = "pmtiles" ## EXPERIMENTAL + MVT = "mvt" ## Experimental class SupportedFilters(Enum): @@ -297,22 +298,22 @@ class StatsRequestParams(BaseModel, GeometryValidatorMixin): max_length=3, example="NPL", ) - geometry: Optional[ - Union[Polygon, MultiPolygon, Feature, FeatureCollection] - ] = Field( - default=None, - example={ - "type": "Polygon", - "coordinates": [ - [ - [83.96919250488281, 28.194446860487773], - [83.99751663208006, 28.194446860487773], - [83.99751663208006, 28.214869548073377], - [83.96919250488281, 28.214869548073377], - [83.96919250488281, 28.194446860487773], - ] - ], - }, + geometry: Optional[Union[Polygon, MultiPolygon, Feature, FeatureCollection]] = ( + Field( + default=None, + example={ + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + }, + ) ) @validator("geometry", pre=True, always=True) @@ -479,7 +480,7 @@ def __init__(self, suffix, driver_name, layer_creation_options, format_option): "gpkg": ExportTypeInfo("gpkg", "GPKG", ["SPATIAL_INDEX=No"], "GDAL"), "sqlite": ExportTypeInfo("sqlite", "SQLite", [], "GDAL"), "fgb": ExportTypeInfo("fgb", "FlatGeobuf", ["VERIFY_BUFFERS=NO"], "GDAL"), - "mvt": ExportTypeInfo("mvt", "MVT", [], "GDAL"), + "mvt": ExportTypeInfo("mbtiles", "MVT", ["MAXZOOM=20"], "GDAL"), "kml": ExportTypeInfo("kml", "KML", [], "GDAL"), "gpx": ExportTypeInfo("gpx", "GPX", [], "GDAL"), "parquet": ExportTypeInfo("parquet", "PARQUET", [], "PARQUET"), @@ -618,22 +619,22 @@ class DynamicCategoriesModel(CategoriesBase, GeometryValidatorMixin): max_length=3, example="USA", ) - geometry: Optional[ - Union[Polygon, MultiPolygon, Feature, FeatureCollection] - ] = Field( - default=None, - example={ - "type": "Polygon", - "coordinates": [ - [ - [83.96919250488281, 28.194446860487773], - [83.99751663208006, 28.194446860487773], - [83.99751663208006, 28.214869548073377], - [83.96919250488281, 28.214869548073377], - [83.96919250488281, 28.194446860487773], - ] - ], - }, + geometry: Optional[Union[Polygon, MultiPolygon, Feature, FeatureCollection]] = ( + Field( + default=None, + example={ + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + }, + ) ) @validator("geometry", pre=True, always=True) diff --git a/tests/test_API.py b/tests/test_API.py index 845bf558..750269e5 100644 --- a/tests/test_API.py +++ b/tests/test_API.py @@ -41,1291 +41,1291 @@ def wait_for_task_completion(track_link, max_attempts=12, interval_seconds=5): ) -# def test_status(): -# response = client.get("/v1/status/") -# assert response.status_code == 200 - - -# # Login -# def test_login_url(): -# response = client.get("/v1/auth/login/") -# assert response.status_code == 200 - - -# def test_login_auth_me(): -# headers = {"access-token": access_token} -# response = client.get("/v1/auth/me/", headers=headers) -# assert response.status_code == 200 - - -# ## Countries - - -# def test_countries_endpoint(): -# response = client.get("/v1/countries/?q=nepal") -# assert response.status_code == 200 - - -# ## test osm_id - - -# def test_osm_id_endpoint(): -# response = client.get("/v1/osm_id/?osm_id=421498318") -# assert response.status_code == 200 - - -# ## Snapshot -# def test_snapshot(): -# response = client.post( -# "/v1/snapshot/", -# json={ -# "geometry": { -# "type": "Polygon", -# "coordinates": [ -# [ -# [83.96919250488281, 28.194446860487773], -# [83.99751663208006, 28.194446860487773], -# [83.99751663208006, 28.214869548073377], -# [83.96919250488281, 28.214869548073377], -# [83.96919250488281, 28.194446860487773], -# ] -# ], -# } -# }, -# ) -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# def test_snapshot_featurecollection(): -# response = client.post( -# "/v1/snapshot/", -# json={ -# "geometry": { -# "type": "FeatureCollection", -# "features": [ -# { -# "type": "Feature", -# "properties": {}, -# "geometry": { -# "coordinates": [ -# [ -# [83.97346137271688, 28.217525272345284], -# [83.97346137271688, 28.192595937414737], -# [84.01473909818759, 28.192595937414737], -# [84.01473909818759, 28.217525272345284], -# [83.97346137271688, 28.217525272345284], -# ] -# ], -# "type": "Polygon", -# }, -# } -# ], -# } -# }, -# ) -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# def test_snapshot_feature(): -# response = client.post( -# "/v1/snapshot/", -# json={ -# "geometry": { -# "type": "Feature", -# "properties": {}, -# "geometry": { -# "coordinates": [ -# [ -# [83.97346137271688, 28.217525272345284], -# [83.97346137271688, 28.192595937414737], -# [84.01473909818759, 28.192595937414737], -# [84.01473909818759, 28.217525272345284], -# [83.97346137271688, 28.217525272345284], -# ] -# ], -# "type": "Polygon", -# }, -# } -# }, -# ) -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# def test_snapshot_feature_fgb(): -# response = client.post( -# "/v1/snapshot/", -# json={ -# "outputType": "fgb", -# "geometry": { -# "type": "Feature", -# "properties": {}, -# "geometry": { -# "coordinates": [ -# [ -# [83.97346137271688, 28.217525272345284], -# [83.97346137271688, 28.192595937414737], -# [84.01473909818759, 28.192595937414737], -# [84.01473909818759, 28.217525272345284], -# [83.97346137271688, 28.217525272345284], -# ] -# ], -# "type": "Polygon", -# }, -# }, -# }, -# ) -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# def test_snapshot_feature_fgb_wrap_geom(): -# response = client.post( -# "/v1/snapshot/", -# json={ -# "fgbWrapGeoms": True, -# "outputType": "fgb", -# "geometry": { -# "type": "Feature", -# "properties": {}, -# "geometry": { -# "coordinates": [ -# [ -# [83.97346137271688, 28.217525272345284], -# [83.97346137271688, 28.192595937414737], -# [84.01473909818759, 28.192595937414737], -# [84.01473909818759, 28.217525272345284], -# [83.97346137271688, 28.217525272345284], -# ] -# ], -# "type": "Polygon", -# }, -# }, -# }, -# ) -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# def test_snapshot_feature_shp(): -# response = client.post( -# "/v1/snapshot/", -# json={ -# "outputType": "shp", -# "geometry": { -# "type": "Feature", -# "properties": {}, -# "geometry": { -# "coordinates": [ -# [ -# [83.97346137271688, 28.217525272345284], -# [83.97346137271688, 28.192595937414737], -# [84.01473909818759, 28.192595937414737], -# [84.01473909818759, 28.217525272345284], -# [83.97346137271688, 28.217525272345284], -# ] -# ], -# "type": "Polygon", -# }, -# }, -# }, -# ) -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# def test_snapshot_feature_gpkg(): -# response = client.post( -# "/v1/snapshot/", -# json={ -# "outputType": "gpkg", -# "geometry": { -# "type": "Feature", -# "properties": {}, -# "geometry": { -# "coordinates": [ -# [ -# [83.97346137271688, 28.217525272345284], -# [83.97346137271688, 28.192595937414737], -# [84.01473909818759, 28.192595937414737], -# [84.01473909818759, 28.217525272345284], -# [83.97346137271688, 28.217525272345284], -# ] -# ], -# "type": "Polygon", -# }, -# }, -# }, -# ) -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# def test_snapshot_feature_kml(): -# response = client.post( -# "/v1/snapshot/", -# json={ -# "outputType": "kml", -# "geometry": { -# "type": "Feature", -# "properties": {}, -# "geometry": { -# "coordinates": [ -# [ -# [83.97346137271688, 28.217525272345284], -# [83.97346137271688, 28.192595937414737], -# [84.01473909818759, 28.192595937414737], -# [84.01473909818759, 28.217525272345284], -# [83.97346137271688, 28.217525272345284], -# ] -# ], -# "type": "Polygon", -# }, -# }, -# }, -# ) -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# def test_snapshot_feature_sql(): -# response = client.post( -# "/v1/snapshot/", -# json={ -# "outputType": "sql", -# "geometry": { -# "type": "Feature", -# "properties": {}, -# "geometry": { -# "coordinates": [ -# [ -# [83.97346137271688, 28.217525272345284], -# [83.97346137271688, 28.192595937414737], -# [84.01473909818759, 28.192595937414737], -# [84.01473909818759, 28.217525272345284], -# [83.97346137271688, 28.217525272345284], -# ] -# ], -# "type": "Polygon", -# }, -# }, -# }, -# ) -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# def test_snapshot_feature_csv(): -# response = client.post( -# "/v1/snapshot/", -# json={ -# "outputType": "csv", -# "geometry": { -# "type": "Feature", -# "properties": {}, -# "geometry": { -# "coordinates": [ -# [ -# [83.97346137271688, 28.217525272345284], -# [83.97346137271688, 28.192595937414737], -# [84.01473909818759, 28.192595937414737], -# [84.01473909818759, 28.217525272345284], -# [83.97346137271688, 28.217525272345284], -# ] -# ], -# "type": "Polygon", -# }, -# }, -# }, -# ) -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# def test_snapshot_centroid(): -# response = client.post( -# "/v1/snapshot/", -# json={ -# "centroid": True, -# "geometry": { -# "type": "Polygon", -# "coordinates": [ -# [ -# [83.96919250488281, 28.194446860487773], -# [83.99751663208006, 28.194446860487773], -# [83.99751663208006, 28.214869548073377], -# [83.96919250488281, 28.214869548073377], -# [83.96919250488281, 28.194446860487773], -# ] -# ], -# }, -# }, -# ) -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# def test_snapshot_filters(): -# response = client.post( -# "/v1/snapshot/", -# json={ -# "fileName": "Example export with all features", -# "geometry": { -# "type": "Polygon", -# "coordinates": [ -# [ -# [83.585701, 28.046607], -# [83.585701, 28.382561], -# [84.391823, 28.382561], -# [84.391823, 28.046607], -# [83.585701, 28.046607], -# ] -# ], -# }, -# "outputType": "geojson", -# "geometryType": ["point", "line", "polygon"], -# "filters": { -# "tags": { -# "point": { -# "join_or": { -# "amenity": [ -# "bank", -# "ferry_terminal", -# "bus_station", -# "fuel", -# "kindergarten", -# "school", -# "college", -# "university", -# "place_of_worship", -# "marketplace", -# "clinic", -# "hospital", -# "police", -# "fire_station", -# ], -# "building": [ -# "bank", -# "aerodrome", -# "ferry_terminal", -# "train_station", -# "bus_station", -# "pumping_station", -# "power_substation", -# "kindergarten", -# "school", -# "college", -# "university", -# "mosque ", -# " church ", -# " temple", -# "supermarket", -# "marketplace", -# "clinic", -# "hospital", -# "police", -# "fire_station", -# "stadium ", -# " sports_centre", -# "governor_office ", -# " townhall ", -# " subdistrict_office ", -# " village_office ", -# " community_group_office", -# "government_office", -# ], -# "man_made": ["tower", "water_tower", "pumping_station"], -# "tower:type": ["communication"], -# "aeroway": ["aerodrome"], -# "railway": ["station"], -# "emergency": ["fire_hydrant"], -# "landuse": ["reservoir", "recreation_gound"], -# "waterway": ["floodgate"], -# "natural": ["spring"], -# "power": ["tower", "substation"], -# "shop": ["supermarket"], -# "leisure": [ -# "stadium ", -# " sports_centre ", -# " pitch ", -# " swimming_pool", -# "park", -# ], -# "office": ["government"], -# } -# }, -# "line": { -# "join_or": { -# "highway": [ -# "motorway ", -# " trunk ", -# " primary ", -# " secondary ", -# " tertiary ", -# " service ", -# " residential ", -# " pedestrian ", -# " path ", -# " living_street ", -# " track", -# ], -# "railway": ["rail"], -# "man_made": ["embankment"], -# "waterway": [], -# } -# }, -# "polygon": { -# "join_or": { -# "amenity": [ -# "bank", -# "ferry_terminal", -# "bus_station", -# "fuel", -# "kindergarten", -# "school", -# "college", -# "university", -# "place_of_worship", -# "marketplace", -# "clinic", -# "hospital", -# "police", -# "fire_station", -# ], -# "building": [ -# "bank", -# "aerodrome", -# "ferry_terminal", -# "train_station", -# "bus_station", -# "pumping_station", -# "power_substation", -# "power_plant", -# "kindergarten", -# "school", -# "college", -# "university", -# "mosque ", -# " church ", -# " temple", -# "supermarket", -# "marketplace", -# "clinic", -# "hospital", -# "police", -# "fire_station", -# "stadium ", -# " sports_centre", -# "governor_office ", -# " townhall ", -# " subdistrict_office ", -# " village_office ", -# " community_group_office", -# "government_office", -# ], -# "man_made": ["tower", "water_tower", "pumping_station"], -# "tower:type": ["communication"], -# "aeroway": ["aerodrome"], -# "railway": ["station"], -# "landuse": ["reservoir", "recreation_gound"], -# "waterway": [], -# "natural": ["spring"], -# "power": ["substation", "plant"], -# "shop": ["supermarket"], -# "leisure": [ -# "stadium ", -# " sports_centre ", -# " pitch ", -# " swimming_pool", -# "park", -# ], -# "office": ["government"], -# "type": ["boundary"], -# "boundary": ["administrative"], -# }, -# }, -# }, -# "attributes": { -# "point": [ -# "building", -# "ground_floor:height", -# "capacity:persons", -# "building:structure", -# "building:condition", -# "name", -# "admin_level", -# "building:material", -# "office", -# "building:roof", -# "backup_generator", -# "access:roof", -# "building:levels", -# "building:floor", -# "addr:full", -# "addr:city", -# "source", -# ], -# "line": ["width", "source", "waterway", "name"], -# "polygon": [ -# "landslide_prone", -# "name", -# "admin_level", -# "type", -# "is_in:town", -# "flood_prone", -# "is_in:province", -# "is_in:city", -# "is_in:municipality", -# "is_in:RW", -# "is_in:village", -# "source", -# "boundary", -# ], -# }, -# }, -# }, -# ) -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# def test_snapshot_and_filter(): -# response = client.post( -# "/v1/snapshot/", -# json={ -# "fileName": "Destroyed_Buildings_Turkey", -# "geometry": { -# "type": "Polygon", -# "coordinates": [ -# [ -# [36.70588085657477, 37.1979648807274], -# [36.70588085657477, 37.1651408422983], -# [36.759267544807194, 37.1651408422983], -# [36.759267544807194, 37.1979648807274], -# [36.70588085657477, 37.1979648807274], -# ] -# ], -# }, -# "outputType": "geojson", -# "geometryType": ["polygon"], -# "filters": { -# "tags": { -# "point": {}, -# "line": {}, -# "polygon": { -# "join_or": {}, -# "join_and": { -# "destroyed:building": ["yes"], -# "damage:date": ["2023-02-06"], -# }, -# }, -# }, -# "attributes": { -# "point": [], -# "line": [], -# "polygon": [ -# "building", -# "destroyed:building", -# "damage:date", -# "name", -# "source", -# ], -# }, -# }, -# }, -# ) -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# def test_snapshot_authentication_uuid(): -# headers = {"access-token": access_token} -# payload = { -# "geometry": { -# "type": "Polygon", -# "coordinates": [ -# [ -# [83.96919250488281, 28.194446860487773], -# [83.99751663208006, 28.194446860487773], -# [83.99751663208006, 28.214869548073377], -# [83.96919250488281, 28.214869548073377], -# [83.96919250488281, 28.194446860487773], -# ] -# ], -# }, -# "uuid": False, -# } - -# response = client.post("/v1/snapshot/", json=payload, headers=headers) - -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# def test_snapshot_bind_zip(): -# headers = {"access-token": access_token} -# payload = { -# "geometry": { -# "type": "Polygon", -# "coordinates": [ -# [ -# [83.96919250488281, 28.194446860487773], -# [83.99751663208006, 28.194446860487773], -# [83.99751663208006, 28.214869548073377], -# [83.96919250488281, 28.214869548073377], -# [83.96919250488281, 28.194446860487773], -# ] -# ], -# }, -# "bindZip": False, -# } - -# response = client.post("/v1/snapshot/", json=payload, headers=headers) - -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# ## Snapshot Plain - - -# def test_snapshot_plain(): -# response = client.post( -# "/v1/snapshot/plain/", -# json={ -# "type": "Feature", -# "properties": {}, -# "geometry": { -# "coordinates": [ -# [ -# [83.98223911755514, 28.21348731781771], -# [83.98223911755514, 28.208639873158987], -# [83.9881146731152, 28.208639873158987], -# [83.9881146731152, 28.21348731781771], -# [83.98223911755514, 28.21348731781771], -# ] -# ], -# "type": "Polygon", -# }, -# }, -# ) -# assert response.status_code == 200 - - -# ## Stats - - -# def test_stats_endpoint_custom_polygon(): -# headers = {"access-token": access_token} -# payload = { -# "geometry": { -# "type": "Polygon", -# "coordinates": [ -# [ -# [83.96919250488281, 28.194446860487773], -# [83.99751663208006, 28.194446860487773], -# [83.99751663208006, 28.214869548073377], -# [83.96919250488281, 28.214869548073377], -# [83.96919250488281, 28.194446860487773], -# ] -# ], -# } -# } - -# response = client.post("/v1/stats/polygon/", json=payload, headers=headers) - -# assert response.status_code == 200 -# res = response.json() -# assert ( -# res["meta"]["indicators"] -# == "https://github.com/hotosm/raw-data-api/tree/develop/docs/src/stats/indicators.md" -# ) - - -# def test_stats_endpoint_iso3(): -# headers = {"access-token": access_token} -# payload = {"iso3": "npl"} - -# response = client.post("/v1/stats/polygon/", json=payload, headers=headers) - -# assert response.status_code == 200 -# res = response.json() -# assert ( -# res["meta"]["indicators"] -# == "https://github.com/hotosm/raw-data-api/tree/develop/docs/src/stats/indicators.md" -# ) - - -# # HDX - - -# def test_hdx_submit_normal_iso3(): -# headers = {"access-token": access_token} -# payload = { -# "iso3": "NPL", -# "hdx_upload": False, -# "categories": [ -# { -# "Roads": { -# "hdx": { -# "tags": ["roads", "transportation", "geodata"], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["lines"], -# "select": ["name", "highway"], -# "where": "tags['highway'] IS NOT NULL", -# "formats": ["geojson"], -# } -# } -# ], -# } - -# response = client.post("/v1/custom/snapshot/", json=payload, headers=headers) - -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# def test_hdx_submit_normal_iso3_multiple_format(): -# headers = {"access-token": access_token} -# payload = { -# "iso3": "NPL", -# "hdx_upload": False, -# "categories": [ -# { -# "Roads": { -# "hdx": { -# "tags": ["roads", "transportation", "geodata"], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["lines"], -# "select": ["name", "highway"], -# "where": "tags['highway'] IS NOT NULL", -# "formats": ["geojson", "gpkg", "kml", "shp"], -# } -# } -# ], -# } - -# response = client.post("/v1/custom/snapshot/", json=payload, headers=headers) - -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# def test_hdx_submit_normal_custom_polygon(): -# headers = {"access-token": access_token} -# payload = { -# "geometry": { -# "type": "Polygon", -# "coordinates": [ -# [ -# [83.96919250488281, 28.194446860487773], -# [83.99751663208006, 28.194446860487773], -# [83.99751663208006, 28.214869548073377], -# [83.96919250488281, 28.214869548073377], -# [83.96919250488281, 28.194446860487773], -# ] -# ], -# }, -# "hdx_upload": False, -# "dataset": { -# "subnational": True, -# "dataset_title": "Pokhara", -# "dataset_prefix": "hotosm_pkr", -# "dataset_locations": ["npl"], -# }, -# "categories": [ -# { -# "Roads": { -# "hdx": { -# "tags": ["roads", "transportation", "geodata"], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["lines"], -# "select": ["name", "highway"], -# "where": "tags['highway'] IS NOT NULL", -# "formats": ["geojson"], -# } -# } -# ], -# } - -# response = client.post("/v1/custom/snapshot/", json=payload, headers=headers) - -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# def test_custom_submit_normal_custom_polygon_TM_project(): -# headers = {"access-token": access_token} -# payload = { -# "geometry": { -# "type": "Polygon", -# "coordinates": [ -# [ -# [83.96919250488281, 28.194446860487773], -# [83.99751663208006, 28.194446860487773], -# [83.99751663208006, 28.214869548073377], -# [83.96919250488281, 28.214869548073377], -# [83.96919250488281, 28.194446860487773], -# ] -# ], -# }, -# "queue": "raw_ondemand", -# "dataset": { -# "dataset_prefix": "hotosm_project_1", -# "dataset_folder": "TM", -# "dataset_title": "Tasking Manger Project 1", -# }, -# "categories": [ -# { -# "Buildings": { -# "types": ["polygons"], -# "select": [ -# "name", -# "building", -# "building:levels", -# "building:materials", -# "addr:full", -# "addr:housenumber", -# "addr:street", -# "addr:city", -# "office", -# "source", -# ], -# "where": "tags['building'] IS NOT NULL", -# "formats": ["geojson", "shp", "kml"], -# }, -# "Roads": { -# "types": ["lines"], -# "select": [ -# "name", -# "highway", -# "surface", -# "smoothness", -# "width", -# "lanes", -# "oneway", -# "bridge", -# "layer", -# "source", -# ], -# "where": "tags['highway'] IS NOT NULL", -# "formats": ["geojson", "shp", "kml"], -# }, -# "Waterways": { -# "types": ["lines", "polygons"], -# "select": [ -# "name", -# "waterway", -# "covered", -# "width", -# "depth", -# "layer", -# "blockage", -# "tunnel", -# "natural", -# "water", -# "source", -# ], -# "where": "tags['waterway'] IS NOT NULL OR tags['water'] IS NOT NULL OR tags['natural'] IN ('water','wetland','bay')", -# "formats": ["geojson", "shp", "kml"], -# }, -# "Landuse": { -# "types": ["points", "polygons"], -# "select": ["name", "amenity", "landuse", "leisure"], -# "where": "tags['landuse'] IS NOT NULL", -# "formats": ["geojson", "shp", "kml"], -# }, -# } -# ], -# } - -# response = client.post("/v1/custom/snapshot/", json=payload, headers=headers) - -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# def test_hdx_submit_normal_custom_polygon_upload(): -# headers = {"access-token": access_token} -# payload = { -# "geometry": { -# "type": "Polygon", -# "coordinates": [ -# [ -# [83.96919250488281, 28.194446860487773], -# [83.99751663208006, 28.194446860487773], -# [83.99751663208006, 28.214869548073377], -# [83.96919250488281, 28.214869548073377], -# [83.96919250488281, 28.194446860487773], -# ] -# ], -# }, -# "hdx_upload": True, -# "dataset": { -# "subnational": True, -# "dataset_title": "Pokhara", -# "dataset_folder": "Test", -# "dataset_prefix": "hotosm_pkr", -# "dataset_locations": ["npl"], -# }, -# "categories": [ -# { -# "Roads": { -# "hdx": { -# "tags": ["roads", "transportation", "geodata"], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["lines"], -# "select": ["name", "highway"], -# "where": "tags['highway'] IS NOT NULL", -# "formats": ["geojson"], -# } -# } -# ], -# } - -# response = client.post("/v1/custom/snapshot/", json=payload, headers=headers) - -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# def test_full_hdx_set_iso(): -# headers = {"access-token": access_token} -# payload = { -# "iso3": "NPL", -# "hdx_upload": False, -# "categories": [ -# { -# "Buildings": { -# "hdx": { -# "tags": [ -# "facilities-infrastructure", -# "geodata", -# ], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["polygons"], -# "select": [ -# "name", -# "building", -# "building:levels", -# "building:materials", -# "addr:full", -# "addr:housenumber", -# "addr:street", -# "addr:city", -# "office", -# "source", -# ], -# "where": "tags['building'] IS NOT NULL", -# "formats": ["geojson"], -# } -# }, -# { -# "Roads": { -# "hdx": { -# "tags": ["transportation", "geodata"], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["lines"], -# "select": [ -# "name", -# "highway", -# "surface", -# "smoothness", -# "width", -# "lanes", -# "oneway", -# "bridge", -# "layer", -# "source", -# ], -# "where": "tags['highway'] IS NOT NULL", -# "formats": ["geojson"], -# } -# }, -# { -# "Waterways": { -# "hdx": { -# "tags": ["hydrology", "geodata"], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["lines", "polygons"], -# "select": [ -# "name", -# "waterway", -# "covered", -# "width", -# "depth", -# "layer", -# "blockage", -# "tunnel", -# "natural", -# "water", -# "source", -# ], -# "where": "tags['waterway'] IS NOT NULL OR tags['water'] IS NOT NULL OR tags['natural'] IN ('water','wetland','bay')", -# "formats": ["geojson"], -# } -# }, -# { -# "Points of Interest": { -# "hdx": { -# "tags": [ -# "facilities-infrastructure", -# "points of interest-poi", -# "geodata", -# ], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["points", "polygons"], -# "select": [ -# "name", -# "amenity", -# "man_made", -# "shop", -# "tourism", -# "opening_hours", -# "beds", -# "rooms", -# "addr:full", -# "addr:housenumber", -# "addr:street", -# "addr:city", -# "source", -# ], -# "where": "tags['amenity'] IS NOT NULL OR tags['man_made'] IS NOT NULL OR tags['shop'] IS NOT NULL OR tags['tourism'] IS NOT NULL", -# "formats": ["geojson"], -# } -# }, -# { -# "Airports": { -# "hdx": { -# "tags": [ -# "aviation", -# "facilities-infrastructure", -# "geodata", -# ], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["points", "lines", "polygons"], -# "select": [ -# "name", -# "aeroway", -# "building", -# "emergency", -# "emergency:helipad", -# "operator:type", -# "capacity:persons", -# "addr:full", -# "addr:city", -# "source", -# ], -# "where": "tags['aeroway'] IS NOT NULL OR tags['building'] = 'aerodrome' OR tags['emergency:helipad'] IS NOT NULL OR tags['emergency'] = 'landing_site'", -# "formats": ["geojson"], -# } -# }, -# { -# "Sea Ports": { -# "hdx": { -# "tags": [ -# "facilities-infrastructure", -# "geodata", -# ], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["points", "lines", "polygons"], -# "select": [ -# "name", -# "amenity", -# "building", -# "port", -# "operator:type", -# "addr:full", -# "addr:city", -# "source", -# ], -# "where": "tags['amenity'] = 'ferry_terminal' OR tags['building'] = 'ferry_terminal' OR tags['port'] IS NOT NULL", -# "formats": ["geojson"], -# } -# }, -# { -# "Education Facilities": { -# "hdx": { -# "tags": [ -# "education facilities-schools", -# "geodata", -# ], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["points", "polygons"], -# "select": [ -# "name", -# "amenity", -# "building", -# "operator:type", -# "capacity:persons", -# "addr:full", -# "addr:city", -# "source", -# ], -# "where": "tags['amenity'] IN ('kindergarten', 'school', 'college', 'university') OR tags['building'] IN ('kindergarten', 'school', 'college', 'university')", -# "formats": ["geojson"], -# } -# }, -# { -# "Health Facilities": { -# "hdx": { -# "tags": ["geodata"], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["points", "polygons"], -# "select": [ -# "name", -# "amenity", -# "building", -# "healthcare", -# "healthcare:speciality", -# "operator:type", -# "capacity:persons", -# "addr:full", -# "addr:city", -# "source", -# ], -# "where": "tags['healthcare'] IS NOT NULL OR tags['amenity'] IN ('doctors', 'dentist', 'clinic', 'hospital', 'pharmacy')", -# "formats": ["geojson"], -# } -# }, -# { -# "Populated Places": { -# "hdx": { -# "tags": [ -# "populated places-settlements", -# "geodata", -# ], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["points"], -# "select": [ -# "name", -# "place", -# "population", -# "is_in", -# "source", -# ], -# "where": "tags['place'] IN ('isolated_dwelling', 'town', 'village', 'hamlet', 'city')", -# "formats": ["geojson"], -# } -# }, -# { -# "Financial Services": { -# "hdx": { -# "tags": ["economics", "geodata"], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["points", "polygons"], -# "select": [ -# "name", -# "amenity", -# "operator", -# "network", -# "addr:full", -# "addr:city", -# "source", -# ], -# "where": "tags['amenity'] IN ('mobile_money_agent','bureau_de_change','bank','microfinance','atm','sacco','money_transfer','post_office')", -# "formats": ["geojson"], -# } -# }, -# { -# "Railways": { -# "hdx": { -# "tags": [ -# "facilities-infrastructure", -# "railways", -# "transportation", -# "geodata", -# ], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["lines"], -# "select": [ -# "name", -# "railway", -# "ele", -# "operator:type", -# "layer", -# "addr:full", -# "addr:city", -# "source", -# ], -# "where": "tags['railway'] IN ('rail','station')", -# "formats": ["geojson"], -# } -# }, -# ], -# } - -# response = client.post("/v1/custom/snapshot/", json=payload, headers=headers) - -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# wait_for_task_completion(track_link) - - -# ## Tasks connection - - -# def test_worker_connection(): -# response = client.get("/v1/tasks/ping/") -# assert response.status_code == 200 +def test_status(): + response = client.get("/v1/status/") + assert response.status_code == 200 + + +# Login +def test_login_url(): + response = client.get("/v1/auth/login/") + assert response.status_code == 200 + + +def test_login_auth_me(): + headers = {"access-token": access_token} + response = client.get("/v1/auth/me/", headers=headers) + assert response.status_code == 200 + + +## Countries + + +def test_countries_endpoint(): + response = client.get("/v1/countries/?q=nepal") + assert response.status_code == 200 + + +## test osm_id + + +def test_osm_id_endpoint(): + response = client.get("/v1/osm_id/?osm_id=421498318") + assert response.status_code == 200 + + +## Snapshot +def test_snapshot(): + response = client.post( + "/v1/snapshot/", + json={ + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + } + }, + ) + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_snapshot_featurecollection(): + response = client.post( + "/v1/snapshot/", + json={ + "geometry": { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { + "coordinates": [ + [ + [83.97346137271688, 28.217525272345284], + [83.97346137271688, 28.192595937414737], + [84.01473909818759, 28.192595937414737], + [84.01473909818759, 28.217525272345284], + [83.97346137271688, 28.217525272345284], + ] + ], + "type": "Polygon", + }, + } + ], + } + }, + ) + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_snapshot_feature(): + response = client.post( + "/v1/snapshot/", + json={ + "geometry": { + "type": "Feature", + "properties": {}, + "geometry": { + "coordinates": [ + [ + [83.97346137271688, 28.217525272345284], + [83.97346137271688, 28.192595937414737], + [84.01473909818759, 28.192595937414737], + [84.01473909818759, 28.217525272345284], + [83.97346137271688, 28.217525272345284], + ] + ], + "type": "Polygon", + }, + } + }, + ) + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_snapshot_feature_fgb(): + response = client.post( + "/v1/snapshot/", + json={ + "outputType": "fgb", + "geometry": { + "type": "Feature", + "properties": {}, + "geometry": { + "coordinates": [ + [ + [83.97346137271688, 28.217525272345284], + [83.97346137271688, 28.192595937414737], + [84.01473909818759, 28.192595937414737], + [84.01473909818759, 28.217525272345284], + [83.97346137271688, 28.217525272345284], + ] + ], + "type": "Polygon", + }, + }, + }, + ) + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_snapshot_feature_fgb_wrap_geom(): + response = client.post( + "/v1/snapshot/", + json={ + "fgbWrapGeoms": True, + "outputType": "fgb", + "geometry": { + "type": "Feature", + "properties": {}, + "geometry": { + "coordinates": [ + [ + [83.97346137271688, 28.217525272345284], + [83.97346137271688, 28.192595937414737], + [84.01473909818759, 28.192595937414737], + [84.01473909818759, 28.217525272345284], + [83.97346137271688, 28.217525272345284], + ] + ], + "type": "Polygon", + }, + }, + }, + ) + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_snapshot_feature_shp(): + response = client.post( + "/v1/snapshot/", + json={ + "outputType": "shp", + "geometry": { + "type": "Feature", + "properties": {}, + "geometry": { + "coordinates": [ + [ + [83.97346137271688, 28.217525272345284], + [83.97346137271688, 28.192595937414737], + [84.01473909818759, 28.192595937414737], + [84.01473909818759, 28.217525272345284], + [83.97346137271688, 28.217525272345284], + ] + ], + "type": "Polygon", + }, + }, + }, + ) + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_snapshot_feature_gpkg(): + response = client.post( + "/v1/snapshot/", + json={ + "outputType": "gpkg", + "geometry": { + "type": "Feature", + "properties": {}, + "geometry": { + "coordinates": [ + [ + [83.97346137271688, 28.217525272345284], + [83.97346137271688, 28.192595937414737], + [84.01473909818759, 28.192595937414737], + [84.01473909818759, 28.217525272345284], + [83.97346137271688, 28.217525272345284], + ] + ], + "type": "Polygon", + }, + }, + }, + ) + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_snapshot_feature_kml(): + response = client.post( + "/v1/snapshot/", + json={ + "outputType": "kml", + "geometry": { + "type": "Feature", + "properties": {}, + "geometry": { + "coordinates": [ + [ + [83.97346137271688, 28.217525272345284], + [83.97346137271688, 28.192595937414737], + [84.01473909818759, 28.192595937414737], + [84.01473909818759, 28.217525272345284], + [83.97346137271688, 28.217525272345284], + ] + ], + "type": "Polygon", + }, + }, + }, + ) + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_snapshot_feature_sql(): + response = client.post( + "/v1/snapshot/", + json={ + "outputType": "sql", + "geometry": { + "type": "Feature", + "properties": {}, + "geometry": { + "coordinates": [ + [ + [83.97346137271688, 28.217525272345284], + [83.97346137271688, 28.192595937414737], + [84.01473909818759, 28.192595937414737], + [84.01473909818759, 28.217525272345284], + [83.97346137271688, 28.217525272345284], + ] + ], + "type": "Polygon", + }, + }, + }, + ) + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_snapshot_feature_csv(): + response = client.post( + "/v1/snapshot/", + json={ + "outputType": "csv", + "geometry": { + "type": "Feature", + "properties": {}, + "geometry": { + "coordinates": [ + [ + [83.97346137271688, 28.217525272345284], + [83.97346137271688, 28.192595937414737], + [84.01473909818759, 28.192595937414737], + [84.01473909818759, 28.217525272345284], + [83.97346137271688, 28.217525272345284], + ] + ], + "type": "Polygon", + }, + }, + }, + ) + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_snapshot_centroid(): + response = client.post( + "/v1/snapshot/", + json={ + "centroid": True, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + }, + }, + ) + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_snapshot_filters(): + response = client.post( + "/v1/snapshot/", + json={ + "fileName": "Example export with all features", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [83.585701, 28.046607], + [83.585701, 28.382561], + [84.391823, 28.382561], + [84.391823, 28.046607], + [83.585701, 28.046607], + ] + ], + }, + "outputType": "geojson", + "geometryType": ["point", "line", "polygon"], + "filters": { + "tags": { + "point": { + "join_or": { + "amenity": [ + "bank", + "ferry_terminal", + "bus_station", + "fuel", + "kindergarten", + "school", + "college", + "university", + "place_of_worship", + "marketplace", + "clinic", + "hospital", + "police", + "fire_station", + ], + "building": [ + "bank", + "aerodrome", + "ferry_terminal", + "train_station", + "bus_station", + "pumping_station", + "power_substation", + "kindergarten", + "school", + "college", + "university", + "mosque ", + " church ", + " temple", + "supermarket", + "marketplace", + "clinic", + "hospital", + "police", + "fire_station", + "stadium ", + " sports_centre", + "governor_office ", + " townhall ", + " subdistrict_office ", + " village_office ", + " community_group_office", + "government_office", + ], + "man_made": ["tower", "water_tower", "pumping_station"], + "tower:type": ["communication"], + "aeroway": ["aerodrome"], + "railway": ["station"], + "emergency": ["fire_hydrant"], + "landuse": ["reservoir", "recreation_gound"], + "waterway": ["floodgate"], + "natural": ["spring"], + "power": ["tower", "substation"], + "shop": ["supermarket"], + "leisure": [ + "stadium ", + " sports_centre ", + " pitch ", + " swimming_pool", + "park", + ], + "office": ["government"], + } + }, + "line": { + "join_or": { + "highway": [ + "motorway ", + " trunk ", + " primary ", + " secondary ", + " tertiary ", + " service ", + " residential ", + " pedestrian ", + " path ", + " living_street ", + " track", + ], + "railway": ["rail"], + "man_made": ["embankment"], + "waterway": [], + } + }, + "polygon": { + "join_or": { + "amenity": [ + "bank", + "ferry_terminal", + "bus_station", + "fuel", + "kindergarten", + "school", + "college", + "university", + "place_of_worship", + "marketplace", + "clinic", + "hospital", + "police", + "fire_station", + ], + "building": [ + "bank", + "aerodrome", + "ferry_terminal", + "train_station", + "bus_station", + "pumping_station", + "power_substation", + "power_plant", + "kindergarten", + "school", + "college", + "university", + "mosque ", + " church ", + " temple", + "supermarket", + "marketplace", + "clinic", + "hospital", + "police", + "fire_station", + "stadium ", + " sports_centre", + "governor_office ", + " townhall ", + " subdistrict_office ", + " village_office ", + " community_group_office", + "government_office", + ], + "man_made": ["tower", "water_tower", "pumping_station"], + "tower:type": ["communication"], + "aeroway": ["aerodrome"], + "railway": ["station"], + "landuse": ["reservoir", "recreation_gound"], + "waterway": [], + "natural": ["spring"], + "power": ["substation", "plant"], + "shop": ["supermarket"], + "leisure": [ + "stadium ", + " sports_centre ", + " pitch ", + " swimming_pool", + "park", + ], + "office": ["government"], + "type": ["boundary"], + "boundary": ["administrative"], + }, + }, + }, + "attributes": { + "point": [ + "building", + "ground_floor:height", + "capacity:persons", + "building:structure", + "building:condition", + "name", + "admin_level", + "building:material", + "office", + "building:roof", + "backup_generator", + "access:roof", + "building:levels", + "building:floor", + "addr:full", + "addr:city", + "source", + ], + "line": ["width", "source", "waterway", "name"], + "polygon": [ + "landslide_prone", + "name", + "admin_level", + "type", + "is_in:town", + "flood_prone", + "is_in:province", + "is_in:city", + "is_in:municipality", + "is_in:RW", + "is_in:village", + "source", + "boundary", + ], + }, + }, + }, + ) + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_snapshot_and_filter(): + response = client.post( + "/v1/snapshot/", + json={ + "fileName": "Destroyed_Buildings_Turkey", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [36.70588085657477, 37.1979648807274], + [36.70588085657477, 37.1651408422983], + [36.759267544807194, 37.1651408422983], + [36.759267544807194, 37.1979648807274], + [36.70588085657477, 37.1979648807274], + ] + ], + }, + "outputType": "geojson", + "geometryType": ["polygon"], + "filters": { + "tags": { + "point": {}, + "line": {}, + "polygon": { + "join_or": {}, + "join_and": { + "destroyed:building": ["yes"], + "damage:date": ["2023-02-06"], + }, + }, + }, + "attributes": { + "point": [], + "line": [], + "polygon": [ + "building", + "destroyed:building", + "damage:date", + "name", + "source", + ], + }, + }, + }, + ) + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_snapshot_authentication_uuid(): + headers = {"access-token": access_token} + payload = { + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + }, + "uuid": False, + } + + response = client.post("/v1/snapshot/", json=payload, headers=headers) + + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_snapshot_bind_zip(): + headers = {"access-token": access_token} + payload = { + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + }, + "bindZip": False, + } + + response = client.post("/v1/snapshot/", json=payload, headers=headers) + + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +## Snapshot Plain + + +def test_snapshot_plain(): + response = client.post( + "/v1/snapshot/plain/", + json={ + "type": "Feature", + "properties": {}, + "geometry": { + "coordinates": [ + [ + [83.98223911755514, 28.21348731781771], + [83.98223911755514, 28.208639873158987], + [83.9881146731152, 28.208639873158987], + [83.9881146731152, 28.21348731781771], + [83.98223911755514, 28.21348731781771], + ] + ], + "type": "Polygon", + }, + }, + ) + assert response.status_code == 200 + + +## Stats + + +def test_stats_endpoint_custom_polygon(): + headers = {"access-token": access_token} + payload = { + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + } + } + + response = client.post("/v1/stats/polygon/", json=payload, headers=headers) + + assert response.status_code == 200 + res = response.json() + assert ( + res["meta"]["indicators"] + == "https://github.com/hotosm/raw-data-api/tree/develop/docs/src/stats/indicators.md" + ) + + +def test_stats_endpoint_iso3(): + headers = {"access-token": access_token} + payload = {"iso3": "npl"} + + response = client.post("/v1/stats/polygon/", json=payload, headers=headers) + + assert response.status_code == 200 + res = response.json() + assert ( + res["meta"]["indicators"] + == "https://github.com/hotosm/raw-data-api/tree/develop/docs/src/stats/indicators.md" + ) + + +# HDX + + +def test_hdx_submit_normal_iso3(): + headers = {"access-token": access_token} + payload = { + "iso3": "NPL", + "hdx_upload": False, + "categories": [ + { + "Roads": { + "hdx": { + "tags": ["roads", "transportation", "geodata"], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["lines"], + "select": ["name", "highway"], + "where": "tags['highway'] IS NOT NULL", + "formats": ["geojson"], + } + } + ], + } + + response = client.post("/v1/custom/snapshot/", json=payload, headers=headers) + + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_hdx_submit_normal_iso3_multiple_format(): + headers = {"access-token": access_token} + payload = { + "iso3": "NPL", + "hdx_upload": False, + "categories": [ + { + "Roads": { + "hdx": { + "tags": ["roads", "transportation", "geodata"], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["lines"], + "select": ["name", "highway"], + "where": "tags['highway'] IS NOT NULL", + "formats": ["geojson", "gpkg", "kml", "shp"], + } + } + ], + } + + response = client.post("/v1/custom/snapshot/", json=payload, headers=headers) + + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_hdx_submit_normal_custom_polygon(): + headers = {"access-token": access_token} + payload = { + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + }, + "hdx_upload": False, + "dataset": { + "subnational": True, + "dataset_title": "Pokhara", + "dataset_prefix": "hotosm_pkr", + "dataset_locations": ["npl"], + }, + "categories": [ + { + "Roads": { + "hdx": { + "tags": ["roads", "transportation", "geodata"], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["lines"], + "select": ["name", "highway"], + "where": "tags['highway'] IS NOT NULL", + "formats": ["geojson"], + } + } + ], + } + + response = client.post("/v1/custom/snapshot/", json=payload, headers=headers) + + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_custom_submit_normal_custom_polygon_TM_project(): + headers = {"access-token": access_token} + payload = { + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + }, + "queue": "raw_ondemand", + "dataset": { + "dataset_prefix": "hotosm_project_1", + "dataset_folder": "TM", + "dataset_title": "Tasking Manger Project 1", + }, + "categories": [ + { + "Buildings": { + "types": ["polygons"], + "select": [ + "name", + "building", + "building:levels", + "building:materials", + "addr:full", + "addr:housenumber", + "addr:street", + "addr:city", + "office", + "source", + ], + "where": "tags['building'] IS NOT NULL", + "formats": ["geojson", "shp", "kml"], + }, + "Roads": { + "types": ["lines"], + "select": [ + "name", + "highway", + "surface", + "smoothness", + "width", + "lanes", + "oneway", + "bridge", + "layer", + "source", + ], + "where": "tags['highway'] IS NOT NULL", + "formats": ["geojson", "shp", "kml"], + }, + "Waterways": { + "types": ["lines", "polygons"], + "select": [ + "name", + "waterway", + "covered", + "width", + "depth", + "layer", + "blockage", + "tunnel", + "natural", + "water", + "source", + ], + "where": "tags['waterway'] IS NOT NULL OR tags['water'] IS NOT NULL OR tags['natural'] IN ('water','wetland','bay')", + "formats": ["geojson", "shp", "kml"], + }, + "Landuse": { + "types": ["points", "polygons"], + "select": ["name", "amenity", "landuse", "leisure"], + "where": "tags['landuse'] IS NOT NULL", + "formats": ["geojson", "shp", "kml"], + }, + } + ], + } + + response = client.post("/v1/custom/snapshot/", json=payload, headers=headers) + + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_hdx_submit_normal_custom_polygon_upload(): + headers = {"access-token": access_token} + payload = { + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + }, + "hdx_upload": True, + "dataset": { + "subnational": True, + "dataset_title": "Pokhara", + "dataset_folder": "Test", + "dataset_prefix": "hotosm_pkr", + "dataset_locations": ["npl"], + }, + "categories": [ + { + "Roads": { + "hdx": { + "tags": ["roads", "transportation", "geodata"], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["lines"], + "select": ["name", "highway"], + "where": "tags['highway'] IS NOT NULL", + "formats": ["geojson"], + } + } + ], + } + + response = client.post("/v1/custom/snapshot/", json=payload, headers=headers) + + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +def test_full_hdx_set_iso(): + headers = {"access-token": access_token} + payload = { + "iso3": "NPL", + "hdx_upload": False, + "categories": [ + { + "Buildings": { + "hdx": { + "tags": [ + "facilities-infrastructure", + "geodata", + ], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["polygons"], + "select": [ + "name", + "building", + "building:levels", + "building:materials", + "addr:full", + "addr:housenumber", + "addr:street", + "addr:city", + "office", + "source", + ], + "where": "tags['building'] IS NOT NULL", + "formats": ["geojson"], + } + }, + { + "Roads": { + "hdx": { + "tags": ["transportation", "geodata"], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["lines"], + "select": [ + "name", + "highway", + "surface", + "smoothness", + "width", + "lanes", + "oneway", + "bridge", + "layer", + "source", + ], + "where": "tags['highway'] IS NOT NULL", + "formats": ["geojson"], + } + }, + { + "Waterways": { + "hdx": { + "tags": ["hydrology", "geodata"], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["lines", "polygons"], + "select": [ + "name", + "waterway", + "covered", + "width", + "depth", + "layer", + "blockage", + "tunnel", + "natural", + "water", + "source", + ], + "where": "tags['waterway'] IS NOT NULL OR tags['water'] IS NOT NULL OR tags['natural'] IN ('water','wetland','bay')", + "formats": ["geojson"], + } + }, + { + "Points of Interest": { + "hdx": { + "tags": [ + "facilities-infrastructure", + "points of interest-poi", + "geodata", + ], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["points", "polygons"], + "select": [ + "name", + "amenity", + "man_made", + "shop", + "tourism", + "opening_hours", + "beds", + "rooms", + "addr:full", + "addr:housenumber", + "addr:street", + "addr:city", + "source", + ], + "where": "tags['amenity'] IS NOT NULL OR tags['man_made'] IS NOT NULL OR tags['shop'] IS NOT NULL OR tags['tourism'] IS NOT NULL", + "formats": ["geojson"], + } + }, + { + "Airports": { + "hdx": { + "tags": [ + "aviation", + "facilities-infrastructure", + "geodata", + ], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["points", "lines", "polygons"], + "select": [ + "name", + "aeroway", + "building", + "emergency", + "emergency:helipad", + "operator:type", + "capacity:persons", + "addr:full", + "addr:city", + "source", + ], + "where": "tags['aeroway'] IS NOT NULL OR tags['building'] = 'aerodrome' OR tags['emergency:helipad'] IS NOT NULL OR tags['emergency'] = 'landing_site'", + "formats": ["geojson"], + } + }, + { + "Sea Ports": { + "hdx": { + "tags": [ + "facilities-infrastructure", + "geodata", + ], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["points", "lines", "polygons"], + "select": [ + "name", + "amenity", + "building", + "port", + "operator:type", + "addr:full", + "addr:city", + "source", + ], + "where": "tags['amenity'] = 'ferry_terminal' OR tags['building'] = 'ferry_terminal' OR tags['port'] IS NOT NULL", + "formats": ["geojson"], + } + }, + { + "Education Facilities": { + "hdx": { + "tags": [ + "education facilities-schools", + "geodata", + ], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["points", "polygons"], + "select": [ + "name", + "amenity", + "building", + "operator:type", + "capacity:persons", + "addr:full", + "addr:city", + "source", + ], + "where": "tags['amenity'] IN ('kindergarten', 'school', 'college', 'university') OR tags['building'] IN ('kindergarten', 'school', 'college', 'university')", + "formats": ["geojson"], + } + }, + { + "Health Facilities": { + "hdx": { + "tags": ["geodata"], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["points", "polygons"], + "select": [ + "name", + "amenity", + "building", + "healthcare", + "healthcare:speciality", + "operator:type", + "capacity:persons", + "addr:full", + "addr:city", + "source", + ], + "where": "tags['healthcare'] IS NOT NULL OR tags['amenity'] IN ('doctors', 'dentist', 'clinic', 'hospital', 'pharmacy')", + "formats": ["geojson"], + } + }, + { + "Populated Places": { + "hdx": { + "tags": [ + "populated places-settlements", + "geodata", + ], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["points"], + "select": [ + "name", + "place", + "population", + "is_in", + "source", + ], + "where": "tags['place'] IN ('isolated_dwelling', 'town', 'village', 'hamlet', 'city')", + "formats": ["geojson"], + } + }, + { + "Financial Services": { + "hdx": { + "tags": ["economics", "geodata"], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["points", "polygons"], + "select": [ + "name", + "amenity", + "operator", + "network", + "addr:full", + "addr:city", + "source", + ], + "where": "tags['amenity'] IN ('mobile_money_agent','bureau_de_change','bank','microfinance','atm','sacco','money_transfer','post_office')", + "formats": ["geojson"], + } + }, + { + "Railways": { + "hdx": { + "tags": [ + "facilities-infrastructure", + "railways", + "transportation", + "geodata", + ], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["lines"], + "select": [ + "name", + "railway", + "ele", + "operator:type", + "layer", + "addr:full", + "addr:city", + "source", + ], + "where": "tags['railway'] IN ('rail','station')", + "formats": ["geojson"], + } + }, + ], + } + + response = client.post("/v1/custom/snapshot/", json=payload, headers=headers) + + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + wait_for_task_completion(track_link) + + +## Tasks connection + + +def test_worker_connection(): + response = client.get("/v1/tasks/ping/") + assert response.status_code == 200 ### Custom Snapshot Using YAML @@ -1346,9 +1346,9 @@ def test_custom_yaml_normal_fmtm_request(): formats: - geojson - fgb + - mvt select: - - name - - name:en + - '*' types: - polygons where: tags['building'] IS NOT NULL