diff --git a/.github/workflows/Unit-Test.yml b/.github/workflows/Unit-Test.yml index 6fd7ebad..be029cc9 100644 --- a/.github/workflows/Unit-Test.yml +++ b/.github/workflows/Unit-Test.yml @@ -56,6 +56,7 @@ jobs: run: | export PGPASSWORD='admin'; psql -U postgres -h localhost -p 5434 raw < tests/fixtures/pokhara.sql + psql -U postgres -h localhost -p 5434 raw < backend/sql/countries.sql - name: Install gdal run: sudo apt-get update && sudo apt-get -y install gdal-bin python3-gdal && sudo apt-get -y autoremove && sudo apt-get clean @@ -75,10 +76,14 @@ jobs: pip install -r requirements.txt pip install -e . - - name: Launch Worker + - name: Launch Default Worker run: | celery --app API.api_worker worker --loglevel=INFO --queues='raw_default' & + - name: Launch Special Worker + run: | + celery --app API.api_worker worker --loglevel=INFO --queues='raw_special' &1 + - name: Run Tests run: | py.test -v -s diff --git a/API/raw_data.py b/API/raw_data.py index 4b2b2546..d76e4d7e 100644 --- a/API/raw_data.py +++ b/API/raw_data.py @@ -467,7 +467,6 @@ def get_osm_current_snapshot_as_plain_geojson( Returns: Featurecollection: Geojson """ - print(params.geometry.model_dump_json()) area_m2 = area(json.loads(params.geometry.model_dump_json())) area_km2 = area_m2 * 1e-6 if area_km2 > 10: diff --git a/src/app.py b/src/app.py index 059b6556..cca3c269 100644 --- a/src/app.py +++ b/src/app.py @@ -1369,7 +1369,7 @@ def process_export_format(export_format): return resource if self.parallel_process_state is False and len(export_formats) > 1: - with concurrent.futures.ProcessPoolExecutor( + with concurrent.futures.ThreadPoolExecutor( max_workers=os.cpu_count() ) as executor: futures = [ @@ -1544,7 +1544,7 @@ def process_hdx_tags(self): dataset_results = [] if len(self.params.categories) > 1: self.parallel_process_state = True - with concurrent.futures.ProcessPoolExecutor( + with concurrent.futures.ThreadPoolExecutor( max_workers=os.cpu_count() ) as executor: futures = { @@ -1742,6 +1742,6 @@ def init_dataset(self): ) self.dataset.set_expected_update_frequency(self.hdx.update_frequency) for location in dataset_locations: - self.dataset.add_country_location(location) + self.dataset.add_other_location(location) for tag in self.category_data.hdx.tags: self.dataset.add_tag(tag) diff --git a/tests/test_API.py b/tests/test_API.py index 4aaf94fb..6c458bf7 100644 --- a/tests/test_API.py +++ b/tests/test_API.py @@ -47,12 +47,24 @@ def test_snapshot(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - time.sleep(15) # wait for worker to complete task - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - assert check_status == "SUCCESS" + max_attempts = 6 + interval_seconds = 10 + for attempt in range(1, max_attempts + 1): + time.sleep(interval_seconds) # wait for worker to complete task + + response = client.get(f"/v1{track_link}") + assert response.status_code == 200 + res = response.json() + check_status = res["status"] + + if check_status == "SUCCESS": + break # exit the loop if the status is SUCCESS + + if attempt == max_attempts: + # If max_attempts reached and status is not SUCCESS, raise an AssertionError + assert ( + False + ), f"Task did not complete successfully after {max_attempts} attempts" def test_snapshot_centroid(): @@ -77,12 +89,24 @@ def test_snapshot_centroid(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - time.sleep(15) # wait for worker to complete task - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - assert check_status == "SUCCESS" + max_attempts = 6 + interval_seconds = 10 + for attempt in range(1, max_attempts + 1): + time.sleep(interval_seconds) # wait for worker to complete task + + response = client.get(f"/v1{track_link}") + assert response.status_code == 200 + res = response.json() + check_status = res["status"] + + if check_status == "SUCCESS": + break # exit the loop if the status is SUCCESS + + if attempt == max_attempts: + # If max_attempts reached and status is not SUCCESS, raise an AssertionError + assert ( + False + ), f"Task did not complete successfully after {max_attempts} attempts" def test_snapshot_filters(): @@ -308,12 +332,24 @@ def test_snapshot_filters(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - time.sleep(15) # wait for worker to complete task - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - assert check_status == "SUCCESS" + max_attempts = 6 + interval_seconds = 10 + for attempt in range(1, max_attempts + 1): + time.sleep(interval_seconds) # wait for worker to complete task + + response = client.get(f"/v1{track_link}") + assert response.status_code == 200 + res = response.json() + check_status = res["status"] + + if check_status == "SUCCESS": + break # exit the loop if the status is SUCCESS + + if attempt == max_attempts: + # If max_attempts reached and status is not SUCCESS, raise an AssertionError + assert ( + False + ), f"Task did not complete successfully after {max_attempts} attempts" def test_snapshot_and_filter(): @@ -364,12 +400,24 @@ def test_snapshot_and_filter(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - time.sleep(15) # wait for worker to complete task - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - assert check_status == "SUCCESS" + max_attempts = 6 + interval_seconds = 10 + for attempt in range(1, max_attempts + 1): + time.sleep(interval_seconds) # wait for worker to complete task + + response = client.get(f"/v1{track_link}") + assert response.status_code == 200 + res = response.json() + check_status = res["status"] + + if check_status == "SUCCESS": + break # exit the loop if the status is SUCCESS + + if attempt == max_attempts: + # If max_attempts reached and status is not SUCCESS, raise an AssertionError + assert ( + False + ), f"Task did not complete successfully after {max_attempts} attempts" def test_snapshot_plain(): @@ -416,12 +464,24 @@ def test_snapshot_authentication_uuid(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - time.sleep(15) # wait for worker to complete task - response = client.get(f"/v1{track_link}") - assert response.status_code == 200 - res = response.json() - check_status = res["status"] - assert check_status == "SUCCESS" + max_attempts = 6 + interval_seconds = 10 + for attempt in range(1, max_attempts + 1): + time.sleep(interval_seconds) # wait for worker to complete task + + response = client.get(f"/v1{track_link}") + assert response.status_code == 200 + res = response.json() + check_status = res["status"] + + if check_status == "SUCCESS": + break # exit the loop if the status is SUCCESS + + if attempt == max_attempts: + # If max_attempts reached and status is not SUCCESS, raise an AssertionError + assert ( + False + ), f"Task did not complete successfully after {max_attempts} attempts" def test_snapshot_bind_zip(): @@ -447,7 +507,505 @@ def test_snapshot_bind_zip(): assert response.status_code == 200 res = response.json() track_link = res["track_link"] - time.sleep(15) # wait for worker to complete task + max_attempts = 6 + interval_seconds = 10 + for attempt in range(1, max_attempts + 1): + time.sleep(interval_seconds) # wait for worker to complete task + + response = client.get(f"/v1{track_link}") + assert response.status_code == 200 + res = response.json() + check_status = res["status"] + + if check_status == "SUCCESS": + break # exit the loop if the status is SUCCESS + + if attempt == max_attempts: + # If max_attempts reached and status is not SUCCESS, raise an AssertionError + assert ( + False + ), f"Task did not complete successfully after {max_attempts} attempts" + + +def test_stats_endpoint_custom_polygon(): + headers = {"access-token": access_token} + payload = { + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + } + } + + response = client.post("/v1/stats/polygon/", json=payload, headers=headers) + + assert response.status_code == 200 + res = response.json() + assert ( + res["meta"]["indicators"] + == "https://github.com/hotosm/raw-data-api/tree/develop/docs/src/stats/indicators.md" + ) + + +def test_stats_endpoint_iso3(): + headers = {"access-token": access_token} + payload = {"iso3": "npl"} + + response = client.post("/v1/stats/polygon/", json=payload, headers=headers) + + assert response.status_code == 200 + res = response.json() + assert ( + res["meta"]["indicators"] + == "https://github.com/hotosm/raw-data-api/tree/develop/docs/src/stats/indicators.md" + ) + + +def test_hdx_submit_normal_iso3(): + headers = {"access-token": access_token} + payload = { + "iso3": "NPL", + "hdx_upload": False, + "categories": [ + { + "Roads": { + "hdx": { + "tags": ["roads", "transportation", "geodata"], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["lines"], + "select": ["name", "highway"], + "where": "tags['highway'] IS NOT NULL", + "formats": ["geojson"], + } + } + ], + } + + response = client.post("/v1/hdx/submit/", json=payload, headers=headers) + + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + max_attempts = 6 + interval_seconds = 10 + for attempt in range(1, max_attempts + 1): + time.sleep(interval_seconds) # wait for worker to complete task + + response = client.get(f"/v1{track_link}") + assert response.status_code == 200 + res = response.json() + check_status = res["status"] + + if check_status == "SUCCESS": + break # exit the loop if the status is SUCCESS + + if attempt == max_attempts: + # If max_attempts reached and status is not SUCCESS, raise an AssertionError + assert ( + False + ), f"Task did not complete successfully after {max_attempts} attempts" + + +def test_hdx_submit_normal_iso3_multiple_format(): + headers = {"access-token": access_token} + payload = { + "iso3": "NPL", + "hdx_upload": False, + "categories": [ + { + "Roads": { + "hdx": { + "tags": ["roads", "transportation", "geodata"], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["lines"], + "select": ["name", "highway"], + "where": "tags['highway'] IS NOT NULL", + "formats": ["geojson", "gpkg", "kml", "shp"], + } + } + ], + } + + response = client.post("/v1/hdx/submit/", json=payload, headers=headers) + + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + max_attempts = 6 + interval_seconds = 10 + for attempt in range(1, max_attempts + 1): + time.sleep(interval_seconds) # wait for worker to complete task + + response = client.get(f"/v1{track_link}") + assert response.status_code == 200 + res = response.json() + check_status = res["status"] + + if check_status == "SUCCESS": + break # exit the loop if the status is SUCCESS + + if attempt == max_attempts: + # If max_attempts reached and status is not SUCCESS, raise an AssertionError + assert ( + False + ), f"Task did not complete successfully after {max_attempts} attempts" + + +def test_hdx_submit_normal_custom_polygon(): + headers = {"access-token": access_token} + payload = { + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + }, + "hdx_upload": False, + "dataset": { + "subnational": True, + "dataset_title": "Pokhara", + "dataset_prefix": "hotosm_pkr", + "dataset_locations": ["npl"], + }, + "categories": [ + { + "Roads": { + "hdx": { + "tags": ["roads", "transportation", "geodata"], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["lines"], + "select": ["name", "highway"], + "where": "tags['highway'] IS NOT NULL", + "formats": ["geojson"], + } + } + ], + } + + response = client.post("/v1/hdx/submit/", json=payload, headers=headers) + + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + max_attempts = 6 + interval_seconds = 10 + for attempt in range(1, max_attempts + 1): + time.sleep(interval_seconds) # wait for worker to complete task + + response = client.get(f"/v1{track_link}") + assert response.status_code == 200 + res = response.json() + check_status = res["status"] + + if check_status == "SUCCESS": + break # exit the loop if the status is SUCCESS + + if attempt == max_attempts: + # If max_attempts reached and status is not SUCCESS, raise an AssertionError + assert ( + False + ), f"Task did not complete successfully after {max_attempts} attempts" + + +def test_full_hdx_set_iso(): + headers = {"access-token": access_token} + payload = { + "iso3": "NPL", + "hdx_upload": False, + "categories": [ + { + "Buildings": { + "hdx": { + "tags": [ + "facilities-infrastructure", + "geodata", + ], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["polygons"], + "select": [ + "name", + "building", + "building:levels", + "building:materials", + "addr:full", + "addr:housenumber", + "addr:street", + "addr:city", + "office", + "source", + ], + "where": "tags['building'] IS NOT NULL", + "formats": ["geojson"], + } + }, + { + "Roads": { + "hdx": { + "tags": ["transportation", "geodata"], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["lines"], + "select": [ + "name", + "highway", + "surface", + "smoothness", + "width", + "lanes", + "oneway", + "bridge", + "layer", + "source", + ], + "where": "tags['highway'] IS NOT NULL", + "formats": ["geojson"], + } + }, + { + "Waterways": { + "hdx": { + "tags": ["hydrology", "geodata"], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["lines", "polygons"], + "select": [ + "name", + "waterway", + "covered", + "width", + "depth", + "layer", + "blockage", + "tunnel", + "natural", + "water", + "source", + ], + "where": "tags['waterway'] IS NOT NULL OR tags['water'] IS NOT NULL OR tags['natural'] IN ('water','wetland','bay')", + "formats": ["geojson"], + } + }, + { + "Points of Interest": { + "hdx": { + "tags": [ + "facilities-infrastructure", + "points of interest-poi", + "geodata", + ], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["points", "polygons"], + "select": [ + "name", + "amenity", + "man_made", + "shop", + "tourism", + "opening_hours", + "beds", + "rooms", + "addr:full", + "addr:housenumber", + "addr:street", + "addr:city", + "source", + ], + "where": "tags['amenity'] IS NOT NULL OR tags['man_made'] IS NOT NULL OR tags['shop'] IS NOT NULL OR tags['tourism'] IS NOT NULL", + "formats": ["geojson"], + } + }, + { + "Airports": { + "hdx": { + "tags": [ + "aviation", + "facilities-infrastructure", + "geodata", + ], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["points", "lines", "polygons"], + "select": [ + "name", + "aeroway", + "building", + "emergency", + "emergency:helipad", + "operator:type", + "capacity:persons", + "addr:full", + "addr:city", + "source", + ], + "where": "tags['aeroway'] IS NOT NULL OR tags['building'] = 'aerodrome' OR tags['emergency:helipad'] IS NOT NULL OR tags['emergency'] = 'landing_site'", + "formats": ["geojson"], + } + }, + { + "Sea Ports": { + "hdx": { + "tags": [ + "facilities-infrastructure", + "geodata", + ], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["points", "lines", "polygons"], + "select": [ + "name", + "amenity", + "building", + "port", + "operator:type", + "addr:full", + "addr:city", + "source", + ], + "where": "tags['amenity'] = 'ferry_terminal' OR tags['building'] = 'ferry_terminal' OR tags['port'] IS NOT NULL", + "formats": ["geojson"], + } + }, + { + "Education Facilities": { + "hdx": { + "tags": [ + "education facilities-schools", + "geodata", + ], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["points", "polygons"], + "select": [ + "name", + "amenity", + "building", + "operator:type", + "capacity:persons", + "addr:full", + "addr:city", + "source", + ], + "where": "tags['amenity'] IN ('kindergarten', 'school', 'college', 'university') OR building IN ('kindergarten', 'school', 'college', 'university')", + "formats": ["geojson"], + } + }, + { + "Health Facilities": { + "hdx": { + "tags": ["geodata"], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["points", "polygons"], + "select": [ + "name", + "amenity", + "building", + "healthcare", + "healthcare:speciality", + "operator:type", + "capacity:persons", + "addr:full", + "addr:city", + "source", + ], + "where": "tags['healthcare'] IS NOT NULL OR tags['amenity'] IN ('doctors', 'dentist', 'clinic', 'hospital', 'pharmacy')", + "formats": ["geojson"], + } + }, + { + "Populated Places": { + "hdx": { + "tags": [ + "populated places-settlements", + "geodata", + ], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["points"], + "select": [ + "name", + "place", + "population", + "is_in", + "source", + ], + "where": "tags['place'] IN ('isolated_dwelling', 'town', 'village', 'hamlet', 'city')", + "formats": ["geojson"], + } + }, + { + "Financial Services": { + "hdx": { + "tags": ["economics", "geodata"], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["points", "polygons"], + "select": [ + "name", + "amenity", + "operator", + "network", + "addr:full", + "addr:city", + "source", + ], + "where": "tags['amenity'] IN ('mobile_money_agent','bureau_de_change','bank','microfinance','atm','sacco','money_transfer','post_office')", + "formats": ["geojson"], + } + }, + { + "Railways": { + "hdx": { + "tags": [ + "facilities-infrastructure", + "railways", + "transportation", + "geodata", + ], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["lines"], + "select": [ + "name", + "railway", + "ele", + "operator:type", + "layer", + "addr:full", + "addr:city", + "source", + ], + "where": "tags['railway'] IN ('rail','station')", + "formats": ["geojson"], + } + }, + ], + } + + response = client.post("/v1/hdx/submit/", json=payload, headers=headers) + + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + time.sleep(60) # wait for worker to complete task response = client.get(f"/v1{track_link}") assert response.status_code == 200 res = response.json() @@ -455,450 +1013,46 @@ def test_snapshot_bind_zip(): assert check_status == "SUCCESS" -# def test_stats_endpoint_custom_polygon(): -# headers = {"access-token": access_token} -# payload = { -# "geometry": { -# "type": "Polygon", -# "coordinates": [ -# [ -# [83.96919250488281, 28.194446860487773], -# [83.99751663208006, 28.194446860487773], -# [83.99751663208006, 28.214869548073377], -# [83.96919250488281, 28.214869548073377], -# [83.96919250488281, 28.194446860487773], -# ] -# ], -# }, -# } - -# response = client.post("/v1/snapshot/stats/polygon/", json=payload, headers=headers) - -# assert response.status_code == 200 -# res = response.json() -# assert ( -# res["meta"]["indicators"] -# == "https://github.com/hotosm/raw-data-api/tree/develop/docs/src/stats/indicators.md" -# ) - - -# def test_stats_endpoint_iso3(): -# headers = {"access-token": access_token} -# payload = {"iso3": "npl"} - -# response = client.post("/v1/snapshot/stats/polygon/", json=payload, headers=headers) - -# assert response.status_code == 200 -# res = response.json() -# assert ( -# res["meta"]["indicators"] -# == "https://github.com/hotosm/raw-data-api/tree/develop/docs/src/stats/indicators.md" -# ) - - -# def test_hdx_submit_normal_iso3(): -# headers = {"access-token": access_token} -# payload = { -# "iso3": "NPL", -# "hdx_upload": False, -# "categories": [ -# { -# "Roads": { -# "hdx": { -# "tags": ["roads", "transportation", "geodata"], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["lines"], -# "select": ["name", "highway"], -# "where": "tags['highway'] IS NOT NULL", -# "formats": ["geojson"], -# } -# } -# ], -# } - -# response = client.post("/v1/snapshot/hdx/submit/", json=payload, headers=headers) - -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# time.sleep(15) # wait for worker to complete task -# response = client.get(f"/v1{track_link}") -# assert response.status_code == 200 -# res = response.json() -# check_status = res["status"] -# assert check_status == "SUCCESS" - - -# def test_hdx_submit_normal_iso3_multiple_format(): -# headers = {"access-token": access_token} -# payload = { -# "iso3": "NPL", -# "hdx_upload": False, -# "categories": [ -# { -# "Roads": { -# "hdx": { -# "tags": ["roads", "transportation", "geodata"], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["lines"], -# "select": ["name", "highway"], -# "where": "tags['highway'] IS NOT NULL", -# "formats": ["geojson", "gpkg", "kml", "shp"], -# } -# } -# ], -# } - -# response = client.post("/v1/snapshot/hdx/submit/", json=payload, headers=headers) - -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# time.sleep(20) # wait for worker to complete task -# response = client.get(f"/v1{track_link}") -# assert response.status_code == 200 -# res = response.json() -# check_status = res["status"] -# assert check_status == "SUCCESS" - - -# def test_hdx_submit_normal_custom_polygon(): -# headers = {"access-token": access_token} -# payload = { -# "geometry": { -# "type": "Polygon", -# "coordinates": [ -# [ -# [83.96919250488281, 28.194446860487773], -# [83.99751663208006, 28.194446860487773], -# [83.99751663208006, 28.214869548073377], -# [83.96919250488281, 28.214869548073377], -# [83.96919250488281, 28.194446860487773], -# ] -# ], -# }, -# "hdx_upload": False, -# "dataset": { -# "subnational": True, -# "dataset_title": "Pokhara", -# "dataset_prefix": "hotosm_pkr", -# "dataset_locations": ["npl"], -# }, -# "categories": [ -# { -# "Roads": { -# "hdx": { -# "tags": ["roads", "transportation", "geodata"], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["lines"], -# "select": ["name", "highway"], -# "where": "tags['highway'] IS NOT NULL", -# "formats": ["geojson"], -# } -# } -# ], -# } - -# response = client.post("/v1/snapshot/hdx/submit/", json=payload, headers=headers) - -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# time.sleep(20) # wait for worker to complete task -# response = client.get(f"/v1{track_link}") -# assert response.status_code == 200 -# res = response.json() -# check_status = res["status"] -# assert check_status == "SUCCESS" - - -# def test_full_hdx_set_iso(): -# headers = {"access-token": access_token} -# payload = { -# "iso3": "NPL", -# "categories": [ -# { -# "Buildings": { -# "hdx": { -# "tags": [ -# "facilities-infrastructure", -# "geodata", -# ], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["polygons"], -# "select": [ -# "name", -# "building", -# "building:levels", -# "building:materials", -# "addr:full", -# "addr:housenumber", -# "addr:street", -# "addr:city", -# "office", -# "source", -# ], -# "where": "tags['building'] IS NOT NULL", -# "formats": ["geojson"], -# } -# }, -# { -# "Roads": { -# "hdx": { -# "tags": ["transportation", "geodata"], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["lines"], -# "select": [ -# "name", -# "highway", -# "surface", -# "smoothness", -# "width", -# "lanes", -# "oneway", -# "bridge", -# "layer", -# "source", -# ], -# "where": "tags['highway'] IS NOT NULL", -# "formats": ["geojson"], -# } -# }, -# { -# "Waterways": { -# "hdx": { -# "tags": ["hydrology", "geodata"], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["lines", "polygons"], -# "select": [ -# "name", -# "waterway", -# "covered", -# "width", -# "depth", -# "layer", -# "blockage", -# "tunnel", -# "natural", -# "water", -# "source", -# ], -# "where": "tags['waterway'] IS NOT NULL OR tags['water'] IS NOT NULL OR tags['natural'] IN ('water','wetland','bay')", -# "formats": ["geojson"], -# } -# }, -# { -# "Points of Interest": { -# "hdx": { -# "tags": [ -# "facilities-infrastructure", -# "points of interest-poi", -# "geodata", -# ], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["points", "polygons"], -# "select": [ -# "name", -# "amenity", -# "man_made", -# "shop", -# "tourism", -# "opening_hours", -# "beds", -# "rooms", -# "addr:full", -# "addr:housenumber", -# "addr:street", -# "addr:city", -# "source", -# ], -# "where": "tags['amenity'] IS NOT NULL OR tags['man_made'] IS NOT NULL OR tags['shop'] IS NOT NULL OR tags['tourism'] IS NOT NULL", -# "formats": ["geojson"], -# } -# }, -# { -# "Airports": { -# "hdx": { -# "tags": [ -# "aviation", -# "facilities-infrastructure", -# "geodata", -# ], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["points", "lines", "polygons"], -# "select": [ -# "name", -# "aeroway", -# "building", -# "emergency", -# "emergency:helipad", -# "operator:type", -# "capacity:persons", -# "addr:full", -# "addr:city", -# "source", -# ], -# "where": "tags['aeroway'] IS NOT NULL OR tags['building'] = 'aerodrome' OR tags['emergency:helipad'] IS NOT NULL OR tags['emergency'] = 'landing_site'", -# "formats": ["geojson"], -# } -# }, -# { -# "Sea Ports": { -# "hdx": { -# "tags": [ -# "facilities-infrastructure", -# "geodata", -# ], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["points", "lines", "polygons"], -# "select": [ -# "name", -# "amenity", -# "building", -# "port", -# "operator:type", -# "addr:full", -# "addr:city", -# "source", -# ], -# "where": "tags['amenity'] = 'ferry_terminal' OR tags['building'] = 'ferry_terminal' OR tags['port'] IS NOT NULL", -# "formats": ["geojson"], -# } -# }, -# { -# "Education Facilities": { -# "hdx": { -# "tags": [ -# "education facilities-schools", -# "geodata", -# ], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["points", "polygons"], -# "select": [ -# "name", -# "amenity", -# "building", -# "operator:type", -# "capacity:persons", -# "addr:full", -# "addr:city", -# "source", -# ], -# "where": "tags['amenity'] IN ('kindergarten', 'school', 'college', 'university') OR building IN ('kindergarten', 'school', 'college', 'university')", -# "formats": ["geojson"], -# } -# }, -# { -# "Health Facilities": { -# "hdx": { -# "tags": ["geodata"], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["points", "polygons"], -# "select": [ -# "name", -# "amenity", -# "building", -# "healthcare", -# "healthcare:speciality", -# "operator:type", -# "capacity:persons", -# "addr:full", -# "addr:city", -# "source", -# ], -# "where": "tags['healthcare'] IS NOT NULL OR tags['amenity'] IN ('doctors', 'dentist', 'clinic', 'hospital', 'pharmacy')", -# "formats": ["geojson"], -# } -# }, -# { -# "Populated Places": { -# "hdx": { -# "tags": [ -# "populated places-settlements", -# "geodata", -# ], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["points"], -# "select": [ -# "name", -# "place", -# "population", -# "is_in", -# "source", -# ], -# "where": "tags['place'] IN ('isolated_dwelling', 'town', 'village', 'hamlet', 'city')", -# "formats": ["geojson"], -# } -# }, -# { -# "Financial Services": { -# "hdx": { -# "tags": ["economics", "geodata"], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["points", "polygons"], -# "select": [ -# "name", -# "amenity", -# "operator", -# "network", -# "addr:full", -# "addr:city", -# "source", -# ], -# "where": "tags['amenity'] IN ('mobile_money_agent','bureau_de_change','bank','microfinance','atm','sacco','money_transfer','post_office')", -# "formats": ["geojson"], -# } -# }, -# { -# "Railways": { -# "hdx": { -# "tags": [ -# "facilities-infrastructure", -# "railways", -# "transportation", -# "geodata", -# ], -# "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", -# }, -# "types": ["lines"], -# "select": [ -# "name", -# "railway", -# "ele", -# "operator:type", -# "layer", -# "addr:full", -# "addr:city", -# "source", -# ], -# "where": "tags['railway'] IN ('rail','station')", -# "formats": ["geojson"], -# } -# }, -# ], -# } - -# response = client.post("/v1/snapshot/hdx/submit/", json=payload, headers=headers) - -# assert response.status_code == 200 -# res = response.json() -# track_link = res["track_link"] -# time.sleep(60) # wait for worker to complete task -# response = client.get(f"/v1{track_link}") -# assert response.status_code == 200 -# res = response.json() -# check_status = res["status"] -# assert check_status == "SUCCESS" +def test_hdx_submit_normal_iso3_upload_option(): + headers = {"access-token": access_token} + payload = { + "iso3": "NPL", + "categories": [ + { + "Roads": { + "hdx": { + "tags": ["roads", "transportation", "geodata"], + "caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive", + }, + "types": ["lines"], + "select": ["name", "highway"], + "where": "tags['highway'] IS NOT NULL", + "formats": ["geojson"], + } + } + ], + } + + response = client.post("/v1/hdx/submit/", json=payload, headers=headers) + + assert response.status_code == 200 + res = response.json() + track_link = res["track_link"] + max_attempts = 6 + interval_seconds = 10 + for attempt in range(1, max_attempts + 1): + time.sleep(interval_seconds) # wait for worker to complete task + + response = client.get(f"/v1{track_link}") + assert response.status_code == 200 + res = response.json() + check_status = res["status"] + + if check_status == "SUCCESS": + break # exit the loop if the status is SUCCESS + + if attempt == max_attempts: + # If max_attempts reached and status is not SUCCESS, raise an AssertionError + assert ( + False + ), f"Task did not complete successfully after {max_attempts} attempts"