Skip to content

Commit

Permalink
Improved logs and example
Browse files Browse the repository at this point in the history
  • Loading branch information
@kshitijrajsharma
kshitijrajsharma committed Dec 24, 2023
1 parent d23adfb commit 5268cc5
Show file tree
Hide file tree
Showing 2 changed files with 281 additions and 2 deletions.
277 changes: 277 additions & 0 deletions API/hdx.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -401,6 +401,283 @@ async def process_hdx_requests(
],
},
},
"fullset_multiple_formats": {
"summary": "Full HDX Dataset default Multiple formats",
"description": "Full yaml conversion for dataset with iso3 example with multiple formats",
"value": {
"iso3": "NPL",
"categories": [
{
"Buildings": {
"hdx": {
"tags": [
"facilities-infrastructure",
"geodata",
],
"caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive",
},
"types": ["polygons"],
"select": [
"name",
"building",
"building:levels",
"building:materials",
"addr:full",
"addr:housenumber",
"addr:street",
"addr:city",
"office",
"source",
],
"where": "tags['building'] IS NOT NULL",
"formats": ["geojson", "shp", "kml"],
}
},
{
"Roads": {
"hdx": {
"tags": ["transportation", "geodata"],
"caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive",
},
"types": ["lines"],
"select": [
"name",
"highway",
"surface",
"smoothness",
"width",
"lanes",
"oneway",
"bridge",
"layer",
"source",
],
"where": "tags['highway'] IS NOT NULL",
"formats": ["geojson", "shp", "kml"],
}
},
{
"Waterways": {
"hdx": {
"tags": ["hydrology", "geodata"],
"caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive",
},
"types": ["lines", "polygons"],
"select": [
"name",
"waterway",
"covered",
"width",
"depth",
"layer",
"blockage",
"tunnel",
"natural",
"water",
"source",
],
"where": "tags['waterway'] IS NOT NULL OR tags['water'] IS NOT NULL OR tags['natural'] IN ('water','wetland','bay')",
"formats": ["geojson", "shp", "kml"],
}
},
{
"Points of Interest": {
"hdx": {
"tags": [
"facilities-infrastructure",
"points of interest-poi",
"geodata",
],
"caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive",
},
"types": ["points", "polygons"],
"select": [
"name",
"amenity",
"man_made",
"shop",
"tourism",
"opening_hours",
"beds",
"rooms",
"addr:full",
"addr:housenumber",
"addr:street",
"addr:city",
"source",
],
"where": "tags['amenity'] IS NOT NULL OR tags['man_made'] IS NOT NULL OR tags['shop'] IS NOT NULL OR tags['tourism'] IS NOT NULL",
"formats": ["geojson", "shp", "kml"],
}
},
{
"Airports": {
"hdx": {
"tags": [
"aviation",
"facilities-infrastructure",
"geodata",
],
"caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive",
},
"types": ["points", "lines", "polygons"],
"select": [
"name",
"aeroway",
"building",
"emergency",
"emergency:helipad",
"operator:type",
"capacity:persons",
"addr:full",
"addr:city",
"source",
],
"where": "tags['aeroway'] IS NOT NULL OR tags['building'] = 'aerodrome' OR tags['emergency:helipad'] IS NOT NULL OR tags['emergency'] = 'landing_site'",
"formats": ["geojson", "shp", "kml"],
}
},
{
"Sea Ports": {
"hdx": {
"tags": [
"facilities-infrastructure",
"geodata",
],
"caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive",
},
"types": ["points", "lines", "polygons"],
"select": [
"name",
"amenity",
"building",
"port",
"operator:type",
"addr:full",
"addr:city",
"source",
],
"where": "tags['amenity'] = 'ferry_terminal' OR tags['building'] = 'ferry_terminal' OR tags['port'] IS NOT NULL",
"formats": ["geojson", "shp", "kml"],
}
},
{
"Education Facilities": {
"hdx": {
"tags": [
"education facilities-schools",
"geodata",
],
"caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive",
},
"types": ["points", "polygons"],
"select": [
"name",
"amenity",
"building",
"operator:type",
"capacity:persons",
"addr:full",
"addr:city",
"source",
],
"where": "tags['amenity'] IN ('kindergarten', 'school', 'college', 'university') OR building IN ('kindergarten', 'school', 'college', 'university')",
"formats": ["geojson", "shp", "kml"],
}
},
{
"Health Facilities": {
"hdx": {
"tags": ["geodata"],
"caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive",
},
"types": ["points", "polygons"],
"select": [
"name",
"amenity",
"building",
"healthcare",
"healthcare:speciality",
"operator:type",
"capacity:persons",
"addr:full",
"addr:city",
"source",
],
"where": "tags['healthcare'] IS NOT NULL OR tags['amenity'] IN ('doctors', 'dentist', 'clinic', 'hospital', 'pharmacy')",
"formats": ["geojson", "shp", "kml"],
}
},
{
"Populated Places": {
"hdx": {
"tags": [
"populated places-settlements",
"geodata",
],
"caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive",
},
"types": ["points"],
"select": [
"name",
"place",
"population",
"is_in",
"source",
],
"where": "tags['place'] IN ('isolated_dwelling', 'town', 'village', 'hamlet', 'city')",
"formats": ["geojson", "shp", "kml"],
}
},
{
"Financial Services": {
"hdx": {
"tags": ["economics", "geodata"],
"caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive",
},
"types": ["points", "polygons"],
"select": [
"name",
"amenity",
"operator",
"network",
"addr:full",
"addr:city",
"source",
],
"where": "tags['amenity'] IN ('mobile_money_agent','bureau_de_change','bank','microfinance','atm','sacco','money_transfer','post_office')",
"formats": ["geojson", "shp", "kml"],
}
},
{
"Railways": {
"hdx": {
"tags": [
"facilities-infrastructure",
"railways",
"transportation",
"geodata",
],
"caveats": "OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive",
},
"types": ["lines"],
"select": [
"name",
"railway",
"ele",
"operator:type",
"layer",
"addr:full",
"addr:city",
"source",
],
"where": "tags['railway'] IN ('rail','station')",
"formats": ["geojson", "shp", "kml"],
}
},
],
},
},
},
),
):
Expand Down
6 changes: 4 additions & 2 deletions src/app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1515,6 +1515,8 @@ def clean_resources(self):
temp_dir = os.path.join(export_path, self.uuid)
if os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
return True
return False

def process_hdx_tags(self):
"""
Expand All @@ -1540,10 +1542,10 @@ def process_hdx_tags(self):
self.params.geometry,
)
start = time.time()
logging.info("Transfer-> Postgres Data to DuckDB Started")
logging.info(f"Transfer-> Postgres Data to DuckDB Started : {table}")
self.duck_db_instance.run_query(create_table.strip(), attach_pgsql=True)
logging.info(
f"Transfer-> Postgres Data to DuckDB Done in {time.time()-start}s"
f"Transfer-> Postgres Data to DuckDB : {table} Done in {humanize.naturaldelta(timedelta(seconds=(time.time()-start)))}s"
)

CategoryResult = namedtuple(
Expand Down

0 comments on commit 5268cc5

Please sign in to comment.