Merge pull request #188 from hotosm/enhance/select_query

Enhance Select Query for Exports : Include type

API/s3.py

@@ -88,16 +88,23 @@ def list_s3_files(
         raise HTTPException(status_code=500, detail="AWS credentials not available")
 
 
-@router.get("/s3-files/{file_path:path}")
+@router.get("/get/{file_path:path}")
 @limiter.limit(f"{POLYGON_STATISTICS_API_RATE_LIMIT}/minute")
 @version(1)
 def get_s3_file(
     request: Request,
     file_path: str = Path(..., description="The path to the file or folder in S3"),
+    expiry: int = Query(
+        default=3600,
+        description="Expiry time for the presigned URL in seconds (default: 1 hour)",
+        gt=60 * 10,
+        le=3600 * 12 * 7,
+    ),
 ):
     bucket_name = BUCKET_NAME
     file_path = file_path.strip("/")
     encoded_file_path = quote(file_path)
+
     try:
         # Check if the file or folder exists
         s3.head_object(Bucket=bucket_name, Key=encoded_file_path)
@@ -112,6 +119,7 @@ def get_s3_file(
     presigned_url = s3.generate_presigned_url(
         "get_object",
         Params={"Bucket": bucket_name, "Key": file_path},
-        ExpiresIn=3600,  # URL expires in 1 hour
+        ExpiresIn=expiry,
     )
+
     return JSONResponse(content=jsonable_encoder({"download_link": presigned_url}))

src/app.py

@@ -1131,7 +1131,7 @@ def __init__(self, db_path, temp_dir=None):
         if temp_dir is None:
             duck_db_temp = os.path.join(export_path, "duckdb_temp")
             os.makedirs(duck_db_temp, exist_ok=True)
-        con.sql(f"""SET temp_directory = '{str(duck_db_temp)}'""")
+        con.sql(f"""SET temp_directory = '{os.path.join(duck_db_temp,'temp.tmp')}'""")
 
         if DUCK_DB_MEMORY_LIMIT:
             con.sql(f"""SET memory_limit = '{DUCK_DB_MEMORY_LIMIT}'""")

src/query_builder/builder.py

@@ -124,9 +124,13 @@ def create_column_filter(
     if len(columns) > 0:
         filter_col = []
         filter_col.append("osm_id")
+        filter_col.append("tableoid::regclass AS type")
+
         if create_schema:
             schema = {}
             schema["osm_id"] = "int64"
+            schema["type"] = "str"
+
         for cl in columns:
             splitted_cl = [cl]
             if "," in cl:
@@ -150,7 +154,7 @@ def create_column_filter(
             return select_condition, schema
         return select_condition
     else:
-        return f"osm_id ,tags,changeset,timestamp,{'ST_Centroid(geom) as geom' if use_centroid else 'geom'}"  # this is default attribute that we will deliver to user if user defines his own attribute column then those will be appended with osm_id only
+        return f"osm_id, tableoid::regclass AS type, tags,changeset,timestamp,{'ST_Centroid(geom) as geom' if use_centroid else 'geom'}"  # this is default attribute that we will deliver to user if user defines his own attribute column then those will be appended with osm_id only
 
 
 def create_tag_sql_logic(key, value, filter_list):
@@ -225,9 +229,10 @@ def extract_geometry_type_query(
         params.geometry,
         "ST_within" if params.use_st_within is True else "ST_intersects",
     )
-    select_condition = f"""osm_id ,tags,changeset,timestamp , {'ST_Centroid(geom) as geom' if params.centroid else 'geom'}"""  # this is default attribute that we will deliver to user if user defines his own attribute column then those will be appended with osm_id only
+    select_condition = f"""osm_id, tableoid::regclass AS type, tags,changeset,timestamp , {'ST_Centroid(geom) as geom' if params.centroid else 'geom'}"""  # this is default attribute that we will deliver to user if user defines his own attribute column then those will be appended with osm_id only
     schema = {
         "osm_id": "int64",
+        "type": "str",
         "tags": "str",
         "changeset": "int64",
         "timestamp": "str",
@@ -519,9 +524,9 @@ def raw_currentdata_extraction_query(
 
     # query_table = []
     if select_all:
-        select_condition = f"""osm_id,version,tags,changeset,timestamp,{'ST_Centroid(geom) as geom' if params.centroid else 'geom'}"""  # FIXme have condition for displaying userinfo after user authentication
+        select_condition = f"""osm_id, tableoid::regclass AS type, version,tags,changeset,timestamp,{'ST_Centroid(geom) as geom' if params.centroid else 'geom'}"""  # FIXme have condition for displaying userinfo after user authentication
     else:
-        select_condition = f"""osm_id ,version,tags,changeset,timestamp,{'ST_Centroid(geom) as geom' if params.centroid else 'geom'}"""  # this is default attribute that we will deliver to user if user defines his own attribute column then those will be appended with osm_id only
+        select_condition = f"""osm_id, tableoid::regclass AS type, version,tags,changeset,timestamp,{'ST_Centroid(geom) as geom' if params.centroid else 'geom'}"""  # this is default attribute that we will deliver to user if user defines his own attribute column then those will be appended with osm_id only
 
     point_select_condition = select_condition  # initializing default
     line_select_condition = select_condition
@@ -766,7 +771,9 @@ def get_countries_query(q):
 
 
 def get_osm_feature_query(osm_id):
-    select_condition = "osm_id ,tags,changeset,timestamp,geom"
+    select_condition = (
+        "osm_id, tableoid::regclass AS type, tags,changeset,timestamp,geom"
+    )
     query = f"""SELECT ST_AsGeoJSON(n.*)
         FROM (select {select_condition} from nodes) n 
         WHERE osm_id = {osm_id}
@@ -864,14 +871,12 @@ def postgres2duckdb_query(
     Returns:
     str: DuckDB query for creating a table.
     """
-    select_query = (
-        """osm_id, version, changeset, timestamp, tags, ST_AsBinary(geom) as geometry"""
-    )
-    create_select_duck_db = """osm_id,version, changeset, timestamp, cast(tags::json AS map(varchar, varchar)) AS tags, cast(ST_GeomFromWKB(geometry) as GEOMETRY) AS geometry"""
+    select_query = """osm_id, tableoid::regclass AS type, version, changeset, timestamp, tags,  ST_AsBinary(geom) as geometry"""
+    create_select_duck_db = """osm_id, type , version, changeset, timestamp, type, cast(tags::json AS map(varchar, varchar)) AS tags, cast(ST_GeomFromWKB(geometry) as GEOMETRY) AS geometry"""
 
     if enable_users_detail:
-        select_query = """osm_id, uid, user, version, changeset, timestamp, tags, ST_AsBinary(geom) as geometry"""
-        create_select_duck_db = """osm_id, uid, user, version, changeset, timestamp, cast(tags::json AS map(varchar, varchar)) AS tags, cast(ST_GeomFromWKB(geometry) as GEOMETRY) AS geometry"""
+        select_query = """osm_id, tableoid::regclass AS type, uid, user, version, changeset, timestamp, tags, ST_AsBinary(geom) as geometry"""
+        create_select_duck_db = """osm_id, type, uid, user, version, changeset, timestamp,type, cast(tags::json AS map(varchar, varchar)) AS tags, cast(ST_GeomFromWKB(geometry) as GEOMETRY) AS geometry"""
 
     row_filter_condition = (
         f"""country <@ ARRAY [{cid}]"""
@@ -916,7 +921,7 @@ def extract_features_duckdb(base_table_name, select, feature_type, where):
     }
 
     select = [f"tags['{item}'][1] as '{item}'" for item in select]
-    select += ["osm_id", "geometry"]
+    select += ["osm_id", "type", "geometry"]
     select_query = ", ".join(select)
 
     from_query = map_tables[feature_type]["table"]