Merge pull request #259 from hotosm/feature/custom-tiles

Feature : Vector tile support for custom exports

API/main.py

@@ -13,11 +13,13 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+# Standard library imports
 # Humanitarian OpenStreetmap Team
 # 1100 13th Street NW Suite 800 Washington, D.C. 20005
 # <[email protected]>
 import time
 
+# Third party imports
 import psycopg2
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
@@ -26,6 +28,7 @@
 from slowapi import _rate_limit_exceeded_handler
 from slowapi.errors import RateLimitExceeded
 
+# Reader imports
 from src.config import (
     ENABLE_CUSTOM_EXPORTS,
     ENABLE_HDX_EXPORTS,
@@ -57,6 +60,7 @@
     from .hdx import router as hdx_router
 
 if SENTRY_DSN:
+    # Third party imports
     import sentry_sdk
 
 # only use sentry if it is specified in config blocks
@@ -71,6 +75,7 @@
 
 if LOG_LEVEL.lower() == "debug":
     # This is used for local setup for auth login
+    # Standard library imports
     import os
 
     os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1"

src/app.py

@@ -599,7 +599,27 @@ def ogr_export(query, outputtype, working_dir, dump_temp_path, params):
                         params.min_zoom, params.max_zoom
                     )
                     if params.min_zoom and params.max_zoom
-                    else "-dsco ZOOM_LEVEL_AUTO=YES"
+                    else "MINZOOM=0 MAXZOOM=20"
+                ),
+            }
+            format_options[RawDataOutputType.PMTILES.value] = {
+                "format": "PMTiles",
+                "extra": (
+                    "-dsco MINZOOM={} -dsco MAXZOOM={} ".format(
+                        params.min_zoom, params.max_zoom
+                    )
+                    if params.min_zoom and params.max_zoom
+                    else "-dsco MINZOOM=0 MAXZOOM=20"
+                ),
+            }
+            format_options[RawDataOutputType.MVT.value] = {
+                "format": "MVT",
+                "extra": (
+                    "-dsco MINZOOM={} -dsco MAXZOOM={} ".format(
+                        params.min_zoom, params.max_zoom
+                    )
+                    if params.min_zoom and params.max_zoom
+                    else "-dsco MINZOOM=0 MAXZOOM=20 FORMAT=MBTILES"
                 ),
             }
 
@@ -689,16 +709,6 @@ def get_grid_id(geom, cur):
             country_export,
         )
 
-    @staticmethod
-    def geojson2tiles(geojson_path, tile_path, tile_layer_name):
-        """Responsible for geojson to tiles"""
-        cmd = """tippecanoe -zg --projection=EPSG:4326 -o {tile_output_path} -l {tile_layer_name} --force {geojson_input_path}""".format(
-            tile_output_path=tile_path,
-            tile_layer_name=tile_layer_name,
-            geojson_input_path=geojson_path,
-        )
-        run_ogr2ogr_cmd(cmd)
-
     def extract_current_data(self, exportname):
         """Responsible for Extracting rawdata current snapshot, Initially it creates a geojson file , Generates query , run it with 1000 chunk size and writes it directly to the geojson file and closes the file after dump
         Args:
@@ -731,25 +741,11 @@ def extract_current_data(self, exportname):
         try:
             # currently we have only geojson binding function written other than that we have depend on ogr
             if ENABLE_TILES:
-                if output_type == RawDataOutputType.PMTILES.value:
-                    geojson_path = os.path.join(
-                        working_dir,
-                        f"{self.params.file_name if self.params.file_name else 'Export'}.geojson",
-                    )
-                    RawData.query2geojson(
-                        self.con,
-                        raw_currentdata_extraction_query(
-                            self.params,
-                            g_id=grid_id,
-                            c_id=country,
-                            country_export=country_export,
-                        ),
-                        geojson_path,
-                    )
-                    RawData.geojson2tiles(
-                        geojson_path, dump_temp_file_path, self.params.file_name
-                    )
-                if output_type == RawDataOutputType.MBTILES.value:
+                if output_type in [
+                    RawDataOutputType.PMTILES.value,
+                    RawDataOutputType.MBTILES.value,
+                    RawDataOutputType.MVT.value,
+                ]:
                     RawData.ogr_export(
                         query=raw_currentdata_extraction_query(
                             self.params,
@@ -762,7 +758,7 @@ def extract_current_data(self, exportname):
                         dump_temp_path=dump_temp_file_path,
                         working_dir=working_dir,
                         params=self.params,
-                    )  # uses ogr export to export
+                    )
 
             if output_type == RawDataOutputType.GEOJSON.value:
                 RawData.query2geojson(

src/query_builder/builder.py

@@ -140,18 +140,22 @@ def create_column_filter(
             schema = {}
             schema["osm_id"] = "int64"
             schema["type"] = "str"
-
-        for cl in columns:
-            splitted_cl = [cl]
-            if "," in cl:
-                splitted_cl = cl.split(",")
-            for cl in splitted_cl:
-                if cl != "":
-                    filter_col.append(
-                        f"""tags ->> '{cl.strip()}' as {remove_spaces(cl.strip())}"""
-                    )
-                    if create_schema:
-                        schema[remove_spaces(cl.strip())] = "str"
+        if "*" in columns:
+            filter_col.append("tags")
+            if create_schema:
+                schema["tags"] = "str"
+        else:
+            for cl in columns:
+                splitted_cl = [cl]
+                if "," in cl:
+                    splitted_cl = cl.split(",")
+                for cl in splitted_cl:
+                    if cl != "":
+                        filter_col.append(
+                            f"""tags ->> '{cl.strip()}' as {remove_spaces(cl.strip())}"""
+                        )
+                        if create_schema:
+                            schema[remove_spaces(cl.strip())] = "str"
         if output_type == "csv":  # if it is csv geom logic is different
             filter_col.append("ST_X(ST_Centroid(geom)) as longitude")
             filter_col.append("ST_Y(ST_Centroid(geom)) as latitude")
@@ -973,7 +977,10 @@ def extract_features_custom_exports(
         },
     }
     if USE_DUCK_DB_FOR_CUSTOM_EXPORTS is True:
-        select = [f"""tags['{item}'][1] as "{item}" """ for item in select]
+        if "*" in select:
+            select = [f"""tags::json as tags """]
+        else:
+            select = [f"""tags['{item}'][1] as "{item}" """ for item in select]
         select += ["osm_id", "osm_type", "geom"]
         select_query = ", ".join(select)
     else:

src/validation/models.py

@@ -65,6 +65,7 @@ class RawDataOutputType(Enum):
     if ENABLE_TILES:
         MBTILES = "mbtiles"
         PMTILES = "pmtiles"  ## EXPERIMENTAL
+        MVT = "mvt"  ## Experimental
 
 
 class SupportedFilters(Enum):
@@ -297,22 +298,22 @@ class StatsRequestParams(BaseModel, GeometryValidatorMixin):
         max_length=3,
         example="NPL",
     )
-    geometry: Optional[
-        Union[Polygon, MultiPolygon, Feature, FeatureCollection]
-    ] = Field(
-        default=None,
-        example={
-            "type": "Polygon",
-            "coordinates": [
-                [
-                    [83.96919250488281, 28.194446860487773],
-                    [83.99751663208006, 28.194446860487773],
-                    [83.99751663208006, 28.214869548073377],
-                    [83.96919250488281, 28.214869548073377],
-                    [83.96919250488281, 28.194446860487773],
-                ]
-            ],
-        },
+    geometry: Optional[Union[Polygon, MultiPolygon, Feature, FeatureCollection]] = (
+        Field(
+            default=None,
+            example={
+                "type": "Polygon",
+                "coordinates": [
+                    [
+                        [83.96919250488281, 28.194446860487773],
+                        [83.99751663208006, 28.194446860487773],
+                        [83.99751663208006, 28.214869548073377],
+                        [83.96919250488281, 28.214869548073377],
+                        [83.96919250488281, 28.194446860487773],
+                    ]
+                ],
+            },
+        )
     )
 
     @validator("geometry", pre=True, always=True)
@@ -479,7 +480,7 @@ def __init__(self, suffix, driver_name, layer_creation_options, format_option):
     "gpkg": ExportTypeInfo("gpkg", "GPKG", ["SPATIAL_INDEX=No"], "GDAL"),
     "sqlite": ExportTypeInfo("sqlite", "SQLite", [], "GDAL"),
     "fgb": ExportTypeInfo("fgb", "FlatGeobuf", ["VERIFY_BUFFERS=NO"], "GDAL"),
-    "mvt": ExportTypeInfo("mvt", "MVT", [], "GDAL"),
+    "mvt": ExportTypeInfo("mbtiles", "MVT", ["MAXZOOM=20"], "GDAL"),
     "kml": ExportTypeInfo("kml", "KML", [], "GDAL"),
     "gpx": ExportTypeInfo("gpx", "GPX", [], "GDAL"),
     "parquet": ExportTypeInfo("parquet", "PARQUET", [], "PARQUET"),
@@ -618,22 +619,22 @@ class DynamicCategoriesModel(CategoriesBase, GeometryValidatorMixin):
         max_length=3,
         example="USA",
     )
-    geometry: Optional[
-        Union[Polygon, MultiPolygon, Feature, FeatureCollection]
-    ] = Field(
-        default=None,
-        example={
-            "type": "Polygon",
-            "coordinates": [
-                [
-                    [83.96919250488281, 28.194446860487773],
-                    [83.99751663208006, 28.194446860487773],
-                    [83.99751663208006, 28.214869548073377],
-                    [83.96919250488281, 28.214869548073377],
-                    [83.96919250488281, 28.194446860487773],
-                ]
-            ],
-        },
+    geometry: Optional[Union[Polygon, MultiPolygon, Feature, FeatureCollection]] = (
+        Field(
+            default=None,
+            example={
+                "type": "Polygon",
+                "coordinates": [
+                    [
+                        [83.96919250488281, 28.194446860487773],
+                        [83.99751663208006, 28.194446860487773],
+                        [83.99751663208006, 28.214869548073377],
+                        [83.96919250488281, 28.214869548073377],
+                        [83.96919250488281, 28.194446860487773],
+                    ]
+                ],
+            },
+        )
     )
 
     @validator("geometry", pre=True, always=True)