Added enable tiles support for app

hotosm · Oct 10, 2023 · 4121330 · 4121330
1 parent 3dc9ffc
commit 4121330
Show file tree

Hide file tree

Showing 3 changed files with 74 additions and 50 deletions.
diff --git a/API/api_worker.py b/API/api_worker.py
@@ -12,6 +12,7 @@
 from src.config import ALLOW_BIND_ZIP_FILTER
 from src.config import CELERY_BROKER_URL as celery_broker_uri
 from src.config import CELERY_RESULT_BACKEND as celery_backend
+from src.config import ENABLE_TILES
 from src.config import USE_S3_TO_UPLOAD as use_s3_to_upload
 from src.config import logger as logging
 from src.query_builder.builder import format_file_name_str
@@ -36,12 +37,13 @@ def process_raw_data(self, params):
             if params.output_type
             else RawDataOutputType.GEOJSON.value
         )
-        if (
-            params.output_type == RawDataOutputType.PMTILES.value
-            or params.output_type == RawDataOutputType.MBTILES.value
-        ):
-            logging.debug("Using STwithin Logic")
-            params.use_st_within = True
+        if ENABLE_TILES:
+            if (
+                params.output_type == RawDataOutputType.PMTILES.value
+                or params.output_type == RawDataOutputType.MBTILES.value
+            ):
+                logging.debug("Using STwithin Logic")
+                params.use_st_within = True
         params.file_name = (
             format_file_name_str(params.file_name) if params.file_name else "Export"
         )

diff --git a/docs/src/installation/configurations.md b/docs/src/installation/configurations.md
@@ -73,7 +73,7 @@ The following are the different configuration options that are accepted.
 | `EXPORT_PATH` | `[API_CONFIG]` | Yes | Yes |
 | `EXPORT_MAX_AREA_SQKM` | `[API_CONFIG]` | Yes | No |
 | `USE_CONNECTION_POOLING` | `[API_CONFIG]` | Yes | Yes |
-| `ENABLE_TILES` | `[API_CONFIG]` | Yes | No |
+| `ENABLE_TILES` | `[API_CONFIG]` | Yes | Yes |
 | `ALLOW_BIND_ZIP_FILTER` | `[API_CONFIG]` | Yes | Yes |
 | `INDEX_THRESHOLD` | `[API_CONFIG]` | No | Yes |
 | `CELERY_BROKER_URL` | TBD | Yes | Yes |

diff --git a/src/app.py b/src/app.py
@@ -34,7 +34,12 @@
 from psycopg2 import OperationalError, connect
 from psycopg2.extras import DictCursor
 
-from src.config import AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME
+from src.config import (
+    AWS_ACCESS_KEY_ID,
+    AWS_SECRET_ACCESS_KEY,
+    BUCKET_NAME,
+    ENABLE_TILES,
+)
 from src.config import EXPORT_PATH as export_path
 from src.config import INDEX_THRESHOLD as index_threshold
 from src.config import USE_CONNECTION_POOLING as use_connection_pooling
@@ -321,28 +326,29 @@ def ogr_export(query, outputtype, working_dir, dump_temp_path, params):
         with open(query_path, "w", encoding="UTF-8") as file:
             file.write(query)
         # for mbtiles we need additional input as well i.e. minzoom and maxzoom , setting default at max=22 and min=10
-        if outputtype == RawDataOutputType.MBTILES.value:
-            if params.min_zoom and params.max_zoom:
-                cmd = """ogr2ogr -overwrite -f MBTILES  -dsco MINZOOM={min_zoom} -dsco MAXZOOM={max_zoom} {export_path} PG:"host={host} user={username} dbname={db} password={password}" -sql @"{pg_sql_select}" -lco ENCODING=UTF-8 -progress""".format(
-                    min_zoom=params.min_zoom,
-                    max_zoom=params.max_zoom,
-                    export_path=dump_temp_path,
-                    host=db_items.get("host"),
-                    username=db_items.get("user"),
-                    db=db_items.get("dbname"),
-                    password=db_items.get("password"),
-                    pg_sql_select=query_path,
-                )
-            else:
-                cmd = """ogr2ogr -overwrite -f MBTILES  -dsco ZOOM_LEVEL_AUTO=YES {export_path} PG:"host={host} user={username} dbname={db} password={password}" -sql @"{pg_sql_select}" -lco ENCODING=UTF-8 -progress""".format(
-                    export_path=dump_temp_path,
-                    host=db_items.get("host"),
-                    username=db_items.get("user"),
-                    db=db_items.get("dbname"),
-                    password=db_items.get("password"),
-                    pg_sql_select=query_path,
-                )
-            run_ogr2ogr_cmd(cmd)
+        if ENABLE_TILES:
+            if outputtype == RawDataOutputType.MBTILES.value:
+                if params.min_zoom and params.max_zoom:
+                    cmd = """ogr2ogr -overwrite -f MBTILES  -dsco MINZOOM={min_zoom} -dsco MAXZOOM={max_zoom} {export_path} PG:"host={host} user={username} dbname={db} password={password}" -sql @"{pg_sql_select}" -lco ENCODING=UTF-8 -progress""".format(
+                        min_zoom=params.min_zoom,
+                        max_zoom=params.max_zoom,
+                        export_path=dump_temp_path,
+                        host=db_items.get("host"),
+                        username=db_items.get("user"),
+                        db=db_items.get("dbname"),
+                        password=db_items.get("password"),
+                        pg_sql_select=query_path,
+                    )
+                else:
+                    cmd = """ogr2ogr -overwrite -f MBTILES  -dsco ZOOM_LEVEL_AUTO=YES {export_path} PG:"host={host} user={username} dbname={db} password={password}" -sql @"{pg_sql_select}" -lco ENCODING=UTF-8 -progress""".format(
+                        export_path=dump_temp_path,
+                        host=db_items.get("host"),
+                        username=db_items.get("user"),
+                        db=db_items.get("dbname"),
+                        password=db_items.get("password"),
+                        pg_sql_select=query_path,
+                    )
+                run_ogr2ogr_cmd(cmd)
 
         if outputtype == RawDataOutputType.FLATGEOBUF.value:
             cmd = """ogr2ogr -overwrite -f FLATGEOBUF {export_path} PG:"host={host} port={port} user={username} dbname={db} password={password}" -sql @"{pg_sql_select}" -lco ENCODING=UTF-8 -progress VERIFY_BUFFERS=NO""".format(
@@ -519,6 +525,40 @@ def extract_current_data(self, exportname):
         )
         try:
             # currently we have only geojson binding function written other than that we have depend on ogr
+            if ENABLE_TILES:
+                if output_type == RawDataOutputType.PMTILES.value:
+                    geojson_path = os.path.join(
+                        working_dir,
+                        f"{self.params.file_name if self.params.file_name else 'Export'}.geojson",
+                    )
+                    RawData.query2geojson(
+                        self.con,
+                        raw_currentdata_extraction_query(
+                            self.params,
+                            g_id=grid_id,
+                            c_id=country,
+                            country_export=country_export,
+                        ),
+                        geojson_path,
+                    )
+                    RawData.geojson2tiles(
+                        geojson_path, dump_temp_file_path, self.params.file_name
+                    )
+                if output_type == RawDataOutputType.MBTILES.value:
+                    RawData.ogr_export(
+                        query=raw_currentdata_extraction_query(
+                            self.params,
+                            grid_id,
+                            country,
+                            ogr_export=True,
+                            country_export=country_export,
+                        ),
+                        outputtype=output_type,
+                        dump_temp_path=dump_temp_file_path,
+                        working_dir=working_dir,
+                        params=self.params,
+                    )  # uses ogr export to export
+
             if output_type == RawDataOutputType.GEOJSON.value:
                 RawData.query2geojson(
                     self.con,
@@ -530,25 +570,7 @@ def extract_current_data(self, exportname):
                     ),
                     dump_temp_file_path,
                 )  # uses own conversion class
-            elif output_type == RawDataOutputType.PMTILES.value:
-                geojson_path = os.path.join(
-                    working_dir,
-                    f"{self.params.file_name if self.params.file_name else 'Export'}.geojson",
-                )
-                RawData.query2geojson(
-                    self.con,
-                    raw_currentdata_extraction_query(
-                        self.params,
-                        g_id=grid_id,
-                        c_id=country,
-                        country_export=country_export,
-                    ),
-                    geojson_path,
-                )
-                RawData.geojson2tiles(
-                    geojson_path, dump_temp_file_path, self.params.file_name
-                )
-            elif output_type == RawDataOutputType.SHAPEFILE.value:
+            if output_type == RawDataOutputType.SHAPEFILE.value:
                 (
                     point_query,
                     line_query,
@@ -572,7 +594,7 @@ def extract_current_data(self, exportname):
                     if self.params.file_name
                     else "Export",
                 )  # using ogr2ogr
-            else:
+            if output_type in ["fgb", "gpkg", "sql", "csv"]:
                 RawData.ogr_export(
                     query=raw_currentdata_extraction_query(
                         self.params,