diff --git a/API/api_worker.py b/API/api_worker.py index b06015c1..ea6a97bf 100644 --- a/API/api_worker.py +++ b/API/api_worker.py @@ -12,6 +12,7 @@ from src.config import ALLOW_BIND_ZIP_FILTER from src.config import CELERY_BROKER_URL as celery_broker_uri from src.config import CELERY_RESULT_BACKEND as celery_backend +from src.config import ENABLE_TILES from src.config import USE_S3_TO_UPLOAD as use_s3_to_upload from src.config import logger as logging from src.query_builder.builder import format_file_name_str @@ -36,12 +37,13 @@ def process_raw_data(self, params): if params.output_type else RawDataOutputType.GEOJSON.value ) - if ( - params.output_type == RawDataOutputType.PMTILES.value - or params.output_type == RawDataOutputType.MBTILES.value - ): - logging.debug("Using STwithin Logic") - params.use_st_within = True + if ENABLE_TILES: + if ( + params.output_type == RawDataOutputType.PMTILES.value + or params.output_type == RawDataOutputType.MBTILES.value + ): + logging.debug("Using STwithin Logic") + params.use_st_within = True params.file_name = ( format_file_name_str(params.file_name) if params.file_name else "Export" ) diff --git a/docs/src/installation/configurations.md b/docs/src/installation/configurations.md index 20a05ac4..3aa4dd67 100644 --- a/docs/src/installation/configurations.md +++ b/docs/src/installation/configurations.md @@ -73,7 +73,7 @@ The following are the different configuration options that are accepted. | `EXPORT_PATH` | `[API_CONFIG]` | Yes | Yes | | `EXPORT_MAX_AREA_SQKM` | `[API_CONFIG]` | Yes | No | | `USE_CONNECTION_POOLING` | `[API_CONFIG]` | Yes | Yes | -| `ENABLE_TILES` | `[API_CONFIG]` | Yes | No | +| `ENABLE_TILES` | `[API_CONFIG]` | Yes | Yes | | `ALLOW_BIND_ZIP_FILTER` | `[API_CONFIG]` | Yes | Yes | | `INDEX_THRESHOLD` | `[API_CONFIG]` | No | Yes | | `CELERY_BROKER_URL` | TBD | Yes | Yes | diff --git a/src/app.py b/src/app.py index c6b95ab1..ba0457dc 100644 --- a/src/app.py +++ b/src/app.py @@ -34,7 +34,12 @@ from psycopg2 import OperationalError, connect from psycopg2.extras import DictCursor -from src.config import AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME +from src.config import ( + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, + BUCKET_NAME, + ENABLE_TILES, +) from src.config import EXPORT_PATH as export_path from src.config import INDEX_THRESHOLD as index_threshold from src.config import USE_CONNECTION_POOLING as use_connection_pooling @@ -321,28 +326,29 @@ def ogr_export(query, outputtype, working_dir, dump_temp_path, params): with open(query_path, "w", encoding="UTF-8") as file: file.write(query) # for mbtiles we need additional input as well i.e. minzoom and maxzoom , setting default at max=22 and min=10 - if outputtype == RawDataOutputType.MBTILES.value: - if params.min_zoom and params.max_zoom: - cmd = """ogr2ogr -overwrite -f MBTILES -dsco MINZOOM={min_zoom} -dsco MAXZOOM={max_zoom} {export_path} PG:"host={host} user={username} dbname={db} password={password}" -sql @"{pg_sql_select}" -lco ENCODING=UTF-8 -progress""".format( - min_zoom=params.min_zoom, - max_zoom=params.max_zoom, - export_path=dump_temp_path, - host=db_items.get("host"), - username=db_items.get("user"), - db=db_items.get("dbname"), - password=db_items.get("password"), - pg_sql_select=query_path, - ) - else: - cmd = """ogr2ogr -overwrite -f MBTILES -dsco ZOOM_LEVEL_AUTO=YES {export_path} PG:"host={host} user={username} dbname={db} password={password}" -sql @"{pg_sql_select}" -lco ENCODING=UTF-8 -progress""".format( - export_path=dump_temp_path, - host=db_items.get("host"), - username=db_items.get("user"), - db=db_items.get("dbname"), - password=db_items.get("password"), - pg_sql_select=query_path, - ) - run_ogr2ogr_cmd(cmd) + if ENABLE_TILES: + if outputtype == RawDataOutputType.MBTILES.value: + if params.min_zoom and params.max_zoom: + cmd = """ogr2ogr -overwrite -f MBTILES -dsco MINZOOM={min_zoom} -dsco MAXZOOM={max_zoom} {export_path} PG:"host={host} user={username} dbname={db} password={password}" -sql @"{pg_sql_select}" -lco ENCODING=UTF-8 -progress""".format( + min_zoom=params.min_zoom, + max_zoom=params.max_zoom, + export_path=dump_temp_path, + host=db_items.get("host"), + username=db_items.get("user"), + db=db_items.get("dbname"), + password=db_items.get("password"), + pg_sql_select=query_path, + ) + else: + cmd = """ogr2ogr -overwrite -f MBTILES -dsco ZOOM_LEVEL_AUTO=YES {export_path} PG:"host={host} user={username} dbname={db} password={password}" -sql @"{pg_sql_select}" -lco ENCODING=UTF-8 -progress""".format( + export_path=dump_temp_path, + host=db_items.get("host"), + username=db_items.get("user"), + db=db_items.get("dbname"), + password=db_items.get("password"), + pg_sql_select=query_path, + ) + run_ogr2ogr_cmd(cmd) if outputtype == RawDataOutputType.FLATGEOBUF.value: cmd = """ogr2ogr -overwrite -f FLATGEOBUF {export_path} PG:"host={host} port={port} user={username} dbname={db} password={password}" -sql @"{pg_sql_select}" -lco ENCODING=UTF-8 -progress VERIFY_BUFFERS=NO""".format( @@ -519,6 +525,40 @@ def extract_current_data(self, exportname): ) try: # currently we have only geojson binding function written other than that we have depend on ogr + if ENABLE_TILES: + if output_type == RawDataOutputType.PMTILES.value: + geojson_path = os.path.join( + working_dir, + f"{self.params.file_name if self.params.file_name else 'Export'}.geojson", + ) + RawData.query2geojson( + self.con, + raw_currentdata_extraction_query( + self.params, + g_id=grid_id, + c_id=country, + country_export=country_export, + ), + geojson_path, + ) + RawData.geojson2tiles( + geojson_path, dump_temp_file_path, self.params.file_name + ) + if output_type == RawDataOutputType.MBTILES.value: + RawData.ogr_export( + query=raw_currentdata_extraction_query( + self.params, + grid_id, + country, + ogr_export=True, + country_export=country_export, + ), + outputtype=output_type, + dump_temp_path=dump_temp_file_path, + working_dir=working_dir, + params=self.params, + ) # uses ogr export to export + if output_type == RawDataOutputType.GEOJSON.value: RawData.query2geojson( self.con, @@ -530,25 +570,7 @@ def extract_current_data(self, exportname): ), dump_temp_file_path, ) # uses own conversion class - elif output_type == RawDataOutputType.PMTILES.value: - geojson_path = os.path.join( - working_dir, - f"{self.params.file_name if self.params.file_name else 'Export'}.geojson", - ) - RawData.query2geojson( - self.con, - raw_currentdata_extraction_query( - self.params, - g_id=grid_id, - c_id=country, - country_export=country_export, - ), - geojson_path, - ) - RawData.geojson2tiles( - geojson_path, dump_temp_file_path, self.params.file_name - ) - elif output_type == RawDataOutputType.SHAPEFILE.value: + if output_type == RawDataOutputType.SHAPEFILE.value: ( point_query, line_query, @@ -572,7 +594,7 @@ def extract_current_data(self, exportname): if self.params.file_name else "Export", ) # using ogr2ogr - else: + if output_type in ["fgb", "gpkg", "sql", "csv"]: RawData.ogr_export( query=raw_currentdata_extraction_query( self.params,