From a38a24e2ee7e32a6f95f710918aa349cda4bc829 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 28 Aug 2024 09:30:08 +0545 Subject: [PATCH] added vector tiles support --- jobs/models.py | 2 + tasks/task_runners.py | 64 ++++++++++++++++++++++++- ui/app/components/help/ExportFormats.js | 8 ++-- ui/app/components/utils.js | 18 ++++++- 4 files changed, 85 insertions(+), 7 deletions(-) diff --git a/jobs/models.py b/jobs/models.py index f2392b8dd..289b42717 100644 --- a/jobs/models.py +++ b/jobs/models.py @@ -95,6 +95,8 @@ def validate_export_formats(value): "shp", "geojson", "fgb", + "mvt", + "pmtiles", "csv", "sql", "geopackage", diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 0737b99d0..e1d654fb4 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -254,7 +254,7 @@ def finish_task(name, created_files=None, response_back=None, planet_file=False) total_bytes += file.size() task.filesize_bytes = total_bytes LOG.debug(total_bytes) - + task.save() is_hdx_export = HDXExportRegion.objects.filter(job_id=run.job_id).exists() @@ -271,6 +271,8 @@ def finish_task(name, created_files=None, response_back=None, planet_file=False) "kml", "shp", "fgb", + "mvt", + "pmtiles", "csv", "sql", "mbtiles", @@ -934,6 +936,66 @@ def add_metadata(z, theme): stop_task("mbtiles") raise ex + if "pmtiles" in export_formats: + try: + pmtiles = Galaxy( + settings.RAW_DATA_API_URL, + geom, + mapping=mapping_filter, + file_name=valid_name, + access_token=settings.RAW_DATA_ACCESS_TOKEN, + ) + start_task("pmtiles") + LOG.debug( + "Raw Data API fetch started for pmtiles run: {0}".format(run_uid) + ) + all_feature_filter_json = join( + os.getcwd(), "tasks/tests/fixtures/all_features_filters.json" + ) + response_back = pmtiles.fetch( + "pmtiles", + all_feature_filter_json=all_feature_filter_json, + min_zoom=job.mbtiles_minzoom, + max_zoom=job.mbtiles_maxzoom, + ) + write_file_size(response_back) + LOG.debug( + "Raw Data API fetch ended for mbtiles run: {0}".format(run_uid) + ) + finish_task("pmtiles", response_back=response_back) + + except Exception as ex: + stop_task("pmtiles") + raise ex + + if "mvt" in export_formats: + try: + mvt = Galaxy( + settings.RAW_DATA_API_URL, + geom, + mapping=mapping_filter, + file_name=valid_name, + access_token=settings.RAW_DATA_ACCESS_TOKEN, + ) + start_task("mvt") + LOG.debug("Raw Data API fetch started for mvt run: {0}".format(run_uid)) + all_feature_filter_json = join( + os.getcwd(), "tasks/tests/fixtures/all_features_filters.json" + ) + response_back = mvt.fetch( + "mvt", + all_feature_filter_json=all_feature_filter_json, + min_zoom=job.mbtiles_minzoom, + max_zoom=job.mbtiles_maxzoom, + ) + write_file_size(response_back) + LOG.debug("Raw Data API fetch ended for mvt run: {0}".format(run_uid)) + finish_task("mvt", response_back=response_back) + + except Exception as ex: + stop_task("mvt") + raise ex + if use_only_galaxy == False: LOG.debug("Source start for run: {0}".format(run_uid)) source_path = source.path() diff --git a/ui/app/components/help/ExportFormats.js b/ui/app/components/help/ExportFormats.js index 2f51eb9f2..d7093475e 100644 --- a/ui/app/components/help/ExportFormats.js +++ b/ui/app/components/help/ExportFormats.js @@ -101,7 +101,7 @@ export default () =>
-

GeoJSON .geojson

+

GeoJSON .geojson

GeoJSON is an open standard geospatial data interchange format that represents simple geographic features and their nonspatial attributes. Based on JavaScript Object Notation (JSON), GeoJSON is a format for encoding a variety of geographic data structures. It uses a geographic coordinate reference system, World Geodetic System 1984, and units of decimal degrees.

@@ -142,7 +142,7 @@ export default () =>
-

FlatGeobuf .fgb

+

FlatGeobuf .fgb

FlatGeobuf is a binary file format for storing geospatial vector data in a compact and efficient manner. It uses a hierarchical structure to organize features into layers, and stores attribute data in a separate file.

@@ -183,7 +183,7 @@ export default () =>
-

CSV .csv

+

CSV .csv

CSV is a file format for storing tabular data in plain text format. Each row of data represents a record, and each column represents a field of that record. CSV files are widely used because they are simple and easy to create and manipulate, making them a popular choice for data exchange.

@@ -203,7 +203,7 @@ export default () =>

-

SQL .sql

+

SQL .sql

SQL files are plain text files that contain SQL commands to create, modify or interact with a relational database. They can be used to define database schemas, constraints, and indexes, as well as to insert, update, and query data.

diff --git a/ui/app/components/utils.js b/ui/app/components/utils.js index 4fcd6e892..f89022de8 100644 --- a/ui/app/components/utils.js +++ b/ui/app/components/utils.js @@ -63,6 +63,16 @@ export const AVAILABLE_EXPORT_FORMATS = { MBTiles .mbtiles ), + pmtiles: ( + + Pmtiles .pmtiles + + ), + mvt: ( + + Mapbox Vector Tiles .mvt + + ), garmin_img: ( Garmin .img @@ -99,6 +109,8 @@ export const REQUIRES_FEATURE_SELECTION = { shp: true, geojson:true, fgb:true, + mvt:true, + pmtiles:true, sql:true, csv:true, geopackage: true, @@ -110,12 +122,14 @@ export const REQUIRES_FEATURE_SELECTION = { }; export const REQUIRES_TILE_SOURCE = { - mbtiles: true + mbtiles: true, + pmtiles:true, + mvt:true, }; export const OMIT_FROM_FORMAT_OPTIONS = { osm_xml: true, - bundle: true + bundle: true }; export const getRootUrl = () => {