From a38a24e2ee7e32a6f95f710918aa349cda4bc829 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 28 Aug 2024 09:30:08 +0545 Subject: [PATCH 1/2] added vector tiles support --- jobs/models.py | 2 + tasks/task_runners.py | 64 ++++++++++++++++++++++++- ui/app/components/help/ExportFormats.js | 8 ++-- ui/app/components/utils.js | 18 ++++++- 4 files changed, 85 insertions(+), 7 deletions(-) diff --git a/jobs/models.py b/jobs/models.py index f2392b8dd..289b42717 100644 --- a/jobs/models.py +++ b/jobs/models.py @@ -95,6 +95,8 @@ def validate_export_formats(value): "shp", "geojson", "fgb", + "mvt", + "pmtiles", "csv", "sql", "geopackage", diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 0737b99d0..e1d654fb4 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -254,7 +254,7 @@ def finish_task(name, created_files=None, response_back=None, planet_file=False) total_bytes += file.size() task.filesize_bytes = total_bytes LOG.debug(total_bytes) - + task.save() is_hdx_export = HDXExportRegion.objects.filter(job_id=run.job_id).exists() @@ -271,6 +271,8 @@ def finish_task(name, created_files=None, response_back=None, planet_file=False) "kml", "shp", "fgb", + "mvt", + "pmtiles", "csv", "sql", "mbtiles", @@ -934,6 +936,66 @@ def add_metadata(z, theme): stop_task("mbtiles") raise ex + if "pmtiles" in export_formats: + try: + pmtiles = Galaxy( + settings.RAW_DATA_API_URL, + geom, + mapping=mapping_filter, + file_name=valid_name, + access_token=settings.RAW_DATA_ACCESS_TOKEN, + ) + start_task("pmtiles") + LOG.debug( + "Raw Data API fetch started for pmtiles run: {0}".format(run_uid) + ) + all_feature_filter_json = join( + os.getcwd(), "tasks/tests/fixtures/all_features_filters.json" + ) + response_back = pmtiles.fetch( + "pmtiles", + all_feature_filter_json=all_feature_filter_json, + min_zoom=job.mbtiles_minzoom, + max_zoom=job.mbtiles_maxzoom, + ) + write_file_size(response_back) + LOG.debug( + "Raw Data API fetch ended for mbtiles run: {0}".format(run_uid) + ) + finish_task("pmtiles", response_back=response_back) + + except Exception as ex: + stop_task("pmtiles") + raise ex + + if "mvt" in export_formats: + try: + mvt = Galaxy( + settings.RAW_DATA_API_URL, + geom, + mapping=mapping_filter, + file_name=valid_name, + access_token=settings.RAW_DATA_ACCESS_TOKEN, + ) + start_task("mvt") + LOG.debug("Raw Data API fetch started for mvt run: {0}".format(run_uid)) + all_feature_filter_json = join( + os.getcwd(), "tasks/tests/fixtures/all_features_filters.json" + ) + response_back = mvt.fetch( + "mvt", + all_feature_filter_json=all_feature_filter_json, + min_zoom=job.mbtiles_minzoom, + max_zoom=job.mbtiles_maxzoom, + ) + write_file_size(response_back) + LOG.debug("Raw Data API fetch ended for mvt run: {0}".format(run_uid)) + finish_task("mvt", response_back=response_back) + + except Exception as ex: + stop_task("mvt") + raise ex + if use_only_galaxy == False: LOG.debug("Source start for run: {0}".format(run_uid)) source_path = source.path() diff --git a/ui/app/components/help/ExportFormats.js b/ui/app/components/help/ExportFormats.js index 2f51eb9f2..d7093475e 100644 --- a/ui/app/components/help/ExportFormats.js +++ b/ui/app/components/help/ExportFormats.js @@ -101,7 +101,7 @@ export default () =>
-

GeoJSON .geojson

+

GeoJSON .geojson

GeoJSON is an open standard geospatial data interchange format that represents simple geographic features and their nonspatial attributes. Based on JavaScript Object Notation (JSON), GeoJSON is a format for encoding a variety of geographic data structures. It uses a geographic coordinate reference system, World Geodetic System 1984, and units of decimal degrees.

@@ -142,7 +142,7 @@ export default () =>
-

FlatGeobuf .fgb

+

FlatGeobuf .fgb

FlatGeobuf is a binary file format for storing geospatial vector data in a compact and efficient manner. It uses a hierarchical structure to organize features into layers, and stores attribute data in a separate file.

@@ -183,7 +183,7 @@ export default () =>
-

CSV .csv

+

CSV .csv

CSV is a file format for storing tabular data in plain text format. Each row of data represents a record, and each column represents a field of that record. CSV files are widely used because they are simple and easy to create and manipulate, making them a popular choice for data exchange.

@@ -203,7 +203,7 @@ export default () =>

-

SQL .sql

+

SQL .sql

SQL files are plain text files that contain SQL commands to create, modify or interact with a relational database. They can be used to define database schemas, constraints, and indexes, as well as to insert, update, and query data.

diff --git a/ui/app/components/utils.js b/ui/app/components/utils.js index 4fcd6e892..f89022de8 100644 --- a/ui/app/components/utils.js +++ b/ui/app/components/utils.js @@ -63,6 +63,16 @@ export const AVAILABLE_EXPORT_FORMATS = { MBTiles .mbtiles ), + pmtiles: ( + + Pmtiles .pmtiles + + ), + mvt: ( + + Mapbox Vector Tiles .mvt + + ), garmin_img: ( Garmin .img @@ -99,6 +109,8 @@ export const REQUIRES_FEATURE_SELECTION = { shp: true, geojson:true, fgb:true, + mvt:true, + pmtiles:true, sql:true, csv:true, geopackage: true, @@ -110,12 +122,14 @@ export const REQUIRES_FEATURE_SELECTION = { }; export const REQUIRES_TILE_SOURCE = { - mbtiles: true + mbtiles: true, + pmtiles:true, + mvt:true, }; export const OMIT_FROM_FORMAT_OPTIONS = { osm_xml: true, - bundle: true + bundle: true }; export const getRootUrl = () => { From 6cee018fb935c8aac8e15521e6c1270eed44d0e2 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Thu, 29 Aug 2024 15:17:42 +0545 Subject: [PATCH 2/2] Adds userinfo on exports to be available --- api/serializers.py | 3 ++- jobs/migrations/0001_initial.py | 2 +- jobs/models.py | 2 +- tasks/task_runners.py | 15 +++++++++++---- ui/app/actions/exports.js | 2 +- ui/app/components/Summary.js | 14 +++++++------- utils/aoi_utils.py | 24 ++++++++++++------------ 7 files changed, 35 insertions(+), 27 deletions(-) diff --git a/api/serializers.py b/api/serializers.py index 3352fef9f..71ddf9c5b 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -5,6 +5,7 @@ Used by the View classes api/views.py to serialize API responses as JSON or HTML. See DEFAULT_RENDERER_CLASSES setting in core.settings.contrib for the enabled renderers. """ + # -*- coding: utf-8 -*- import logging @@ -113,7 +114,7 @@ class Meta: "mbtiles_maxzoom", "pinned", "unfiltered", - "preserve_geom", + "userinfo", ) extra_kwargs = { "the_geom": {"write_only": True}, diff --git a/jobs/migrations/0001_initial.py b/jobs/migrations/0001_initial.py index f1b07dd3a..150de097e 100644 --- a/jobs/migrations/0001_initial.py +++ b/jobs/migrations/0001_initial.py @@ -96,7 +96,7 @@ class Migration(migrations.Migration): ("expire_old_runs", models.BooleanField(default=True)), ("pinned", models.BooleanField(default=False)), ("unfiltered", models.BooleanField(default=False)), - ("preserve_geom", models.BooleanField(default=False)), + ("userinfo", models.BooleanField(default=False)), ( "user", models.ForeignKey( diff --git a/jobs/models.py b/jobs/models.py index 289b42717..d8beefc72 100644 --- a/jobs/models.py +++ b/jobs/models.py @@ -205,7 +205,7 @@ class Job(models.Model): expire_old_runs = models.BooleanField(default=True) pinned = models.BooleanField(default=False) unfiltered = models.BooleanField(default=False) - preserve_geom = models.BooleanField(default=False) + userinfo = models.BooleanField(default=False) class Meta: # pragma: no cover managed = True diff --git a/tasks/task_runners.py b/tasks/task_runners.py index e1d654fb4..beaf67b52 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -675,17 +675,16 @@ def add_metadata(z, theme): mapping_filter = mapping if job.unfiltered: mapping_filter = None + userinfo = job.userinfo if "geojson" in export_formats: - preserved_geom = geom - if job.preserve_geom: - preserved_geom = load_geometry(job.the_geom.json) geojson = Galaxy( settings.RAW_DATA_API_URL, - preserved_geom, + geom, mapping=mapping_filter, file_name=valid_name, access_token=settings.RAW_DATA_ACCESS_TOKEN, + userinfo=userinfo, ) start_task("geojson") @@ -696,6 +695,7 @@ def add_metadata(z, theme): mapping=mapping_filter, file_name=valid_name, access_token=settings.RAW_DATA_ACCESS_TOKEN, + userinfo=userinfo, ) start_task("fgb") @@ -716,6 +716,7 @@ def add_metadata(z, theme): mapping=mapping_filter, file_name=valid_name, access_token=settings.RAW_DATA_ACCESS_TOKEN, + userinfo=userinfo, ) start_task("sql") @@ -726,6 +727,7 @@ def add_metadata(z, theme): mapping=mapping_filter, file_name=valid_name, access_token=settings.RAW_DATA_ACCESS_TOKEN, + userinfo=userinfo, ) # geopackage = tabular.Geopackage(join(stage_dir,valid_name),mapping) # tabular_outputs.append(geopackage) @@ -738,6 +740,7 @@ def add_metadata(z, theme): mapping=mapping_filter, file_name=valid_name, access_token=settings.RAW_DATA_ACCESS_TOKEN, + userinfo=userinfo, ) start_task("shp") @@ -748,6 +751,7 @@ def add_metadata(z, theme): mapping=mapping_filter, file_name=valid_name, access_token=settings.RAW_DATA_ACCESS_TOKEN, + userinfo=userinfo, ) # kml = tabular.Kml(join(stage_dir,valid_name),mapping) # tabular_outputs.append(kml) @@ -912,6 +916,7 @@ def add_metadata(z, theme): mapping=mapping_filter, file_name=valid_name, access_token=settings.RAW_DATA_ACCESS_TOKEN, + userinfo=userinfo, ) start_task("mbtiles") LOG.debug( @@ -944,6 +949,7 @@ def add_metadata(z, theme): mapping=mapping_filter, file_name=valid_name, access_token=settings.RAW_DATA_ACCESS_TOKEN, + userinfo=userinfo, ) start_task("pmtiles") LOG.debug( @@ -976,6 +982,7 @@ def add_metadata(z, theme): mapping=mapping_filter, file_name=valid_name, access_token=settings.RAW_DATA_ACCESS_TOKEN, + userinfo=userinfo, ) start_task("mvt") LOG.debug("Raw Data API fetch started for mvt run: {0}".format(run_uid)) diff --git a/ui/app/actions/exports.js b/ui/app/actions/exports.js index d583ba5f9..ee57a675b 100644 --- a/ui/app/actions/exports.js +++ b/ui/app/actions/exports.js @@ -64,7 +64,7 @@ export const cloneExport = e => (dispatch, getState) => { name: e.name, published: e.published, unfiltered: e.unfiltered, - preserve_geom: e.preserve_geom, + userinfo: e.userinfo, the_geom: rsp.data.the_geom, aoi: { description: "Cloned Area", diff --git a/ui/app/components/Summary.js b/ui/app/components/Summary.js index 686852d37..dc0b1af69 100644 --- a/ui/app/components/Summary.js +++ b/ui/app/components/Summary.js @@ -15,9 +15,9 @@ const messages = defineMessages({ // id: "export.bundle_for_posm.description", // defaultMessage: "Bundle for POSM" // }, - preserveGeometry: { - id: "export.preserve_geom.description", - defaultMessage: "Preserve Geometry - Avoid simplify ( Only supports for geojson )" + userinfo: { + id: "export.userinfo.description", + defaultMessage: "Include user info on exports" }, publishedDescription: { id: "export.published.description", @@ -84,12 +84,12 @@ export default injectIntl( component={renderCheckbox} type="checkbox" /> */} - {/* */} + />