diff --git a/api/serializers.py b/api/serializers.py index 3352fef9f..71ddf9c5b 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -5,6 +5,7 @@ Used by the View classes api/views.py to serialize API responses as JSON or HTML. See DEFAULT_RENDERER_CLASSES setting in core.settings.contrib for the enabled renderers. """ + # -*- coding: utf-8 -*- import logging @@ -113,7 +114,7 @@ class Meta: "mbtiles_maxzoom", "pinned", "unfiltered", - "preserve_geom", + "userinfo", ) extra_kwargs = { "the_geom": {"write_only": True}, diff --git a/jobs/migrations/0001_initial.py b/jobs/migrations/0001_initial.py index f1b07dd3a..150de097e 100644 --- a/jobs/migrations/0001_initial.py +++ b/jobs/migrations/0001_initial.py @@ -96,7 +96,7 @@ class Migration(migrations.Migration): ("expire_old_runs", models.BooleanField(default=True)), ("pinned", models.BooleanField(default=False)), ("unfiltered", models.BooleanField(default=False)), - ("preserve_geom", models.BooleanField(default=False)), + ("userinfo", models.BooleanField(default=False)), ( "user", models.ForeignKey( diff --git a/jobs/models.py b/jobs/models.py index f2392b8dd..d8beefc72 100644 --- a/jobs/models.py +++ b/jobs/models.py @@ -95,6 +95,8 @@ def validate_export_formats(value): "shp", "geojson", "fgb", + "mvt", + "pmtiles", "csv", "sql", "geopackage", @@ -203,7 +205,7 @@ class Job(models.Model): expire_old_runs = models.BooleanField(default=True) pinned = models.BooleanField(default=False) unfiltered = models.BooleanField(default=False) - preserve_geom = models.BooleanField(default=False) + userinfo = models.BooleanField(default=False) class Meta: # pragma: no cover managed = True diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 0737b99d0..beaf67b52 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -254,7 +254,7 @@ def finish_task(name, created_files=None, response_back=None, planet_file=False) total_bytes += file.size() task.filesize_bytes = total_bytes LOG.debug(total_bytes) - + task.save() is_hdx_export = HDXExportRegion.objects.filter(job_id=run.job_id).exists() @@ -271,6 +271,8 @@ def finish_task(name, created_files=None, response_back=None, planet_file=False) "kml", "shp", "fgb", + "mvt", + "pmtiles", "csv", "sql", "mbtiles", @@ -673,17 +675,16 @@ def add_metadata(z, theme): mapping_filter = mapping if job.unfiltered: mapping_filter = None + userinfo = job.userinfo if "geojson" in export_formats: - preserved_geom = geom - if job.preserve_geom: - preserved_geom = load_geometry(job.the_geom.json) geojson = Galaxy( settings.RAW_DATA_API_URL, - preserved_geom, + geom, mapping=mapping_filter, file_name=valid_name, access_token=settings.RAW_DATA_ACCESS_TOKEN, + userinfo=userinfo, ) start_task("geojson") @@ -694,6 +695,7 @@ def add_metadata(z, theme): mapping=mapping_filter, file_name=valid_name, access_token=settings.RAW_DATA_ACCESS_TOKEN, + userinfo=userinfo, ) start_task("fgb") @@ -714,6 +716,7 @@ def add_metadata(z, theme): mapping=mapping_filter, file_name=valid_name, access_token=settings.RAW_DATA_ACCESS_TOKEN, + userinfo=userinfo, ) start_task("sql") @@ -724,6 +727,7 @@ def add_metadata(z, theme): mapping=mapping_filter, file_name=valid_name, access_token=settings.RAW_DATA_ACCESS_TOKEN, + userinfo=userinfo, ) # geopackage = tabular.Geopackage(join(stage_dir,valid_name),mapping) # tabular_outputs.append(geopackage) @@ -736,6 +740,7 @@ def add_metadata(z, theme): mapping=mapping_filter, file_name=valid_name, access_token=settings.RAW_DATA_ACCESS_TOKEN, + userinfo=userinfo, ) start_task("shp") @@ -746,6 +751,7 @@ def add_metadata(z, theme): mapping=mapping_filter, file_name=valid_name, access_token=settings.RAW_DATA_ACCESS_TOKEN, + userinfo=userinfo, ) # kml = tabular.Kml(join(stage_dir,valid_name),mapping) # tabular_outputs.append(kml) @@ -910,6 +916,7 @@ def add_metadata(z, theme): mapping=mapping_filter, file_name=valid_name, access_token=settings.RAW_DATA_ACCESS_TOKEN, + userinfo=userinfo, ) start_task("mbtiles") LOG.debug( @@ -934,6 +941,68 @@ def add_metadata(z, theme): stop_task("mbtiles") raise ex + if "pmtiles" in export_formats: + try: + pmtiles = Galaxy( + settings.RAW_DATA_API_URL, + geom, + mapping=mapping_filter, + file_name=valid_name, + access_token=settings.RAW_DATA_ACCESS_TOKEN, + userinfo=userinfo, + ) + start_task("pmtiles") + LOG.debug( + "Raw Data API fetch started for pmtiles run: {0}".format(run_uid) + ) + all_feature_filter_json = join( + os.getcwd(), "tasks/tests/fixtures/all_features_filters.json" + ) + response_back = pmtiles.fetch( + "pmtiles", + all_feature_filter_json=all_feature_filter_json, + min_zoom=job.mbtiles_minzoom, + max_zoom=job.mbtiles_maxzoom, + ) + write_file_size(response_back) + LOG.debug( + "Raw Data API fetch ended for mbtiles run: {0}".format(run_uid) + ) + finish_task("pmtiles", response_back=response_back) + + except Exception as ex: + stop_task("pmtiles") + raise ex + + if "mvt" in export_formats: + try: + mvt = Galaxy( + settings.RAW_DATA_API_URL, + geom, + mapping=mapping_filter, + file_name=valid_name, + access_token=settings.RAW_DATA_ACCESS_TOKEN, + userinfo=userinfo, + ) + start_task("mvt") + LOG.debug("Raw Data API fetch started for mvt run: {0}".format(run_uid)) + all_feature_filter_json = join( + os.getcwd(), "tasks/tests/fixtures/all_features_filters.json" + ) + response_back = mvt.fetch( + "mvt", + all_feature_filter_json=all_feature_filter_json, + min_zoom=job.mbtiles_minzoom, + max_zoom=job.mbtiles_maxzoom, + ) + write_file_size(response_back) + LOG.debug("Raw Data API fetch ended for mvt run: {0}".format(run_uid)) + finish_task("mvt", response_back=response_back) + + except Exception as ex: + stop_task("mvt") + raise ex + if use_only_galaxy == False: LOG.debug("Source start for run: {0}".format(run_uid)) source_path = source.path() diff --git a/ui/app/actions/exports.js b/ui/app/actions/exports.js index d583ba5f9..ee57a675b 100644 --- a/ui/app/actions/exports.js +++ b/ui/app/actions/exports.js @@ -64,7 +64,7 @@ export const cloneExport = e => (dispatch, getState) => { name: e.name, published: e.published, unfiltered: e.unfiltered, - preserve_geom: e.preserve_geom, + userinfo: e.userinfo, the_geom: rsp.data.the_geom, aoi: { description: "Cloned Area", diff --git a/ui/app/components/Summary.js b/ui/app/components/Summary.js index 686852d37..dc0b1af69 100644 --- a/ui/app/components/Summary.js +++ b/ui/app/components/Summary.js @@ -15,9 +15,9 @@ const messages = defineMessages({ // id: "export.bundle_for_posm.description", // defaultMessage: "Bundle for POSM" // }, - preserveGeometry: { - id: "export.preserve_geom.description", - defaultMessage: "Preserve Geometry - Avoid simplify ( Only supports for geojson )" + userinfo: { + id: "export.userinfo.description", + defaultMessage: "Include user info on exports" }, publishedDescription: { id: "export.published.description", @@ -84,12 +84,12 @@ export default injectIntl( component={renderCheckbox} type="checkbox" /> */} - {/* */} + />