Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature : PMtiles , MVT tiles #530

Merged
merged 2 commits into from
Sep 10, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion api/serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
Used by the View classes api/views.py to serialize API responses as JSON or HTML.
See DEFAULT_RENDERER_CLASSES setting in core.settings.contrib for the enabled renderers.
"""

# -*- coding: utf-8 -*-
import logging

Expand Down Expand Up @@ -113,7 +114,7 @@ class Meta:
"mbtiles_maxzoom",
"pinned",
"unfiltered",
"preserve_geom",
"userinfo",
)
extra_kwargs = {
"the_geom": {"write_only": True},
Expand Down
2 changes: 1 addition & 1 deletion jobs/migrations/0001_initial.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ class Migration(migrations.Migration):
("expire_old_runs", models.BooleanField(default=True)),
("pinned", models.BooleanField(default=False)),
("unfiltered", models.BooleanField(default=False)),
("preserve_geom", models.BooleanField(default=False)),
("userinfo", models.BooleanField(default=False)),
(
"user",
models.ForeignKey(
Expand Down
4 changes: 3 additions & 1 deletion jobs/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,8 @@ def validate_export_formats(value):
"shp",
"geojson",
"fgb",
"mvt",
"pmtiles",
"csv",
"sql",
"geopackage",
Expand Down Expand Up @@ -203,7 +205,7 @@ class Job(models.Model):
expire_old_runs = models.BooleanField(default=True)
pinned = models.BooleanField(default=False)
unfiltered = models.BooleanField(default=False)
preserve_geom = models.BooleanField(default=False)
userinfo = models.BooleanField(default=False)

class Meta: # pragma: no cover
managed = True
Expand Down
79 changes: 74 additions & 5 deletions tasks/task_runners.py
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ def finish_task(name, created_files=None, response_back=None, planet_file=False)
total_bytes += file.size()
task.filesize_bytes = total_bytes
LOG.debug(total_bytes)

task.save()

is_hdx_export = HDXExportRegion.objects.filter(job_id=run.job_id).exists()
Expand All @@ -271,6 +271,8 @@ def finish_task(name, created_files=None, response_back=None, planet_file=False)
"kml",
"shp",
"fgb",
"mvt",
"pmtiles",
"csv",
"sql",
"mbtiles",
Expand Down Expand Up @@ -673,17 +675,16 @@ def add_metadata(z, theme):
mapping_filter = mapping
if job.unfiltered:
mapping_filter = None
userinfo = job.userinfo

if "geojson" in export_formats:
preserved_geom = geom
if job.preserve_geom:
preserved_geom = load_geometry(job.the_geom.json)
geojson = Galaxy(
settings.RAW_DATA_API_URL,
preserved_geom,
geom,
mapping=mapping_filter,
file_name=valid_name,
access_token=settings.RAW_DATA_ACCESS_TOKEN,
userinfo=userinfo,
)
start_task("geojson")

Expand All @@ -694,6 +695,7 @@ def add_metadata(z, theme):
mapping=mapping_filter,
file_name=valid_name,
access_token=settings.RAW_DATA_ACCESS_TOKEN,
userinfo=userinfo,
)
start_task("fgb")

Expand All @@ -714,6 +716,7 @@ def add_metadata(z, theme):
mapping=mapping_filter,
file_name=valid_name,
access_token=settings.RAW_DATA_ACCESS_TOKEN,
userinfo=userinfo,
)
start_task("sql")

Expand All @@ -724,6 +727,7 @@ def add_metadata(z, theme):
mapping=mapping_filter,
file_name=valid_name,
access_token=settings.RAW_DATA_ACCESS_TOKEN,
userinfo=userinfo,
)
# geopackage = tabular.Geopackage(join(stage_dir,valid_name),mapping)
# tabular_outputs.append(geopackage)
Expand All @@ -736,6 +740,7 @@ def add_metadata(z, theme):
mapping=mapping_filter,
file_name=valid_name,
access_token=settings.RAW_DATA_ACCESS_TOKEN,
userinfo=userinfo,
)
start_task("shp")

Expand All @@ -746,6 +751,7 @@ def add_metadata(z, theme):
mapping=mapping_filter,
file_name=valid_name,
access_token=settings.RAW_DATA_ACCESS_TOKEN,
userinfo=userinfo,
)
# kml = tabular.Kml(join(stage_dir,valid_name),mapping)
# tabular_outputs.append(kml)
Expand Down Expand Up @@ -910,6 +916,7 @@ def add_metadata(z, theme):
mapping=mapping_filter,
file_name=valid_name,
access_token=settings.RAW_DATA_ACCESS_TOKEN,
userinfo=userinfo,
)
start_task("mbtiles")
LOG.debug(
Expand All @@ -934,6 +941,68 @@ def add_metadata(z, theme):
stop_task("mbtiles")
raise ex

if "pmtiles" in export_formats:
try:
pmtiles = Galaxy(
settings.RAW_DATA_API_URL,
geom,
mapping=mapping_filter,
file_name=valid_name,
access_token=settings.RAW_DATA_ACCESS_TOKEN,
userinfo=userinfo,
)
start_task("pmtiles")
LOG.debug(
"Raw Data API fetch started for pmtiles run: {0}".format(run_uid)
)
all_feature_filter_json = join(
os.getcwd(), "tasks/tests/fixtures/all_features_filters.json"
)
response_back = pmtiles.fetch(
"pmtiles",
all_feature_filter_json=all_feature_filter_json,
min_zoom=job.mbtiles_minzoom,
max_zoom=job.mbtiles_maxzoom,
)
write_file_size(response_back)
LOG.debug(
"Raw Data API fetch ended for mbtiles run: {0}".format(run_uid)
)
finish_task("pmtiles", response_back=response_back)

except Exception as ex:
stop_task("pmtiles")
raise ex

if "mvt" in export_formats:
try:
mvt = Galaxy(
settings.RAW_DATA_API_URL,
geom,
mapping=mapping_filter,
file_name=valid_name,
access_token=settings.RAW_DATA_ACCESS_TOKEN,
userinfo=userinfo,
)
start_task("mvt")
LOG.debug("Raw Data API fetch started for mvt run: {0}".format(run_uid))
all_feature_filter_json = join(
os.getcwd(), "tasks/tests/fixtures/all_features_filters.json"
)
response_back = mvt.fetch(
"mvt",
all_feature_filter_json=all_feature_filter_json,
min_zoom=job.mbtiles_minzoom,
max_zoom=job.mbtiles_maxzoom,
)
write_file_size(response_back)
LOG.debug("Raw Data API fetch ended for mvt run: {0}".format(run_uid))
finish_task("mvt", response_back=response_back)

except Exception as ex:
stop_task("mvt")
raise ex

if use_only_galaxy == False:
LOG.debug("Source start for run: {0}".format(run_uid))
source_path = source.path()
Expand Down
2 changes: 1 addition & 1 deletion ui/app/actions/exports.js
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ export const cloneExport = e => (dispatch, getState) => {
name: e.name,
published: e.published,
unfiltered: e.unfiltered,
preserve_geom: e.preserve_geom,
userinfo: e.userinfo,
the_geom: rsp.data.the_geom,
aoi: {
description: "Cloned Area",
Expand Down
14 changes: 7 additions & 7 deletions ui/app/components/Summary.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ const messages = defineMessages({
// id: "export.bundle_for_posm.description",
// defaultMessage: "Bundle for POSM"
// },
preserveGeometry: {
id: "export.preserve_geom.description",
defaultMessage: "Preserve Geometry - Avoid simplify ( Only supports for geojson )"
userinfo: {
id: "export.userinfo.description",
defaultMessage: "Include user info on exports"
},
publishedDescription: {
id: "export.published.description",
Expand Down Expand Up @@ -84,12 +84,12 @@ export default injectIntl(
component={renderCheckbox}
type="checkbox"
/> */}
{/* <Field
name="preserve_geom"
description={formatMessage(messages.preserveGeometry)}
<Field
name="userinfo"
description={formatMessage(messages.userinfo)}
component={renderCheckbox}
type="checkbox"
/> */}
/>
<Button
bsStyle="danger"
disabled={submitting}
Expand Down
8 changes: 4 additions & 4 deletions ui/app/components/help/ExportFormats.js
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ export default () =>
</ul>
</div>
<div>
<h2 id="gpkg">GeoJSON .geojson</h2>
<h2 id="geojson">GeoJSON .geojson</h2>
<p>
GeoJSON is an open standard geospatial data interchange format that represents simple geographic features and their nonspatial attributes. Based on JavaScript Object Notation (JSON), GeoJSON is a format for encoding a variety of geographic data structures. It uses a geographic coordinate reference system, World Geodetic System 1984, and units of decimal degrees.
</p>
Expand Down Expand Up @@ -142,7 +142,7 @@ export default () =>
</ul>
</div>
<div>
<h2 id="gpkg">FlatGeobuf .fgb</h2>
<h2 id="fgb">FlatGeobuf .fgb</h2>
<p>
FlatGeobuf is a binary file format for storing geospatial vector data in a compact and efficient manner. It uses a hierarchical structure to organize features into layers, and stores attribute data in a separate file.
</p>
Expand Down Expand Up @@ -183,7 +183,7 @@ export default () =>
</ul>
</div>
<div>
<h2 id="gpkg">CSV .csv</h2>
<h2 id="csv">CSV .csv</h2>
<p>
CSV is a file format for storing tabular data in plain text format. Each row of data represents a record, and each column represents a field of that record. CSV files are widely used because they are simple and easy to create and manipulate, making them a popular choice for data exchange. </p>
<h4>
Expand All @@ -203,7 +203,7 @@ export default () =>
</ul>
</div>
<div>
<h2 id="gpkg">SQL .sql</h2>
<h2 id="sql">SQL .sql</h2>
<p>
SQL files are plain text files that contain SQL commands to create, modify or interact with a relational database. They can be used to define database schemas, constraints, and indexes, as well as to insert, update, and query data.
</p>
Expand Down
18 changes: 16 additions & 2 deletions ui/app/components/utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,16 @@ export const AVAILABLE_EXPORT_FORMATS = {
MBTiles <code>.mbtiles</code>
</span>
),
pmtiles: (
<span key="pmtiles">
Pmtiles <code>.pmtiles</code>
</span>
),
mvt: (
<span key="mvt">
Mapbox Vector Tiles <code>.mvt</code>
</span>
),
garmin_img: (
<span key="garmin_img">
Garmin <code>.img</code>
Expand Down Expand Up @@ -99,6 +109,8 @@ export const REQUIRES_FEATURE_SELECTION = {
shp: true,
geojson:true,
fgb:true,
mvt:true,
pmtiles:true,
sql:true,
csv:true,
geopackage: true,
Expand All @@ -110,12 +122,14 @@ export const REQUIRES_FEATURE_SELECTION = {
};

export const REQUIRES_TILE_SOURCE = {
mbtiles: true
mbtiles: true,
pmtiles:true,
mvt:true,
};

export const OMIT_FROM_FORMAT_OPTIONS = {
osm_xml: true,
bundle: true
bundle: true
};

export const getRootUrl = () => {
Expand Down
24 changes: 12 additions & 12 deletions utils/aoi_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,22 +11,22 @@
# be buffered efficiently, they must be simplified first.
# so first simplify them to 0.01 degrees.


def force2d(geom):
# force geom to be 2d: https://groups.google.com/forum/#!topic/django-users/7c1NZ76UwRU
wkt = wkt_w(dim=2).write(geom).decode()
return GEOSGeometry(wkt)

def simplify_geom(geom,force_buffer=False, preserve_geom=False):
if preserve_geom is False:
if geom.num_coords > 10000:
geom = geom.simplify(0.01)
if geom.num_coords > 500:
geom = geom.buffer(0.02)
param = 0.01
while geom.num_coords > 500:
geom = geom.simplify(param, preserve_topology=True)
param = param * 2

def simplify_geom(geom, force_buffer=False):
if geom.num_coords > 10000:
geom = geom.simplify(0.01)
if geom.num_coords > 500:
geom = geom.buffer(0.02)
param = 0.01
while geom.num_coords > 500:
geom = geom.simplify(param, preserve_topology=True)
param = param * 2
if force_buffer:
geom = geom.buffer(0.02)
geom = geom.buffer(0.02)
return geom

Loading