Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature : Vector tile support for custom exports #259

Merged
merged 4 commits into from
Jul 30, 2024
Merged

Feature : Vector tile support for custom exports #259

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
79d47d6
feat(vector-tiles): support for vector tiles , allow user to select a…
kshitijrajsharma Jul 22, 2024
58381f7
test(test-yaml): add vector tiles format for test and select *
kshitijrajsharma Jul 22, 2024
30bd688
Add max zoom param
kshitijrajsharma Jul 30, 2024
45dd7dd
fix(wild-card-select): fixes bug on wild card select for the query bu…
kshitijrajsharma Jul 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions API/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,13 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.

# Standard library imports
# Humanitarian OpenStreetmap Team
# 1100 13th Street NW Suite 800 Washington, D.C. 20005
# <[email protected]>
import time

# Third party imports
import psycopg2
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
Expand All @@ -26,6 +28,7 @@
from slowapi import _rate_limit_exceeded_handler
from slowapi.errors import RateLimitExceeded

# Reader imports
from src.config import (
ENABLE_CUSTOM_EXPORTS,
ENABLE_HDX_EXPORTS,
Expand Down Expand Up @@ -57,6 +60,7 @@
from .hdx import router as hdx_router

if SENTRY_DSN:
# Third party imports
import sentry_sdk

# only use sentry if it is specified in config blocks
Expand All @@ -71,6 +75,7 @@

if LOG_LEVEL.lower() == "debug":
# This is used for local setup for auth login
# Standard library imports
import os

os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1"
Expand Down
58 changes: 27 additions & 31 deletions src/app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -599,7 +599,27 @@ def ogr_export(query, outputtype, working_dir, dump_temp_path, params):
params.min_zoom, params.max_zoom
)
if params.min_zoom and params.max_zoom
else "-dsco ZOOM_LEVEL_AUTO=YES"
else "MINZOOM=0 MAXZOOM=20"
),
}
format_options[RawDataOutputType.PMTILES.value] = {
"format": "PMTiles",
"extra": (
"-dsco MINZOOM={} -dsco MAXZOOM={} ".format(
params.min_zoom, params.max_zoom
)
if params.min_zoom and params.max_zoom
else "-dsco MINZOOM=0 MAXZOOM=20"
),
}
format_options[RawDataOutputType.MVT.value] = {
"format": "MVT",
"extra": (
"-dsco MINZOOM={} -dsco MAXZOOM={} ".format(
params.min_zoom, params.max_zoom
)
if params.min_zoom and params.max_zoom
else "-dsco MINZOOM=0 MAXZOOM=20 FORMAT=MBTILES"
),
}

Expand Down Expand Up @@ -689,16 +709,6 @@ def get_grid_id(geom, cur):
country_export,
)

@staticmethod
def geojson2tiles(geojson_path, tile_path, tile_layer_name):
"""Responsible for geojson to tiles"""
cmd = """tippecanoe -zg --projection=EPSG:4326 -o {tile_output_path} -l {tile_layer_name} --force {geojson_input_path}""".format(
tile_output_path=tile_path,
tile_layer_name=tile_layer_name,
geojson_input_path=geojson_path,
)
run_ogr2ogr_cmd(cmd)

def extract_current_data(self, exportname):
"""Responsible for Extracting rawdata current snapshot, Initially it creates a geojson file , Generates query , run it with 1000 chunk size and writes it directly to the geojson file and closes the file after dump
Args:
Expand Down Expand Up @@ -731,25 +741,11 @@ def extract_current_data(self, exportname):
try:
# currently we have only geojson binding function written other than that we have depend on ogr
if ENABLE_TILES:
if output_type == RawDataOutputType.PMTILES.value:
geojson_path = os.path.join(
working_dir,
f"{self.params.file_name if self.params.file_name else 'Export'}.geojson",
)
RawData.query2geojson(
self.con,
raw_currentdata_extraction_query(
self.params,
g_id=grid_id,
c_id=country,
country_export=country_export,
),
geojson_path,
)
RawData.geojson2tiles(
geojson_path, dump_temp_file_path, self.params.file_name
)
if output_type == RawDataOutputType.MBTILES.value:
if output_type in [
RawDataOutputType.PMTILES.value,
RawDataOutputType.MBTILES.value,
RawDataOutputType.MVT.value,
]:
RawData.ogr_export(
query=raw_currentdata_extraction_query(
self.params,
Expand All @@ -762,7 +758,7 @@ def extract_current_data(self, exportname):
dump_temp_path=dump_temp_file_path,
working_dir=working_dir,
params=self.params,
) # uses ogr export to export
)

if output_type == RawDataOutputType.GEOJSON.value:
RawData.query2geojson(
Expand Down
33 changes: 20 additions & 13 deletions src/query_builder/builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,18 +140,22 @@ def create_column_filter(
schema = {}
schema["osm_id"] = "int64"
schema["type"] = "str"

for cl in columns:
splitted_cl = [cl]
if "," in cl:
splitted_cl = cl.split(",")
for cl in splitted_cl:
if cl != "":
filter_col.append(
f"""tags ->> '{cl.strip()}' as {remove_spaces(cl.strip())}"""
)
if create_schema:
schema[remove_spaces(cl.strip())] = "str"
if "*" in columns:
filter_col.append("tags")
if create_schema:
schema["tags"] = "str"
else:
for cl in columns:
splitted_cl = [cl]
if "," in cl:
splitted_cl = cl.split(",")
for cl in splitted_cl:
if cl != "":
filter_col.append(
f"""tags ->> '{cl.strip()}' as {remove_spaces(cl.strip())}"""
)
if create_schema:
schema[remove_spaces(cl.strip())] = "str"
if output_type == "csv": # if it is csv geom logic is different
filter_col.append("ST_X(ST_Centroid(geom)) as longitude")
filter_col.append("ST_Y(ST_Centroid(geom)) as latitude")
Expand Down Expand Up @@ -973,7 +977,10 @@ def extract_features_custom_exports(
},
}
if USE_DUCK_DB_FOR_CUSTOM_EXPORTS is True:
select = [f"""tags['{item}'][1] as "{item}" """ for item in select]
if "*" in select:
select = [f"""tags::json as tags """]
else:
select = [f"""tags['{item}'][1] as "{item}" """ for item in select]
select += ["osm_id", "osm_type", "geom"]
select_query = ", ".join(select)
else:
Expand Down
67 changes: 34 additions & 33 deletions src/validation/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ class RawDataOutputType(Enum):
if ENABLE_TILES:
MBTILES = "mbtiles"
PMTILES = "pmtiles" ## EXPERIMENTAL
MVT = "mvt" ## Experimental


class SupportedFilters(Enum):
Expand Down Expand Up @@ -297,22 +298,22 @@ class StatsRequestParams(BaseModel, GeometryValidatorMixin):
max_length=3,
example="NPL",
)
geometry: Optional[
Union[Polygon, MultiPolygon, Feature, FeatureCollection]
] = Field(
default=None,
example={
"type": "Polygon",
"coordinates": [
[
[83.96919250488281, 28.194446860487773],
[83.99751663208006, 28.194446860487773],
[83.99751663208006, 28.214869548073377],
[83.96919250488281, 28.214869548073377],
[83.96919250488281, 28.194446860487773],
]
],
},
geometry: Optional[Union[Polygon, MultiPolygon, Feature, FeatureCollection]] = (
Field(
default=None,
example={
"type": "Polygon",
"coordinates": [
[
[83.96919250488281, 28.194446860487773],
[83.99751663208006, 28.194446860487773],
[83.99751663208006, 28.214869548073377],
[83.96919250488281, 28.214869548073377],
[83.96919250488281, 28.194446860487773],
]
],
},
)
)

@validator("geometry", pre=True, always=True)
Expand Down Expand Up @@ -479,7 +480,7 @@ def __init__(self, suffix, driver_name, layer_creation_options, format_option):
"gpkg": ExportTypeInfo("gpkg", "GPKG", ["SPATIAL_INDEX=No"], "GDAL"),
"sqlite": ExportTypeInfo("sqlite", "SQLite", [], "GDAL"),
"fgb": ExportTypeInfo("fgb", "FlatGeobuf", ["VERIFY_BUFFERS=NO"], "GDAL"),
"mvt": ExportTypeInfo("mvt", "MVT", [], "GDAL"),
"mvt": ExportTypeInfo("mbtiles", "MVT", ["MAXZOOM=20"], "GDAL"),
"kml": ExportTypeInfo("kml", "KML", [], "GDAL"),
"gpx": ExportTypeInfo("gpx", "GPX", [], "GDAL"),
"parquet": ExportTypeInfo("parquet", "PARQUET", [], "PARQUET"),
Expand Down Expand Up @@ -618,22 +619,22 @@ class DynamicCategoriesModel(CategoriesBase, GeometryValidatorMixin):
max_length=3,
example="USA",
)
geometry: Optional[
Union[Polygon, MultiPolygon, Feature, FeatureCollection]
] = Field(
default=None,
example={
"type": "Polygon",
"coordinates": [
[
[83.96919250488281, 28.194446860487773],
[83.99751663208006, 28.194446860487773],
[83.99751663208006, 28.214869548073377],
[83.96919250488281, 28.214869548073377],
[83.96919250488281, 28.194446860487773],
]
],
},
geometry: Optional[Union[Polygon, MultiPolygon, Feature, FeatureCollection]] = (
Field(
default=None,
example={
"type": "Polygon",
"coordinates": [
[
[83.96919250488281, 28.194446860487773],
[83.99751663208006, 28.194446860487773],
[83.99751663208006, 28.214869548073377],
[83.96919250488281, 28.214869548073377],
[83.96919250488281, 28.194446860487773],
]
],
},
)
)

@validator("geometry", pre=True, always=True)
Expand Down
Loading
Loading