From c8d3fee5e81eb8ebb7199ac08d527621c2360c08 Mon Sep 17 00:00:00 2001 From: Brandon Liu Date: Thu, 27 Jul 2023 17:04:56 +0800 Subject: [PATCH] Use rtree spatial index from PlanetilerConfig bounds by default for Geopackages. [#478] --- .../com/onthegomap/planetiler/Planetiler.java | 1 + .../planetiler/reader/GeoPackageReader.java | 30 ++++++++++++-- .../reader/GeoPackageReaderTest.java | 39 ++++++++++++++++++- 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java index c44c7c95b4..9c6a1a91c0 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java @@ -371,6 +371,7 @@ public Planetiler addShapefileSource(String projection, String name, Path defaul * @see Downloader */ public Planetiler addGeoPackageSource(String projection, String name, Path defaultPath, String defaultUrl) { + Path path = getPath(name, "geopackage", defaultPath, defaultUrl); boolean keepUnzipped = getKeepUnzipped(name); return addStage(name, "Process features in " + path, diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoPackageReader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoPackageReader.java index e326436a41..773ae99185 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoPackageReader.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoPackageReader.java @@ -14,14 +14,20 @@ import java.util.HashMap; import java.util.List; import java.util.function.Consumer; +import mil.nga.geopackage.BoundingBox; import mil.nga.geopackage.GeoPackage; import mil.nga.geopackage.GeoPackageManager; +import mil.nga.geopackage.features.index.FeatureIndexManager; +import mil.nga.geopackage.features.index.FeatureIndexResults; +import mil.nga.geopackage.features.index.FeatureIndexType; import mil.nga.geopackage.features.user.FeatureColumns; import mil.nga.geopackage.features.user.FeatureDao; +import mil.nga.geopackage.features.user.FeatureRow; import mil.nga.geopackage.geom.GeoPackageGeometryData; import org.geotools.geometry.jts.JTS; import org.geotools.geometry.jts.WKBReader; import org.geotools.referencing.CRS; +import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.opengis.referencing.FactoryException; import org.opengis.referencing.operation.MathTransform; @@ -39,9 +45,14 @@ public class GeoPackageReader extends SimpleReader { private final GeoPackage geoPackage; private final MathTransform coordinateTransform; - GeoPackageReader(String sourceProjection, String sourceName, Path input, Path tmpDir, boolean keepUnzipped) { + private final Envelope bounds; + + GeoPackageReader(String sourceProjection, String sourceName, Path input, Path tmpDir, boolean keepUnzipped, + Envelope bounds) { + super(sourceName); this.keepUnzipped = keepUnzipped; + this.bounds = bounds; if (sourceProjection != null) { try { @@ -105,7 +116,7 @@ public static void process(String sourceProjection, String sourceName, List new GeoPackageReader(sourceProjection, sourceName, path, tmpDir, keepUnzipped), + path -> new GeoPackageReader(sourceProjection, sourceName, path, tmpDir, keepUnzipped, config.bounds().latLon()), writer, config, profile, stats ); } @@ -138,7 +149,20 @@ public void readFeatures(Consumer next) throws Exception { MathTransform transform = (coordinateTransform != null) ? coordinateTransform : CRS.findMathTransform(CRS.decode("EPSG:" + srsId), latLonCRS); - for (var feature : features.queryForAll()) { + FeatureIndexManager indexer = new FeatureIndexManager(geoPackage, + features); + indexer.setIndexLocation(FeatureIndexType.RTREE); + + BoundingBox boundingBox = BoundingBox.worldWGS84(); + + if (this.bounds != null) { + var l = this.bounds; + + boundingBox = new BoundingBox(l.getMinX(), l.getMinY(), l.getMaxX(), l.getMaxY()); + } + FeatureIndexResults indexResults = indexer.query(boundingBox); + + for (FeatureRow feature : indexResults) { GeoPackageGeometryData geometryData = feature.getGeometry(); byte[] wkb; if (geometryData == null || (wkb = geometryData.getWkb()).length == 0) { diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/GeoPackageReaderTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/GeoPackageReaderTest.java index ea0675beac..623b0ada4d 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/GeoPackageReaderTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/GeoPackageReaderTest.java @@ -18,6 +18,7 @@ import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; class GeoPackageReaderTest { @@ -37,7 +38,7 @@ void testReadGeoPackage(boolean keepUnzipped) throws IOException { for (var path : List.of(pathOutsideZip, pathInZip)) { for (var proj : projections) { try ( - var reader = new GeoPackageReader(proj, "test", path, tmpDir, keepUnzipped) + var reader = new GeoPackageReader(proj, "test", path, tmpDir, keepUnzipped, null) ) { for (int iter = 0; iter < 2; iter++) { String id = "path=" + path + " proj=" + proj + " iter=" + iter; @@ -66,13 +67,47 @@ void testReadGeoPackage(boolean keepUnzipped) throws IOException { } } + + @Test + @Timeout(30) + void testReadGeoPackageSpatialIndex() throws IOException { + Path pathOutsideZip = TestUtils.pathToResource("geopackage.gpkg"); + Path zipPath = TestUtils.pathToResource("geopackage.gpkg.zip"); + Path pathInZip = FileUtils.walkPathWithPattern(zipPath, "*.gpkg").get(0); + + var projections = new String[]{null, "EPSG:4326"}; + + for (var path : List.of(pathOutsideZip, pathInZip)) { + for (var proj : projections) { + try ( + var reader = + new GeoPackageReader(proj, "test", path, tmpDir, false, new Envelope(-77.0306, -77.0192, 38.8894, 38.9014)) + ) { + for (int iter = 0; iter < 2; iter++) { + String id = "path=" + path + " proj=" + proj + " iter=" + iter; + assertEquals(86, reader.getFeatureCount(), id); + List points = new ArrayList<>(); + List names = new ArrayList<>(); + WorkerPipeline.start("test", Stats.inMemory()) + .fromGenerator("geopackage", reader::readFeatures, 1) + .addBuffer("reader_queue", 100, 1) + .sinkToConsumer("counter", 1, elem -> { + points.add(elem.latLonGeometry()); + }).await(); + assertEquals(4, points.size(), id); + } + } + } + } + } + @Test @Timeout(30) void testReadEmptyGeoPackage() throws IOException { Path path = TestUtils.pathToResource("empty-geom.gpkg"); try ( - var reader = new GeoPackageReader(null, "test", path, tmpDir, false) + var reader = new GeoPackageReader(null, "test", path, tmpDir, false, null) ) { for (int iter = 0; iter < 2; iter++) { String id = "iter=" + iter;