Optimize tile sizes (#112)

src/main/java/org/openmaptiles/layers/Building.java

@@ -178,7 +178,10 @@ public void process(Tables.OsmBuildingPolygon element, FeatureCollector features
   @Override
   public List<VectorTile.Feature> postProcess(int zoom,
     List<VectorTile.Feature> items) throws GeometryException {
-    return (mergeZ13Buildings && zoom == 13) ? FeatureMerge.mergeNearbyPolygons(items, 4, 4, 0.5, 0.5) : items;
+    return (mergeZ13Buildings && zoom == 13) ?
+      FeatureMerge.mergeNearbyPolygons(items, 4, 4, 0.5, 0.5) :
+      // reduces the size of some heavy z14 tiles with many small buildings by 60% or more
+      FeatureMerge.mergeMultiPolygon(items);
   }
 
   private record BuildingRelationInfo(long id) implements OsmRelationInfo {

src/main/java/org/openmaptiles/layers/Housenumber.java

@@ -36,9 +36,14 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 package org.openmaptiles.layers;
 
 import com.onthegomap.planetiler.FeatureCollector;
+import com.onthegomap.planetiler.FeatureMerge;
+import com.onthegomap.planetiler.ForwardingProfile;
+import com.onthegomap.planetiler.VectorTile;
 import com.onthegomap.planetiler.config.PlanetilerConfig;
+import com.onthegomap.planetiler.geo.GeometryException;
 import com.onthegomap.planetiler.stats.Stats;
 import com.onthegomap.planetiler.util.Translations;
+import java.util.List;
 import org.openmaptiles.generated.OpenMapTilesSchema;
 import org.openmaptiles.generated.Tables;
 
@@ -51,7 +56,8 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  */
 public class Housenumber implements
   OpenMapTilesSchema.Housenumber,
-  Tables.OsmHousenumberPoint.Handler {
+  Tables.OsmHousenumberPoint.Handler,
+  ForwardingProfile.FeaturePostProcessor {
 
   public Housenumber(Translations translations, PlanetilerConfig config, Stats stats) {}
 
@@ -62,4 +68,10 @@ public void process(Tables.OsmHousenumberPoint element, FeatureCollector feature
       .setAttr(Fields.HOUSENUMBER, element.housenumber())
       .setMinZoom(14);
   }
+
+  @Override
+  public List<VectorTile.Feature> postProcess(int zoom, List<VectorTile.Feature> list) throws GeometryException {
+    // reduces the size of some heavy z14 tiles with many repeated housenumber values by 60% or more
+    return FeatureMerge.mergeMultiPoint(list);
+  }
 }

src/main/java/org/openmaptiles/layers/Landcover.java

@@ -125,6 +125,8 @@ public void process(Tables.OsmLandcoverPolygon element, FeatureCollector feature
     if (clazz != null) {
       features.polygon(LAYER_NAME).setBufferPixels(BUFFER_SIZE)
         .setMinPixelSizeOverrides(MIN_PIXEL_SIZE_THRESHOLDS)
+        // default is 0.1, this helps reduce size of some heavy z7-10 tiles
+        .setPixelToleranceBelowZoom(10, 0.25)
         .setAttr(Fields.CLASS, clazz)
         .setAttr(Fields.SUBCLASS, subclass)
         .setNumPointsAttr(TEMP_NUM_POINTS_ATTR)

src/main/java/org/openmaptiles/layers/Landuse.java

@@ -48,10 +48,10 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 import com.onthegomap.planetiler.util.Parse;
 import com.onthegomap.planetiler.util.Translations;
 import com.onthegomap.planetiler.util.ZoomFunction;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;
 import org.openmaptiles.OpenMapTilesProfile;
 import org.openmaptiles.generated.OpenMapTilesSchema;
 import org.openmaptiles.generated.Tables;
@@ -126,19 +126,20 @@ public void process(Tables.OsmLandusePolygon element, FeatureCollector features)
   @Override
   public List<VectorTile.Feature> postProcess(int zoom,
     List<VectorTile.Feature> items) throws GeometryException {
-    if (zoom < 6 || zoom > 12) {
-      return items;
-    } else {
-      // merging only merges polygons with class "residential" for z6-z12
-      Map<Boolean, List<VectorTile.Feature>> splitLists =
-        items.stream().collect(Collectors.partitioningBy(
-          i -> FieldValues.CLASS_RESIDENTIAL.equals(i.attrs().get(Fields.CLASS)))
-        );
-      List<VectorTile.Feature> result = splitLists.get(Boolean.FALSE);
-      List<VectorTile.Feature> toMerge = splitLists.get(Boolean.TRUE);
-      var merged = FeatureMerge.mergeNearbyPolygons(toMerge, 1, 1, 0.1, 0.1);
-      result.addAll(merged);
-      return result;
+    List<VectorTile.Feature> toMerge = new ArrayList<>();
+    List<VectorTile.Feature> result = new ArrayList<>();
+    for (var item : items) {
+      if (FieldValues.CLASS_RESIDENTIAL.equals(item.attrs().get(Fields.CLASS))) {
+        toMerge.add(item);
+      } else {
+        result.add(item);
+      }
     }
+    var merged = zoom <= 12 ?
+      FeatureMerge.mergeNearbyPolygons(toMerge, 1, 1, 0.1, 0.1) :
+      // reduces size of some heavy z13-14 tiles with lots of small polygons
+      FeatureMerge.mergeMultiPolygon(toMerge);
+    result.addAll(merged);
+    return result;
   }
 }

src/main/java/org/openmaptiles/layers/Place.java

@@ -267,6 +267,8 @@ public void process(Tables.OsmStatePoint element, FeatureCollector features) {
           .putAttrs(names)
           .setAttr(Fields.CLASS, element.place())
           .setAttr(Fields.RANK, rank)
+          // TODO: This starts including every "state" point at z2, even before many countries show up.
+          //       Instead we might want to set state min zooms based on rank from natural earth?
           .setMinZoom(2)
           .setSortKey(rank);
       }

src/main/java/org/openmaptiles/layers/Transportation.java

@@ -361,8 +361,10 @@ public void process(Tables.OsmHighwayLinestring element, FeatureCollector featur
         // main attributes at all zoom levels (used for grouping <= z8)
         .setAttr(Fields.CLASS, highwayClass)
         .setAttr(Fields.SUBCLASS, highwaySubclass(highwayClass, element.publicTransport(), highway))
-        .setAttr(Fields.BRUNNEL, brunnel(element.isBridge(), element.isTunnel(), element.isFord()))
         .setAttr(Fields.NETWORK, networkType != null ? networkType.name : null)
+        // TODO: including brunnel at low zooms leads to some large 300-400+kb z4-7 tiles, instead
+        //       we should only set brunnel if the line is above a certain length
+        .setAttr(Fields.BRUNNEL, brunnel(element.isBridge(), element.isTunnel(), element.isFord()))
         // z8+
         .setAttrWithMinzoom(Fields.EXPRESSWAY, element.expressway() && !"motorway".equals(highway) ? 1 : null, 8)
         // z9+

src/test/java/org/openmaptiles/OpenMapTilesTest.java

@@ -8,6 +8,7 @@
 
 import com.onthegomap.planetiler.TestUtils;
 import com.onthegomap.planetiler.VectorTile;
+import com.onthegomap.planetiler.archive.Tile;
 import com.onthegomap.planetiler.config.Arguments;
 import com.onthegomap.planetiler.mbtiles.Mbtiles;
 import com.onthegomap.planetiler.util.FileUtils;
@@ -91,7 +92,7 @@ void testMetadata() {
 
   @Test
   void ensureValidGeometries() throws Exception {
-    Set<Mbtiles.TileEntry> parsedTiles = TestUtils.getAllTiles(mbtiles);
+    Set<Tile> parsedTiles = TestUtils.getTiles(mbtiles);
     for (var tileEntry : parsedTiles) {
       var decoded = VectorTile.decode(gunzip(tileEntry.bytes()));
       for (VectorTile.Feature feature : decoded) {

src/test/java/org/openmaptiles/layers/BuildingTest.java

@@ -143,13 +143,24 @@ void testMergePolygonsZ13() throws GeometryException {
     );
 
     Assertions.assertEquals(
-      2,
+      1, // merged into 1 multipolygon
       profile.postProcessLayerFeatures(Building.LAYER_NAME, 14, List.of(poly1, poly2)).size()
     );
+
+    Assertions.assertEquals(
+      2, // merged into 1 multipolygon
+      profile.postProcessLayerFeatures(Building.LAYER_NAME, 14, List.of(poly1, poly2)).get(0).geometry().decode()
+        .getNumGeometries()
+    );
     Assertions.assertEquals(
       1,
       profile.postProcessLayerFeatures(Building.LAYER_NAME, 13, List.of(poly1, poly2)).size()
     );
+    Assertions.assertEquals(
+      1,
+      profile.postProcessLayerFeatures(Building.LAYER_NAME, 13, List.of(poly1, poly2)).get(0).geometry().decode()
+        .getNumGeometries()
+    );
   }
 
   @Test

src/test/java/org/openmaptiles/layers/LanduseTest.java

@@ -120,12 +120,26 @@ void testMergePolygonsZ12() throws GeometryException {
     );
 
     Assertions.assertEquals(
-      3,
-      profile.postProcessLayerFeatures(Landuse.LAYER_NAME, 13, List.of(poly1, poly2, poly3)).size()
+      List.of(1, 2),
+      profile.postProcessLayerFeatures(Landuse.LAYER_NAME, 13, List.of(poly1, poly2, poly3)).stream()
+        .map(d -> {
+          try {
+            return d.geometry().decode().getNumGeometries();
+          } catch (GeometryException e) {
+            throw new AssertionError(e);
+          }
+        }).toList()
     );
     Assertions.assertEquals(
-      2,
-      profile.postProcessLayerFeatures(Landuse.LAYER_NAME, 12, List.of(poly1, poly2, poly3)).size()
+      List.of(1, 1),
+      profile.postProcessLayerFeatures(Landuse.LAYER_NAME, 12, List.of(poly1, poly2, poly3)).stream()
+        .map(d -> {
+          try {
+            return d.geometry().decode().getNumGeometries();
+          } catch (GeometryException e) {
+            throw new AssertionError(e);
+          }
+        }).toList()
     );
   }
 }

src/test/java/org/openmaptiles/util/VerifyMonacoTest.java

@@ -42,7 +42,7 @@ void testEmptyTablesInvalid() {
   }
 
   @Test
-  void testStilInvalidWithOneTile() throws IOException {
+  void testStillInvalidWithOneTile() throws IOException {
     mbtiles.createTablesWithIndexes();
     mbtiles.metadataTable().setMetadata("name", "name");
     try (var writer = mbtiles.newBatchedTileWriter()) {