Add point retention multiplier (#179)

* Add an option to retain N times as many points as usual at each zoom

* Tests for point multipler with specified and guessed maxzooms

* Work in progress on inverse spatial ordering

* Fix inverse spatial feature order

* --reorder was depending on a feature index that wasn't being preserved

* Separate ordering by feature_minzoom from ordering inverse-spatially

* Add a test for the inverse spatial ordering

* Store the basezoom/droprate/multiplier decisions in tileset metadata

* Progress on adding filters to tippecanoe-overzoom

* Type promotion for comparison

* Look up the attribute value for ordering

* Add test of thinning and ordering features

* Plumb tippecanoe_decisions metadata through pmtiles

* Be careful not to put infinities in JSON

* Fix accidental dropping in what is meant to preserve sparse points

* Start distinguishing true, false, and null in expressions

* Most of the type conversions

* Add boolean conversions

* Literals and conjunctions

* Add filtering to tippecanoe-overzoom

* Add a test of filtering in overzoom

* Fix boolean conjunctions

* Handle the combination of cluster size and filtering

* Rework dot dropping to reconcile density threshold and multiplier

* Revert "Rework dot dropping to reconcile density threshold and multiplier"

This reverts commit f253a66.

* Retain points by multiplier within each tile, not in global probability

* Test that intends to verify that the multiplier is reversible

* Get the test to detect the discrepancy

* Mark the start of multiplier clusters with a magic attribute

* Add string-contains

* Add in and ni operators

* Revert "Look up the attribute value for ordering"

This reverts commit 56bc73e.

* Revert "Type promotion for comparison"

This reverts commit 6f3256f.

* Make number formatting in tippecanoe_decisions consistent

* Revert "Add a test for the inverse spatial ordering"

This reverts commit c8047de.

* Revert "Separate ordering by feature_minzoom from ordering inverse-spatially"

This reverts commit 35b19a2.

* Revert "Fix inverse spatial feature order"

This reverts commit 5978ecd.

* Revert "Work in progress on inverse spatial ordering"

This reverts commit fdf230f.

* Somehow missed the tests associated with that last revert

* Round-robin assign attributes to partials from across the multiplier

* Count the multiplier separately in each layer

* Fix distribution of accumulated attribute across multiplier features

* Update changelog, version, and docs

* Add "is null" and "isnt null" expressions

* Update interpretation of FSL expressions to pass the tests

* Test to assert that polygons are unaffected by the multiplier

* Clean up and comment

* Remove accidental unused case

CHANGELOG.md

@@ -1,3 +1,10 @@
+# 2.41.0
+
+* Add Felt-style expression support for -j feature filters
+* Add --retain-points-multiplier option
+* Add tippecanoe_decisions metadata field to record basezoom, drop rate, and multiplier
+* Add multiplier thinning (-m) and feature filters (-j) to tippecanoe-overzoom
+
 # 2.40.0
 
 * Slightly reduce compression aggressiveness to improve as-needed dropping speed

Makefile

@@ -76,7 +76,7 @@ tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o
 unit: unit.o text.o sort.o mvt.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
-tippecanoe-overzoom: overzoom.o mvt.o clip.o
+tippecanoe-overzoom: overzoom.o mvt.o clip.o evaluator.o jsonpull/jsonpull.o text.o
 	$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
 
 -include $(wildcard *.d)
@@ -287,6 +287,37 @@ overzoom-test: tippecanoe-overzoom
 	./tippecanoe-overzoom -o tests/pbf/14-2616-6331.pbf tests/pbf/11-327-791.pbf 11/327/791 14/2616/6331
 	cmp tests/pbf/14-2616-6331.pbf /dev/null
 	rm tests/pbf/14-2616-6331.pbf
+	# Thinning
+	# 243 features in the source tile tests/pbf/0-0-0-pop.pbf
+	# 9 of them survive as the best of each cluster of 30
+	# ./tippecanoe -z1 -r30 --retain-points-multiplier 30 -f -e out.dir tests/ne_110m_populated_places/in.json
+	# cp out.dir/0/0/0.pbf tests/pbf/0-0-0-pop.pbf
+	./tippecanoe-overzoom -y NAME -m -o tests/pbf/0-0-0-pop-filtered.pbf tests/pbf/0-0-0-pop.pbf 0/0/0 0/0/0
+	./tippecanoe-decode tests/pbf/0-0-0-pop-filtered.pbf 0 0 0 > tests/pbf/0-0-0-pop-filtered.pbf.json.check
+	cmp tests/pbf/0-0-0-pop-filtered.pbf.json.check tests/pbf/0-0-0-pop-filtered.pbf.json
+	rm tests/pbf/0-0-0-pop-filtered.pbf tests/pbf/0-0-0-pop-filtered.pbf.json.check
+	# Filtering
+	# 243 features in the source tile tests/pbf/0-0-0-pop.pbf
+	# 27 of them match the filter and are retained
+	./tippecanoe-overzoom -y NAME -j'{"*":["SCALERANK","eq",0]}' -o tests/pbf/0-0-0-pop-expr.pbf tests/pbf/0-0-0-pop.pbf 0/0/0 0/0/0
+	./tippecanoe-decode tests/pbf/0-0-0-pop-expr.pbf 0 0 0 > tests/pbf/0-0-0-pop-expr.pbf.json.check
+	cmp tests/pbf/0-0-0-pop-expr.pbf.json.check tests/pbf/0-0-0-pop-expr.pbf.json
+	rm tests/pbf/0-0-0-pop-expr.pbf tests/pbf/0-0-0-pop-expr.pbf.json.check
+	# Filtering with multiplier
+	# 243 features in the source tile tests/pbf/0-0-0-pop.pbf
+	# 8 features survive into the output, from 9 clusters of 30
+	./tippecanoe-overzoom -y NAME -y SCALERANK -j'{"*":["SCALERANK","eq",0]}' -m -o tests/pbf/0-0-0-filter-mult.pbf tests/pbf/0-0-0-pop.pbf 0/0/0 0/0/0
+	./tippecanoe-decode tests/pbf/0-0-0-filter-mult.pbf 0 0 0 > tests/pbf/0-0-0-filter-mult.pbf.json.check
+	cmp tests/pbf/0-0-0-filter-mult.pbf.json.check tests/pbf/0-0-0-filter-mult.pbf.json
+	rm tests/pbf/0-0-0-filter-mult.pbf tests/pbf/0-0-0-filter-mult.pbf.json.check
+	# Test that overzooming with a multiplier exactly reverses the effect of tiling with a multiplier
+	./tippecanoe -q -z5 --preserve-point-density-threshold 8 --retain-points-multiplier 3 -f -e tests/muni/out/out.dir tests/muni/muni.json
+	./tippecanoe -q -z5 --preserve-point-density-threshold 8 -f -o tests/muni/out/out.mbtiles tests/muni/muni.json
+	./tippecanoe-overzoom -m -o tests/muni/out/out.dir/000.pbf tests/muni/out/out.dir/0/0/0.pbf 0/0/0 0/0/0
+	./tippecanoe-decode tests/muni/out/out.mbtiles 0 0 0 > tests/muni/out/out.dir/direct.json
+	./tippecanoe-decode tests/muni/out/out.dir/000.pbf 0 0 0 > tests/muni/out/out.dir/overzoomed.json
+	cmp tests/muni/out/out.dir/overzoomed.json tests/muni/out/out.dir/direct.json
+	rm -rf tests/muni/out/out.dir tests/muni/out/out.mbtiles tests/muni/out/out.dir/overzoomed.json tests/muni/out/out.dir/direct.json
 
 join-test: tippecanoe tippecanoe-decode tile-join
 	./tippecanoe -q -f -z12 -o tests/join-population/tabblock_06001420.mbtiles -YALAND10:'Land area' -L'{"file": "tests/join-population/tabblock_06001420.json", "description": "population"}'

README.md

@@ -421,7 +421,7 @@ be reduced to the maximum that can be used with the specified _maxzoom_.
 
 ### Filtering features by attributes
 
- * `-j` *filter* or `--feature-filter`=*filter*: Check features against a per-layer filter (as defined in the [Mapbox GL Style Specification](https://docs.mapbox.com/mapbox-gl-js/style-spec/#other-filter)) and only include those that match. Any features in layers that have no filter specified will be passed through. Filters for the layer `"*"` apply to all layers. The special variable `$zoom` refers to the current zoom level.
+ * `-j` *filter* or `--feature-filter`=*filter*: Check features against a per-layer filter (as defined in the [Mapbox GL Style Specification](https://docs.mapbox.com/mapbox-gl-js/style-spec/#other-filter) or in a Felt filter specification still to be finalized) and only include those that match. Any features in layers that have no filter specified will be passed through. Filters for the layer `"*"` apply to all layers. The special variable `$zoom` refers to the current zoom level.
  * `-J` *filter-file* or `--feature-filter-file`=*filter-file*: Like `-j`, but read the filter from a file.
 
 Example: to find the Natural Earth countries with low `scalerank` but high `LABELRANK`:
@@ -459,6 +459,7 @@ the same layer, enclose them in an `all` expression so they will all be evaluate
    If you use `-Bg`, it will guess a zoom level that will keep at most 50,000 features in the densest tile.
    You can also specify a marker-width with `-Bg`*width* to allow fewer features in the densest tile to
    compensate for the larger marker, or `-Bf`*number* to allow at most *number* features in the densest tile.
+ * `--retain-points-multiplier=`_multiple_: Retain the specified multiple of points instead of just the number of points that would ordinarily be retained by the drop rate. These can be thinned out later with the `-m` option to `tippecanoe-overzoom`. The start of each cluster is marked in the feature sequence by the `tippecanoe:retain_points_multiplier_first` attribute.
  * `--drop-denser=`_percentage_: When dropping dots at zoom levels below the base zoom, give the specified _percentage_
    preference to retaining points in sparse areas and dropping points in dense areas.
  * `--limit-base-zoom-to-maximum-zoom` or `-Pb`: Limit the guessed base zoom not to exceed the maxzoom, even if this would put more than the requested number of features in a base zoom tile.
@@ -993,4 +994,6 @@ reads tile `inz/inx/iny` of `in.mvt.gz` and produces tile `outz/outx/outy` of `o
  * `-b` *buffer*: Set the tile buffer in the output tile (default 5)
  * `-d` *detail*: Set the detail of the output tile (default 12)
  * `-y` *attribute*: Retain the specified *attribute* in the output features. All attributes that are not named in a `-y` option will be removed.
+ * `-j` *filter*: Filter features using the same expression syntax as in tippecanoe.
+ * `-m`: If a tile was created with the `--retain-points-multiplier` option, thin the tile back down to its normal feature count during overzooming. The first feature from each cluster will be retained, unless `-j` is used to specify a filter, in which case the first matching filter from each cluster will be retained instead.
 

clip.cpp

@@ -7,6 +7,7 @@
 #include "errors.hpp"
 #include "compression.hpp"
 #include "mvt.hpp"
+#include "evaluator.hpp"
 
 static std::vector<std::pair<double, double>> clip_poly1(std::vector<std::pair<double, double>> &geom,
 							 long long minx, long long miny, long long maxx, long long maxy,
@@ -754,7 +755,8 @@ static std::vector<std::pair<double, double>> clip_poly1(std::vector<std::pair<d
 
 std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int ny,
 		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
-		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles) {
+		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles,
+		     bool demultiply, json_object *filter) {
 	mvt_tile tile;
 
 	try {
@@ -768,12 +770,49 @@ std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int
 		exit(EXIT_PROTOBUF);
 	}
 
-	return overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, do_compress, next_overzoomed_tiles);
+	return overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, do_compress, next_overzoomed_tiles, demultiply, filter);
+}
+
+struct tile_feature {
+	drawvec geom;
+	int t;
+	bool has_id;
+	unsigned long long id;
+	std::vector<unsigned> tags;
+	mvt_layer const *layer;
+};
+
+void feature_out(tile_feature const &feature, mvt_layer &outlayer, std::set<std::string> const &keep) {
+	// Add geometry to output feature
+
+	mvt_feature outfeature;
+	outfeature.type = feature.t;
+	for (auto const &g : feature.geom) {
+		outfeature.geometry.emplace_back(g.op, g.x, g.y);
+	}
+
+	// ID and attributes, if it didn't get clipped away
+
+	if (outfeature.geometry.size() > 0) {
+		if (feature.has_id) {
+			outfeature.has_id = true;
+			outfeature.id = feature.id;
+		}
+
+		for (size_t i = 0; i + 1 < feature.tags.size(); i += 2) {
+			if (keep.size() == 0 || keep.find(feature.layer->keys[feature.tags[i]]) != keep.end()) {
+				outlayer.tag(outfeature, feature.layer->keys[feature.tags[i]], feature.layer->values[feature.tags[i + 1]]);
+			}
+		}
+
+		outlayer.features.push_back(outfeature);
+	}
 }
 
 std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int ny,
 		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
-		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles) {
+		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles,
+		     bool demultiply, json_object *filter) {
 	mvt_tile outtile;
 
 	for (auto const &layer : tile.layers) {
@@ -788,8 +827,37 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int
 		outlayer.version = layer.version;
 		outlayer.extent = 1LL << det;
 
-		for (auto const &feature : layer.features) {
-			mvt_feature outfeature;
+		std::vector<tile_feature> pending_tile_features;
+
+		for (auto feature : layer.features) {
+			bool flush_multiplier_cluster = false;
+			if (demultiply) {
+				for (size_t i = 0; i + 1 < feature.tags.size(); i += 2) {
+					if (layer.keys[feature.tags[i]] == "tippecanoe:retain_points_multiplier_first") {
+						mvt_value v = layer.values[feature.tags[i + 1]];
+						if (v.type == mvt_bool && v.numeric_value.bool_value) {
+							flush_multiplier_cluster = true;
+							feature.tags.erase(feature.tags.begin() + i, feature.tags.begin() + i + 2);
+							break;
+						}
+					}
+				}
+			} else {
+				flush_multiplier_cluster = true;
+			}
+
+			if (flush_multiplier_cluster) {
+				if (pending_tile_features.size() > 0) {
+					feature_out(pending_tile_features[0], outlayer, keep);
+					pending_tile_features.clear();
+				}
+			}
+
+			std::set<std::string> exclude_attributes;
+			if (!evaluate(feature, layer, filter, exclude_attributes, nz)) {
+				continue;
+			}
+
 			drawvec geom;
 			int t = feature.type;
 
@@ -862,29 +930,20 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int
 				geom = close_poly(geom);
 			}
 
-			// Add geometry to output feature
-
-			outfeature.type = t;
-			for (auto const &g : geom) {
-				outfeature.geometry.emplace_back(g.op, g.x, g.y);
-			}
-
-			// ID and attributes, if it didn't get clipped away
+			tile_feature tf;
+			tf.geom = geom;
+			tf.t = t;
+			tf.has_id = feature.has_id;
+			tf.id = feature.id;
+			tf.tags = feature.tags;
+			tf.layer = &layer;
 
-			if (outfeature.geometry.size() > 0) {
-				if (feature.has_id) {
-					outfeature.has_id = true;
-					outfeature.id = feature.id;
-				}
-
-				for (size_t i = 0; i + 1 < feature.tags.size(); i += 2) {
-					if (keep.size() == 0 || keep.find(layer.keys[feature.tags[i]]) != keep.end()) {
-						outlayer.tag(outfeature, layer.keys[feature.tags[i]], layer.values[feature.tags[i + 1]]);
-					}
-				}
+			pending_tile_features.push_back(tf);
+		}
 
-				outlayer.features.push_back(outfeature);
-			}
+		if (pending_tile_features.size() > 0) {
+			feature_out(pending_tile_features[0], outlayer, keep);
+			pending_tile_features.clear();
 		}
 
 		if (outlayer.features.size() > 0) {
@@ -905,7 +964,8 @@ std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int
 				for (size_t y = 0; y < 2; y++) {
 					std::string child = overzoom(outtile, nz, nx, ny,
 								     nz + 1, nx * 2 + x, ny * 2 + y,
-								     detail, buffer, keep, false, NULL);
+								     detail, buffer, keep, false, NULL,
+								     demultiply, filter);
 					if (child.size() > 0) {
 						next_overzoomed_tiles->emplace_back(nx * 2 + x, ny * 2 + y);
 					}

dirtiles.cpp

@@ -318,6 +318,9 @@ void dir_write_metadata(const char *outdir, const metadata &m) {
 		if (m.strategies_json.size() > 0) {
 			out(state, "strategies", m.strategies_json);
 		}
+		if (m.decisions_json.size() > 0) {
+			out(state, "tippecanoe_decisions", m.decisions_json);
+		}
 		out(state, "format", m.format);
 		out(state, "generator", m.generator);
 		out(state, "generator_options", m.generator_options);