Drop or retain whole multiplier clusters when dropping as needed (#198)

* Prep to track conditions other than just "dropped" or "kept"

* Count up instead of down

* Drop or retain whole multiplier clusters based on their first feature

* Calculate a global feature dropping sequence

* Switch over to using the drop sequence for drop-fraction

* Remove unused arguments for the old drop-fraction implementation

* Fix copy-and-paste bugs, update tests

* Properly incorporate feature_minzoom into the drop sequence, I hope

* Rename drop_by to drop_sequence

* See if sorting within clusters fixes filter stability between zooms

* Remove very chatty debug print

* Update changelog and version

* Use named constants instead of numbers for feature dropping/keeping

* Add comment to explain purpose and method of bit reversal

CHANGELOG.md

@@ -1,3 +1,9 @@
+# 2.43.0
+
+* Change -fraction-as-needed feature dropping to be consistent across tiles and zoom levels, and to follow the same pattern as point dropping by zoom level
+* With -as-needed feature dropping, drop or retain entire multiplier clusters instead of individual features
+* Sort the features within each multiplier cluster by its retention priority, for more consistency between zoom levels in filtered feature choice
+
 # 2.42.0
 
 * Improve tiling speed

mvt.hpp

@@ -54,7 +54,7 @@ struct mvt_feature {
 	int /* mvt_geometry_type */ type = 0;
 	unsigned long long id = 0;
 	bool has_id = false;
-	bool dropped = false;
+	int dropped = 0;
 	size_t seq = 0;	 // used for ordering in overzoom
 
 	mvt_feature() {

plugin.cpp

@@ -428,7 +428,9 @@ serial_feature parse_feature(json_pull *jp, int z, unsigned x, unsigned y, std::
 
 				json_object *dropped = json_hash_get(tippecanoe, "dropped");
 				if (dropped != NULL && dropped->type == JSON_TRUE) {
-					sf.dropped = true;
+					sf.dropped = FEATURE_DROPPED;  // dropped
+				} else {
+					sf.dropped = FEATURE_KEPT;  // kept
 				}
 			}
 

serial.cpp

@@ -709,7 +709,20 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf, std::
 		}
 	}
 
-	if (additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_COALESCE_DENSEST_AS_NEEDED] || additional[A_CLUSTER_DENSEST_AS_NEEDED] || additional[A_CALCULATE_FEATURE_DENSITY] || additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED] || additional[A_INCREASE_GAMMA_AS_NEEDED] || additional[A_GENERATE_POLYGON_LABEL_POINTS] || sst->uses_gamma || cluster_distance != 0) {
+	if (additional[A_DROP_DENSEST_AS_NEEDED] ||
+	    additional[A_COALESCE_DENSEST_AS_NEEDED] ||
+	    additional[A_CLUSTER_DENSEST_AS_NEEDED] ||
+	    additional[A_CALCULATE_FEATURE_DENSITY] ||
+	    additional[A_DROP_SMALLEST_AS_NEEDED] ||
+	    additional[A_COALESCE_SMALLEST_AS_NEEDED] ||
+	    additional[A_DROP_FRACTION_AS_NEEDED] ||
+	    additional[A_COALESCE_FRACTION_AS_NEEDED] ||
+	    prevent[P_DYNAMIC_DROP] ||
+	    additional[A_INCREASE_GAMMA_AS_NEEDED] ||
+	    additional[A_GENERATE_POLYGON_LABEL_POINTS] ||
+	    sst->uses_gamma ||
+	    retain_points_multiplier > 1 ||
+	    cluster_distance != 0) {
 		sf.index = bbox_index;
 	} else {
 		sf.index = 0;

serial.hpp

@@ -89,9 +89,16 @@ struct serial_feature {
 	// to be provided by frontends:
 
 	long long bbox[4] = {0, 0, 0, 0};
-	bool dropped = false;  // was this feature dropped by rate?
-	drawvec edge_nodes;    // what nodes at the tile edge were added during clipping?
+	drawvec edge_nodes;  // what nodes at the tile edge were added during clipping?
 
+#define FEATURE_DROPPED -1
+#define FEATURE_KEPT 0
+	// <0: dropped
+	//  0: kept
+	// >0: sequence number of additional feature kept by retain-points-multiplier
+	int dropped = FEATURE_DROPPED;	// was this feature dropped by rate?
+
+	// unsigned long long drop_by;  // dot-dropping priority
 	bool reduced;	   // is polygon dust
 	bool coalesced;	   // was coalesced from multiple features
 	int line_detail;   // current tile resolution being used for simplification

text.cpp

@@ -225,3 +225,31 @@ unsigned long long fnv1a(size_t size, void *p) {
 	}
 	return h;
 }
+
+// This function reverses the order of the bits in a 64-bit word.
+// Instead of shifting individual bits in a loop, it shifts them
+// in blocks, starting with swapping the halfwords, and working downward
+// until it is swapping individual pairs of adjacent bits.
+//
+// The purpose is to permute the order in which features are visited:
+// instead of working in an orderly fashion from the top left to the
+// bottom right of the tile, instead jump around to minimize adjacency,
+// like a hash function, but taking advantage of the knowledge that we
+// are operating on a fixed-size input that can be directly inverted.
+// https://en.wikipedia.org/wiki/Bit-reversal_permutation
+//
+// This allows calculating an appropriate set of features to appear
+// at a fractional zoom level: at what is effectively z4.25, for example,
+// we can bring in a quarter of the features that will be added in the
+// transition from z4 to z5, and have them be spatially distributed
+// across the tile rather than clumped together.
+
+unsigned long long bit_reverse(unsigned long long v) {
+	v = ((v & 0x00000000FFFFFFFF) << 32) | ((v & 0xFFFFFFFF00000000) >> 32);
+	v = ((v & 0x0000FFFF0000FFFF) << 16) | ((v & 0xFFFF0000FFFF0000) >> 16);
+	v = ((v & 0x00FF00FF00FF00FF) << 8) | ((v & 0xFF00FF00FF00FF00) >> 8);
+	v = ((v & 0x0F0F0F0F0F0F0F0F) << 4) | ((v & 0xF0F0F0F0F0F0F0F0) >> 4);
+	v = ((v & 0x3333333333333333) << 2) | ((v & 0xCCCCCCCCCCCCCCCC) >> 2);
+	v = ((v & 0x5555555555555555) << 1) | ((v & 0xAAAAAAAAAAAAAAAA) >> 1);
+	return v;
+}

text.hpp

@@ -11,5 +11,6 @@ std::string format_commandline(int argc, char **argv);
 unsigned long long fnv1a(std::string const &s);
 unsigned long long fnv1a(const char *s, char additional);
 unsigned long long fnv1a(size_t size, void *p);
+unsigned long long bit_reverse(unsigned long long v);
 
 #endif