Accept tilestats limiting options in tile-join, not just tippecanoe (#8)

felt · Aug 12, 2022 · 4ea8a37 · 4ea8a37
1 parent fec5e83
commit 4ea8a37
Show file tree

Hide file tree

Showing 11 changed files with 7,547 additions and 3 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 2.4.1
+
+* Accept tilestats limiting options in tile-join, not just tippecanoe
+
 ## 2.4.0
 
 * Change maxzoom guessing to take into account the standard deviation of the distances between features, so data with tight clusters will choose a higher maxzoom

diff --git a/Makefile b/Makefile
@@ -197,18 +197,27 @@ join-test: tile-join
 	./tile-join --quiet --force -o tests/join-population/joined.mbtiles -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles
 	./tile-join --quiet --force -o tests/join-population/joined-null.mbtiles --empty-csv-columns-are-null -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles
 	./tile-join --quiet --force --no-tile-stats -o tests/join-population/joined-no-tile-stats.mbtiles -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles
+	./tile-join --quiet --force --tile-stats-attributes-limit=1 -o tests/join-population/joined-tile-stats-attributes-limit.mbtiles -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles
+	./tile-join --quiet --force --tile-stats-sample-values-limit=1 -o tests/join-population/joined-tile-stats-sample-values-limit.mbtiles -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles
+	./tile-join --quiet --force --tile-stats-values-limit=1 -o tests/join-population/joined-tile-stats-values-limit.mbtiles -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles
 	./tile-join -q -f -i -o tests/join-population/joined-i.mbtiles -x GEOID10 -c tests/join-population/population.csv tests/join-population/tabblock_06001420.mbtiles
 	./tile-join -q -f -o tests/join-population/merged.mbtiles tests/join-population/tabblock_06001420.mbtiles tests/join-population/macarthur.mbtiles tests/join-population/macarthur2.mbtiles
 	./tile-join -q -f -c tests/join-population/windows.csv -o tests/join-population/windows.mbtiles tests/join-population/macarthur.mbtiles
 	./tippecanoe-decode -x generator --maximum-zoom=11 --minimum-zoom=4 tests/join-population/joined.mbtiles > tests/join-population/joined.mbtiles.json.check
 	./tippecanoe-decode -x generator --maximum-zoom=11 --minimum-zoom=4 tests/join-population/joined-null.mbtiles > tests/join-population/joined-null.mbtiles.json.check
 	./tippecanoe-decode -x generator --maximum-zoom=11 --minimum-zoom=4 tests/join-population/joined-no-tile-stats.mbtiles > tests/join-population/joined-no-tile-stats.mbtiles.json.check
+	./tippecanoe-decode -x generator --maximum-zoom=11 --minimum-zoom=4 tests/join-population/joined-tile-stats-attributes-limit.mbtiles > tests/join-population/joined-tile-stats-attributes-limit.mbtiles.json.check
+	./tippecanoe-decode -x generator --maximum-zoom=11 --minimum-zoom=4 tests/join-population/joined-tile-stats-values-limit.mbtiles > tests/join-population/joined-tile-stats-values-limit.mbtiles.json.check
+	./tippecanoe-decode -x generator --maximum-zoom=11 --minimum-zoom=4 tests/join-population/joined-tile-stats-sample-values-limit.mbtiles > tests/join-population/joined-tile-stats-sample-values-limit.mbtiles.json.check
 	./tippecanoe-decode -x generator tests/join-population/joined-i.mbtiles > tests/join-population/joined-i.mbtiles.json.check
 	./tippecanoe-decode -x generator tests/join-population/merged.mbtiles > tests/join-population/merged.mbtiles.json.check
 	./tippecanoe-decode -x generator tests/join-population/windows.mbtiles > tests/join-population/windows.mbtiles.json.check
 	cmp tests/join-population/joined.mbtiles.json.check tests/join-population/joined.mbtiles.json
 	cmp tests/join-population/joined-null.mbtiles.json.check tests/join-population/joined-null.mbtiles.json
 	cmp tests/join-population/joined-no-tile-stats.mbtiles.json.check tests/join-population/joined-no-tile-stats.mbtiles.json
+	cmp tests/join-population/joined-tile-stats-attributes-limit.mbtiles.json.check tests/join-population/joined-tile-stats-attributes-limit.mbtiles.json
+	cmp tests/join-population/joined-tile-stats-sample-values-limit.mbtiles.json.check tests/join-population/joined-tile-stats-sample-values-limit.mbtiles.json
+	cmp tests/join-population/joined-tile-stats-values-limit.mbtiles.json.check tests/join-population/joined-tile-stats-values-limit.mbtiles.json
 	cmp tests/join-population/joined-i.mbtiles.json.check tests/join-population/joined-i.mbtiles.json
 	cmp tests/join-population/merged.mbtiles.json.check tests/join-population/merged.mbtiles.json
 	cmp tests/join-population/windows.mbtiles.json.check tests/join-population/windows.mbtiles.json

diff --git a/README.md b/README.md
@@ -779,6 +779,9 @@ The options are:
  * `-pk` or `--no-tile-size-limit`: Don't skip tiles larger than 500K.
  * `-pC` or `--no-tile-compression`: Don't compress the PBF vector tile data.
  * `-pg` or `--no-tile-stats`: Don't generate the `tilestats` row in the tileset metadata. Uploads without [tilestats](https://github.com/mapbox/mapbox-geostats) will take longer to process.
+ * `--tile-stats-attributes-limit=`*count*: Include `tilestats` information about at most *count* attributes instead of the default 1000.
+ * `--tile-stats-sample-values-limit=`*count*: Calculate `tilestats` attribute statistics based on *count* values instead of the default 1000.
+ * `--tile-stats-values-limit=`*count*: Report *count* unique attribute values in `tilestats` instead of the default 100.
 
 Because tile-join just copies the geometries to the new .mbtiles without processing them
 (except to rescale the extents if necessary),

diff --git a/man/tippecanoe.1 b/man/tippecanoe.1
@@ -979,6 +979,12 @@ The options are:
 \fB\fC\-pC\fR or \fB\fC\-\-no\-tile\-compression\fR: Don't compress the PBF vector tile data.
 .IP \(bu 2
 \fB\fC\-pg\fR or \fB\fC\-\-no\-tile\-stats\fR: Don't generate the \fB\fCtilestats\fR row in the tileset metadata. Uploads without tilestats \[la]https://github.com/mapbox/mapbox-geostats\[ra] will take longer to process.
+.IP \(bu 2
+\fB\fC\-\-tile\-stats\-attributes\-limit=\fR\fIcount\fP: Include \fB\fCtilestats\fR information about at most \fIcount\fP attributes instead of the default 1000.
+.IP \(bu 2
+\fB\fC\-\-tile\-stats\-sample\-values\-limit=\fR\fIcount\fP: Calculate \fB\fCtilestats\fR attribute statistics based on \fIcount\fP values instead of the default 1000.
+.IP \(bu 2
+\fB\fC\-\-tile\-stats\-values\-limit=\fR\fIcount\fP: Report \fIcount\fP unique attribute values in \fB\fCtilestats\fR instead of the default 100.
 .RE
 .PP
 Because tile\-join just copies the geometries to the new .mbtiles without processing them

diff --git a/mbtiles.cpp b/mbtiles.cpp
@@ -500,6 +500,7 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
 				state.nospace = true;
 
 				bool first = true;
+				size_t attribute_count = 0;
 				for (auto j = fk->second.file_keys.begin(); j != fk->second.file_keys.end(); ++j) {
 					if (first) {
 						first = false;
@@ -526,6 +527,11 @@ void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fnam
 					} else {
 						state.json_write_string(f->second);
 					}
+
+					attribute_count++;
+					if (attribute_count >= max_tilestats_attributes) {
+						break;
+					}
 				}
 
 				state.nospace = true;