From 3456fa75eddfa919bec068671787d3e352ef31ee Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Thu, 19 Sep 2024 13:02:56 -0700 Subject: [PATCH] Implement and test attribute prefix stripping in overzoom --- Makefile | 14 +++++++++++++- clip.cpp | 27 ++++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 72f7bbfa..aa0564fc 100644 --- a/Makefile +++ b/Makefile @@ -97,7 +97,7 @@ indent: TESTS = $(wildcard tests/*/out/*.json) SPACE = $(NULL) $(NULL) -test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) raw-tiles-test parallel-test pbf-test join-test enumerate-test decode-test join-filter-test unit json-tool-test allow-existing-test csv-test layer-json-test pmtiles-test decode-pmtiles-test overzoom-test +test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) raw-tiles-test parallel-test pbf-test join-test enumerate-test decode-test join-filter-test unit json-tool-test allow-existing-test csv-test layer-json-test pmtiles-test decode-pmtiles-test overzoom-test accumulate-test ./unit suffixes = json json.gz @@ -574,7 +574,19 @@ accumulate-test: test `./tippecanoe-decode -c tests/pbf/accum-0-0-0.pbf 0 0 0 | grep sum:clustered:unrelated | wc -l` == 0 # But that we *do* preserve those attributes into the output features: test `./tippecanoe-decode -c tests/pbf/accum-0-0-0.pbf 0 0 0 | grep clustered:unrelated | wc -l` == 22 + # the cluster sizes still add up to the 243 original features + test `./tippecanoe-decode -c tests/pbf/accum-0-0-0.pbf 0 0 0 | sed 's/.*clustered:cluster_size": //' | awk '{sum += $$1} END {print sum}'` == 243 # + # We actually want to serve point tiles without the numeric accumulations, + # but with cluster size, so test that combination: + ./tippecanoe-overzoom --accumulate-attribute '{"clustered:cluster_size":"sum"}' --exclude-prefix clustered:sum --exclude-prefix clustered:count --exclude-prefix clustered:min --exclude-prefix clustered:max --exclude-prefix clustered:mean -m -o tests/pbf/accum-0-0-0.pbf tests/pbf/accum.dir/0/0/0.pbf 0/0/0 0/0/0 + # There are no POP1950 clusters + test `./tippecanoe-decode -c tests/pbf/accum-0-0-0.pbf 0 0 0 | grep 'clustered:count:POP1950' | wc -l` == 0 + # But there are still 28 with bare POP1950 + test `./tippecanoe-decode -c tests/pbf/accum-0-0-0.pbf 0 0 0 | grep -v 'clustered:count:POP1950' | grep 'POP1950' | wc -l` == 28 + # And 18 with no POP1950 at all + test `./tippecanoe-decode -c tests/pbf/accum-0-0-0.pbf 0 0 0 | grep -v 'POP1950' | wc -l` == 18 + # which matches the 46 features that you get if you tile without --retain-points-multiplier. # the cluster sizes still add up to the 243 original features test `./tippecanoe-decode -c tests/pbf/accum-0-0-0.pbf 0 0 0 | sed 's/.*clustered:cluster_size": //' | awk '{sum += $$1} END {print sum}'` == 243 # diff --git a/clip.cpp b/clip.cpp index 5663e06d..c1bf81f1 100644 --- a/clip.cpp +++ b/clip.cpp @@ -1239,6 +1239,27 @@ static void preserve_numeric(const std::string &key, const mvt_value &val, / } } +static bool should_keep(std::string const &key, + std::set const &keep, + std::set const &exclude, + std::vector const &exclude_prefix) { + if (keep.size() == 0 || keep.find(key) != keep.end()) { + if (exclude.find(key) != exclude.end()) { + return false; + } + + for (auto const &prefix : exclude_prefix) { + if (starts_with(key, prefix)) { + return false; + } + } + + return true; + } + + return false; +} + static void feature_out(std::vector const &features, mvt_layer &outlayer, std::set const &keep, std::set const &exclude, @@ -1289,7 +1310,7 @@ static void feature_out(std::vector const &features, mvt_layer &ou full_values.push_back(mvt_value_to_serial_val(features[0].layer->values[features[0].tags[i + 1]])); } else { // otherwise just tag it directly onto the output feature - if (keep.size() == 0 || keep.find(features[0].layer->keys[features[0].tags[i]]) != keep.end()) { + if (should_keep(features[0].layer->keys[features[0].tags[i]], keep, exclude, exclude_prefix)) { outlayer.tag(outfeature, features[0].layer->keys[features[0].tags[i]], features[0].layer->values[features[0].tags[i + 1]]); } } @@ -1329,7 +1350,7 @@ static void feature_out(std::vector const &features, mvt_layer &ou // and tag them onto the output feature for (size_t i = 0; i < full_keys.size(); i++) { - if (keep.size() == 0 || keep.find(full_keys[i]) != keep.end()) { + if (should_keep(full_keys[i], keep, exclude, exclude_prefix)) { outlayer.tag(outfeature, full_keys[i], stringified_to_mvt_value(full_values[i].type, full_values[i].s.c_str(), tile_stringpool)); } } @@ -1339,7 +1360,7 @@ static void feature_out(std::vector const &features, mvt_layer &ou } } else { for (size_t i = 0; i + 1 < features[0].tags.size(); i += 2) { - if (keep.size() == 0 || keep.find(features[0].layer->keys[features[0].tags[i]]) != keep.end()) { + if (should_keep(features[0].layer->keys[features[0].tags[i]], keep, exclude, exclude_prefix)) { outlayer.tag(outfeature, features[0].layer->keys[features[0].tags[i]], features[0].layer->values[features[0].tags[i + 1]]); } }