diff --git a/Makefile b/Makefile index a47d2b09..699716a4 100644 --- a/Makefile +++ b/Makefile @@ -367,15 +367,15 @@ overzoom-test: tippecanoe-overzoom cmp tests/pbf/bin-11-327-791.pbf.out.json.check tests/pbf/bin-11-327-791.pbf.out.json rm tests/pbf/bin-11-327-791.pbf.out.json.check tests/pbf/bin-11-327-791.pbf.out # Binning with longitude wraparound problems - ./tippecanoe-overzoom -o tests/pbf/0-0-0-pop-2-0-1.pbf.out --accumulate-numeric-attributes --assign-to-bins tests/pbf/h3-2-0-1.geojson tests/pbf/0-0-0.pbf 2/0/1 2/0/1 + ./tippecanoe-overzoom -o tests/pbf/0-0-0-pop-2-0-1.pbf.out --accumulate-numeric-attributes=tippecanoe --assign-to-bins tests/pbf/h3-2-0-1.geojson tests/pbf/0-0-0.pbf 2/0/1 2/0/1 ./tippecanoe-decode tests/pbf/0-0-0-pop-2-0-1.pbf.out 2 0 1 > tests/pbf/0-0-0-pop-2-0-1.pbf.out.json.check cmp tests/pbf/0-0-0-pop-2-0-1.pbf.out.json.check tests/pbf/0-0-0-pop-2-0-1.pbf.out.json rm tests/pbf/0-0-0-pop-2-0-1.pbf.out tests/pbf/0-0-0-pop-2-0-1.pbf.out.json.check - ./tippecanoe-overzoom -o tests/pbf/0-0-0-pop-1-1-0.pbf.out --accumulate-numeric-attributes --assign-to-bins tests/pbf/h3-1-1-0.geojson tests/pbf/0-0-0.pbf 1/1/0 1/1/0 + ./tippecanoe-overzoom -o tests/pbf/0-0-0-pop-1-1-0.pbf.out --accumulate-numeric-attributes=tippecanoe --assign-to-bins tests/pbf/h3-1-1-0.geojson tests/pbf/0-0-0.pbf 1/1/0 1/1/0 ./tippecanoe-decode tests/pbf/0-0-0-pop-1-1-0.pbf.out 1 1 0 > tests/pbf/0-0-0-pop-1-1-0.pbf.out.json.check cmp tests/pbf/0-0-0-pop-1-1-0.pbf.out.json.check tests/pbf/0-0-0-pop-1-1-0.pbf.out.json rm tests/pbf/0-0-0-pop-1-1-0.pbf.out tests/pbf/0-0-0-pop-1-1-0.pbf.out.json.check - ./tippecanoe-overzoom -o tests/pbf/0-0-0-pop-0-0-0.pbf.out --accumulate-numeric-attributes --assign-to-bins tests/pbf/h3-0-0-0.geojson tests/pbf/0-0-0.pbf 0/0/0 0/0/0 + ./tippecanoe-overzoom -o tests/pbf/0-0-0-pop-0-0-0.pbf.out --accumulate-numeric-attributes=tippecanoe --assign-to-bins tests/pbf/h3-0-0-0.geojson tests/pbf/0-0-0.pbf 0/0/0 0/0/0 ./tippecanoe-decode tests/pbf/0-0-0-pop-0-0-0.pbf.out 0 0 0 > tests/pbf/0-0-0-pop-0-0-0.pbf.out.json.check cmp tests/pbf/0-0-0-pop-0-0-0.pbf.out.json.check tests/pbf/0-0-0-pop-0-0-0.pbf.out.json rm tests/pbf/0-0-0-pop-0-0-0.pbf.out tests/pbf/0-0-0-pop-0-0-0.pbf.out.json.check @@ -520,7 +520,7 @@ accumulate-test: test `grep '"POP1950": [0-9]' tests/ne_110m_populated_places_nulls/in.json | wc -l` == 144 # and 99 without it test `grep '"POP1950": null' tests/ne_110m_populated_places_nulls/in.json | wc -l` == 99 - ./tippecanoe -yNAME -yPOP1950 -q -z3 -r1.75 -b0 -f -e tests/pbf/accum.dir --accumulate-numeric-attributes --set-attribute clustersize:1 --accumulate-attribute clustersize:sum --retain-points-multiplier 3 tests/ne_110m_populated_places_nulls/in.json + ./tippecanoe -yNAME -yPOP1950 -q -z3 -r1.75 -b0 -f -e tests/pbf/accum.dir --accumulate-numeric-attributes=tippecanoe --set-attribute clustersize:1 --accumulate-attribute clustersize:sum --retain-points-multiplier 3 tests/ne_110m_populated_places_nulls/in.json # at this drop rate, there are 6 points at z0 that have no POP1950s clustered onto them.... test `./tippecanoe-decode -c tests/pbf/accum.dir/0/0/0.pbf 0 0 0 | grep -v 'tippecanoe:count:POP1950' | wc -l` == 78 # 35 of which have no POP1950 at all @@ -544,7 +544,7 @@ accumulate-test: # which is the correct 161590 # # OK, so do these still hold after megatile filtering? - ./tippecanoe-overzoom --accumulate-numeric-attributes --accumulate-attribute clustersize:sum -m -o tests/pbf/accum-0-0-0.pbf tests/pbf/accum.dir/0/0/0.pbf 0/0/0 0/0/0 + ./tippecanoe-overzoom --accumulate-numeric-attributes=tippecanoe --accumulate-attribute clustersize:sum -m -o tests/pbf/accum-0-0-0.pbf tests/pbf/accum.dir/0/0/0.pbf 0/0/0 0/0/0 # Now there are 40 features with POP1950 clusters test `./tippecanoe-decode -c tests/pbf/accum-0-0-0.pbf 0 0 0 | grep 'tippecanoe:count:POP1950' | wc -l` == 40 # There are 4 with bare POP1950 @@ -560,7 +560,7 @@ accumulate-test: # which add up to 161590 so we have the right global total # # Now on to binning! - ./tippecanoe-overzoom --assign-to-bins tests/pbf/h3-0-0-0.geojson --accumulate-numeric-attributes --accumulate-attribute clustersize:sum -m -o tests/pbf/bins-0-0-0.pbf tests/pbf/accum.dir/0/0/0.pbf 0/0/0 0/0/0 + ./tippecanoe-overzoom --assign-to-bins tests/pbf/h3-0-0-0.geojson --accumulate-numeric-attributes=tippecanoe --accumulate-attribute clustersize:sum -m -o tests/pbf/bins-0-0-0.pbf tests/pbf/accum.dir/0/0/0.pbf 0/0/0 0/0/0 # Now there are 30 bins with POP1950 clusters echo test `./tippecanoe-decode -c tests/pbf/bins-0-0-0.pbf 0 0 0 | grep 'tippecanoe:count:POP1950' | wc -l` == 30 # There are none with bare POP1950 (which is expected; we should only have summary statistics) diff --git a/clip.cpp b/clip.cpp index 1b2b1452..12ee8c73 100644 --- a/clip.cpp +++ b/clip.cpp @@ -1082,7 +1082,7 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int n std::unordered_map const &attribute_accum, std::vector const &unidecode_data, double simplification, double tiny_polygon_size, std::vector const &bins, - bool accumulate_numeric) { + std::string const &accumulate_numeric) { std::vector decoded; for (auto const &t : tiles) { @@ -1122,20 +1122,24 @@ struct tile_feature { size_t seq = 0; }; -static void add_mean(mvt_feature &feature, mvt_layer &layer) { +static void add_mean(mvt_feature &feature, mvt_layer &layer, std::string const &accumulate_numeric) { + std::string accumulate_numeric_colon = accumulate_numeric + ":"; + std::unordered_map attributes; for (size_t i = 0; i + 1 < feature.tags.size(); i += 2) { std::string const &key = layer.keys[feature.tags[i]]; - if (starts_with(key, "tippecanoe:")) { + if (starts_with(key, accumulate_numeric_colon)) { attributes.emplace(key, i); } } for (size_t i = 0; i + 1 < feature.tags.size(); i += 2) { + std::string accumulate_numeric_sum_colon = accumulate_numeric + ":sum:"; + std::string const &key = layer.keys[feature.tags[i]]; - if (starts_with(key, "tippecanoe:sum:")) { - std::string trunc = key.substr(15); - auto const f = attributes.find("tippecanoe:count:" + trunc); + if (starts_with(key, accumulate_numeric_sum_colon)) { + std::string trunc = key.substr(accumulate_numeric_sum_colon.size()); + auto const f = attributes.find(accumulate_numeric + ":count:" + trunc); if (f != attributes.end()) { mvt_value const &sum = layer.values[feature.tags[i + 1]]; mvt_value const &count = layer.values[feature.tags[f->second + 1]]; @@ -1147,7 +1151,7 @@ static void add_mean(mvt_feature &feature, mvt_layer &layer) { mvt_value mean; mean.type = mvt_double; mean.numeric_value.double_value = mvt_value_to_double(sum) / count_val; - layer.tag(feature, "tippecanoe:mean:" + trunc, mean); + layer.tag(feature, accumulate_numeric + ":mean:" + trunc, mean); } } } @@ -1157,7 +1161,7 @@ static void feature_out(std::vector const &features, mvt_layer &ou std::set const &keep, std::unordered_map const &attribute_accum, std::shared_ptr const &tile_stringpool, - bool accumulate_numeric) { + std::string const &accumulate_numeric) { // Add geometry to output feature mvt_feature outfeature; @@ -1176,7 +1180,7 @@ static void feature_out(std::vector const &features, mvt_layer &ou outfeature.seq = features[0].seq; - if (attribute_accum.size() > 0 || accumulate_numeric) { + if (attribute_accum.size() > 0 || accumulate_numeric.size() > 0) { // convert the attributes of the output feature // from mvt_value to serial_val so they can have // attributes from the other features of the @@ -1194,7 +1198,7 @@ static void feature_out(std::vector const &features, mvt_layer &ou // this attribute has an accumulator, so convert it full_keys.push_back(features[0].layer->keys[features[0].tags[i]]); full_values.push_back(mvt_value_to_serial_val(features[0].layer->values[features[0].tags[i + 1]])); - } else if (accumulate_numeric && features[0].layer->values[features[0].tags[i + 1]].is_numeric()) { + } else if (accumulate_numeric.size() > 0 && features[0].layer->values[features[0].tags[i + 1]].is_numeric()) { // convert numeric for accumulation numeric_out_field.emplace(key, full_keys.size()); full_keys.push_back(key); @@ -1226,14 +1230,14 @@ static void feature_out(std::vector const &features, mvt_layer &ou if (f != attribute_accum.end()) { serial_val val = mvt_value_to_serial_val(features[i].layer->values[features[i].tags[j + 1]]); preserve_attribute(f->second, key, val, full_keys, full_values, attribute_accum_state); - } else if (accumulate_numeric) { + } else if (accumulate_numeric.size() > 0) { const mvt_value &val = features[i].layer->values[features[i].tags[j + 1]]; if (val.is_numeric()) { // If this is a numeric attribute, but there is also a tippecanoe:sum (etc.) for the // same attribute, we want to use that one instead of this one. for (auto const &op : numeric_operations) { - std::string compound_key = "tippecanoe:" + op.first + ":" + key; + std::string compound_key = accumulate_numeric + ":" + op.first + ":" + key; auto compound_found = keys.find(compound_key); if (compound_found != keys.end()) { // found, so skip this one @@ -1245,8 +1249,8 @@ static void feature_out(std::vector const &features, mvt_layer &ou // it is the wrong one. std::string outkey = key; - if (starts_with(outkey, "tippecanoe:")) { - std::string prefix = "tippecanoe:" + op.first + ":"; + if (starts_with(outkey, accumulate_numeric + ":")) { + std::string prefix = accumulate_numeric + ":" + op.first + ":"; if (starts_with(outkey, prefix)) { outkey = outkey.substr(prefix.size()); } else { @@ -1254,7 +1258,7 @@ static void feature_out(std::vector const &features, mvt_layer &ou } } // and then put it back on for the output field - std::string prefixed = "tippecanoe:" + op.first + ":" + outkey; + std::string prefixed = accumulate_numeric + ":" + op.first + ":" + outkey; // Does it exist in the output feature already? @@ -1310,8 +1314,8 @@ static void feature_out(std::vector const &features, mvt_layer &ou } } - if (accumulate_numeric) { - add_mean(outfeature, outlayer); + if (accumulate_numeric.size() > 0) { + add_mean(outfeature, outlayer, accumulate_numeric); } } else { for (size_t i = 0; i + 1 < features[0].tags.size(); i += 2) { @@ -1433,7 +1437,7 @@ static bool bbox_intersects(long long x1min, long long y1min, long long x1max, l } mvt_tile assign_to_bins(mvt_tile const &features, std::vector const &bins, int z, int x, int y, int detail, - bool accumulate_numeric) { + std::string const &accumulate_numeric) { std::vector events; // Index bins @@ -1537,7 +1541,14 @@ mvt_tile assign_to_bins(mvt_tile const &features, std::vector const & mvt_value val; val.type = mvt_uint; val.numeric_value.uint_value = outfeatures[found->outfeature].size() - 1; - outlayer.tag(nfeature, "tippecanoe:count", val); + + std::string attrname; + if (accumulate_numeric.size() == 0) { + attrname = "tippecanoe:count"; + } else { + attrname = accumulate_numeric + ":count"; + } + outlayer.tag(nfeature, attrname, val); } active.erase(found); @@ -1560,7 +1571,7 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int std::unordered_map const &attribute_accum, std::vector const &unidecode_data, double simplification, double tiny_polygon_size, std::vector const &bins, - bool accumulate_numeric) { + std::string const &accumulate_numeric) { mvt_tile outtile; std::shared_ptr tile_stringpool = std::make_shared(); diff --git a/geometry.hpp b/geometry.hpp index cac84e82..c9401730 100644 --- a/geometry.hpp +++ b/geometry.hpp @@ -122,7 +122,7 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int std::unordered_map const &attribute_accum, std::vector const &unidecode_data, double simplification, double tiny_polygon_size, std::vector const &bins, - bool accumulate_numeric); + std::string const &accumulate_numeric); std::string overzoom(std::vector const &tiles, int nz, int nx, int ny, int detail, int buffer, std::set const &keep, bool do_compress, @@ -131,7 +131,7 @@ std::string overzoom(std::vector const &tiles, int nz, int nx, int n std::unordered_map const &attribute_accum, std::vector const &unidecode_data, double simplification, double tiny_polygon_size, std::vector const &bins, - bool accumulate_numeric); + std::string const &accumulate_numeric); draw center_of_mass_mp(const drawvec &dv); diff --git a/main.cpp b/main.cpp index 940480ab..9a912dbd 100644 --- a/main.cpp +++ b/main.cpp @@ -97,6 +97,7 @@ long long extend_zooms_max = 0; int retain_points_multiplier = 1; std::vector unidecode_data; size_t maximum_string_attribute_length = 0; +std::string accumulate_numeric; std::vector order_by; bool order_reverse; @@ -3084,7 +3085,7 @@ int main(int argc, char **argv) { {"attribute-type", required_argument, 0, 'T'}, {"attribute-description", required_argument, 0, 'Y'}, {"accumulate-attribute", required_argument, 0, 'E'}, - {"accumulate-numeric-attributes", no_argument, &additional[A_ACCUMULATE_NUMERIC], 1}, + {"accumulate-numeric-attributes", required_argument, 0, '~'}, {"empty-csv-columns-are-null", no_argument, &prevent[P_EMPTY_CSV_COLUMNS], 1}, {"convert-stringified-ids-to-numbers", no_argument, &additional[A_CONVERT_NUMERIC_IDS], 1}, {"use-attribute-for-id", required_argument, 0, '~'}, @@ -3327,6 +3328,8 @@ int main(int argc, char **argv) { unidecode_data = read_unidecode(optarg); } else if (strcmp(opt, "maximum-string-attribute-length") == 0) { maximum_string_attribute_length = atoll_require(optarg, "Maximum string attribute length"); + } else if (strcmp(opt, "accumulate-numeric-attributes") == 0) { + accumulate_numeric = optarg; } else { fprintf(stderr, "%s: Unrecognized option --%s\n", argv[0], opt); exit(EXIT_ARGS); diff --git a/main.hpp b/main.hpp index a25d04be..b5844d91 100644 --- a/main.hpp +++ b/main.hpp @@ -64,6 +64,7 @@ extern std::map set_attributes; extern long long extend_zooms_max; extern int retain_points_multiplier; extern size_t maximum_string_attribute_length; +extern std::string accumulate_numeric; struct order_field { std::string name; diff --git a/options.hpp b/options.hpp index 5d7c15a7..ce50ef80 100644 --- a/options.hpp +++ b/options.hpp @@ -12,7 +12,6 @@ #define A_PREFER_RADIX_SORT ((int) 'R') #define A_COALESCE_DENSEST_AS_NEEDED ((int) 'S') #define A_CALCULATE_INDEX ((int) 'X') -#define A_ACCUMULATE_NUMERIC ((int) 'a') #define A_DETECT_SHARED_BORDERS ((int) 'b') #define A_COALESCE ((int) 'c') #define A_DROP_FRACTION_AS_NEEDED ((int) 'd') diff --git a/overzoom.cpp b/overzoom.cpp index 05714b32..ef296a3f 100644 --- a/overzoom.cpp +++ b/overzoom.cpp @@ -22,7 +22,7 @@ bool preserve_input_order = false; std::unordered_map attribute_accum; std::vector unidecode_data; std::vector bins; -bool accumulate_numeric = false; +std::string accumulate_numeric; std::set keep; @@ -58,7 +58,7 @@ int main(int argc, char **argv) { {"tiny-polygon-size", required_argument, 0, 's' & 0x1F}, {"source-tile", required_argument, 0, 't'}, {"assign-to-bins", required_argument, 0, 'b' & 0x1F}, - {"accumulate-numeric-attributes", no_argument, 0, 'a' & 0x1F}, + {"accumulate-numeric-attributes", required_argument, 0, 'a' & 0x1F}, {0, 0, 0, 0}, }; @@ -130,7 +130,7 @@ int main(int argc, char **argv) { break; case 'a' & 0x1F: - accumulate_numeric = true; + accumulate_numeric = optarg; break; default: diff --git a/tile-join.cpp b/tile-join.cpp index 2e74fd53..0850fb89 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -712,7 +712,7 @@ struct tileset_reader { t.y = parent_tile.y; tv.push_back(std::move(t)); - std::string ret = overzoom(tv, tile.z, tile.x, tile.y, -1, buffer, std::set(), false, &next_overzoomed_tiles, false, NULL, false, std::unordered_map(), unidecode_data, 0, 0, std::vector(), false); + std::string ret = overzoom(tv, tile.z, tile.x, tile.y, -1, buffer, std::set(), false, &next_overzoomed_tiles, false, NULL, false, std::unordered_map(), unidecode_data, 0, 0, std::vector(), ""); return ret; } diff --git a/tile.cpp b/tile.cpp index e5fe8a2a..1a1466fe 100644 --- a/tile.cpp +++ b/tile.cpp @@ -1472,13 +1472,13 @@ void preserve_attributes(std::unordered_map const *at promote_attribute(key, p); preserve_attribute(f->second, key, sv, p.full_keys, p.full_values, p.attribute_accum_state); - } else if (type == mvt_double && additional[A_ACCUMULATE_NUMERIC]) { + } else if (type == mvt_double && accumulate_numeric.size() > 0) { for (auto const &operation : numeric_operations) { serial_val sv; sv.type = sf.stringpool[sf.values[i]]; sv.s = sf.stringpool + sf.values[i] + 1; - std::string prefixed_key = "tippecanoe:" + operation.first + ":" + key; + std::string prefixed_key = accumulate_numeric + ":" + operation.first + ":" + key; promote_attribute_prefix(key, prefixed_key, p); preserve_attribute(operation.second, prefixed_key, sv, p.full_keys, p.full_values, p.attribute_accum_state); } @@ -1494,9 +1494,9 @@ void preserve_attributes(std::unordered_map const *at promote_attribute(key, p); // promotes it in the target feature preserve_attribute(f->second, key, sv, p.full_keys, p.full_values, p.attribute_accum_state); - } else if (type == mvt_double && additional[A_ACCUMULATE_NUMERIC]) { + } else if (type == mvt_double && accumulate_numeric.size() > 0) { for (auto const &operation : numeric_operations) { - std::string prefixed_key = "tippecanoe:" + operation.first + ":" + key; + std::string prefixed_key = accumulate_numeric + ":" + operation.first + ":" + key; promote_attribute_prefix(key, prefixed_key, p); preserve_attribute(operation.second, prefixed_key, sf.full_values[i], p.full_keys, p.full_values, p.attribute_accum_state); }