Skip to content

Commit

Permalink
Try adding --drop-sparsest-as-needed
Browse files Browse the repository at this point in the history
  • Loading branch information
@e-n-f
e-n-f committed Aug 9, 2024
1 parent d891ca7 commit 459aef1
Show file tree
Hide file tree
Showing 4 changed files with 64 additions and 7 deletions.
2 changes: 2 additions & 0 deletions main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3113,9 +3113,11 @@ int main(int argc, char **argv) {
{"drop-densest-as-needed", no_argument, &additional[A_DROP_DENSEST_AS_NEEDED], 1},
{"drop-fraction-as-needed", no_argument, &additional[A_DROP_FRACTION_AS_NEEDED], 1},
{"drop-smallest-as-needed", no_argument, &additional[A_DROP_SMALLEST_AS_NEEDED], 1},
{"drop-sparsest-as-needed", no_argument, &additional[A_DROP_SPARSEST_AS_NEEDED], 1},
{"coalesce-densest-as-needed", no_argument, &additional[A_COALESCE_DENSEST_AS_NEEDED], 1},
{"coalesce-fraction-as-needed", no_argument, &additional[A_COALESCE_FRACTION_AS_NEEDED], 1},
{"coalesce-smallest-as-needed", no_argument, &additional[A_COALESCE_SMALLEST_AS_NEEDED], 1},
{"coalesce-sparsest-as-needed", no_argument, &additional[A_COALESCE_SPARSEST_AS_NEEDED], 1},
{"force-feature-limit", no_argument, &prevent[P_DYNAMIC_DROP], 1},
{"cluster-densest-as-needed", no_argument, &additional[A_CLUSTER_DENSEST_AS_NEEDED], 1},

Expand Down
2 changes: 2 additions & 0 deletions options.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,11 @@
#define A_DROP_DENSEST_AS_NEEDED ((int) 's')
#define A_DROP_FRACTION_AS_NEEDED ((int) 'd')
#define A_DROP_SMALLEST_AS_NEEDED ((int) 'n')
#define A_DROP_SPARSEST_AS_NEEDED ((int) 's' & 0x1F)
#define A_COALESCE_DENSEST_AS_NEEDED ((int) 'S')
#define A_COALESCE_SMALLEST_AS_NEEDED ((int) 'N')
#define A_COALESCE_FRACTION_AS_NEEDED ((int) 'D')
#define A_COALESCE_SPARSEST_AS_NEEDED ((int) 't' & 0x1F)
#define A_GRID_LOW_ZOOMS ((int) 'L')
#define A_DETECT_WRAPAROUND ((int) 'w')
#define A_EXTEND_ZOOMS ((int) 'e')
Expand Down
4 changes: 3 additions & 1 deletion serial.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -681,7 +681,7 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf, std::
// keep old behavior, which loses one bit of precision at the bottom
midx = (sf.bbox[0] / 2 + sf.bbox[2] / 2) & ((1LL << 32) - 1);
midy = (sf.bbox[1] / 2 + sf.bbox[3] / 2) & ((1LL << 32) - 1);
} else if ((additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_COALESCE_DENSEST_AS_NEEDED]) && sf.t == VT_POLYGON) {
} else if ((additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_COALESCE_DENSEST_AS_NEEDED] || additional[A_DROP_SPARSEST_AS_NEEDED] || additional[A_COALESCE_SPARSEST_AS_NEEDED]) && sf.t == VT_POLYGON) {
// This probably should really apply to all polygons,
// but I hate to change the feature sequence in all the
// test fixtures again
Expand Down Expand Up @@ -725,6 +725,8 @@ int serialize_feature(struct serialization_state *sst, serial_feature &sf, std::

if (additional[A_DROP_DENSEST_AS_NEEDED] ||
additional[A_COALESCE_DENSEST_AS_NEEDED] ||
additional[A_DROP_SPARSEST_AS_NEEDED] ||
additional[A_COALESCE_SPARSEST_AS_NEEDED] ||
additional[A_CLUSTER_DENSEST_AS_NEEDED] ||
additional[A_CALCULATE_FEATURE_DENSITY] ||
additional[A_DROP_SMALLEST_AS_NEEDED] ||
Expand Down
63 changes: 57 additions & 6 deletions tile.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -772,6 +772,17 @@ static unsigned long long choose_mingap(std::vector<unsigned long long> &gaps, d
return gaps[ix];
}

static unsigned long long choose_maxgap(std::vector<unsigned long long> &gaps, double f, unsigned long long existing_gap) {
std::stable_sort(gaps.begin(), gaps.end());

size_t ix = (gaps.size() - 1) * f;
while (ix > 0 && gaps[ix] == existing_gap) {
ix--;
}

return gaps[ix];
}

// This function is called to choose the new "extent" threshold to try when a tile exceeds the
// tile size limit or feature limit and `--drop-smallest-as-needed` or `--coalesce-smallest-as-needed`
// has been set.
Expand Down Expand Up @@ -888,6 +899,8 @@ struct write_tile_args {
size_t pass = 0;
unsigned long long mingap = 0;
unsigned long long mingap_out = 0;
unsigned long long maxgap = 0;
unsigned long long maxgap_out = 0;
long long minextent = 0;
long long minextent_out = 0;
unsigned long long mindrop_sequence = 0;
Expand Down Expand Up @@ -1547,9 +1560,10 @@ void skip_tile(decompressor *geoms, std::atomic<long long> *geompos_in, bool com
}
}

long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic<long long> *geompos, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, long long minextent, unsigned long long mindrop_sequence, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy_out, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, std::vector<std::string> const &unidecode_data, long long estimated_complexity, std::set<zxy> &skip_children_out) {
long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, char *global_stringpool, int z, const unsigned tx, const unsigned ty, const int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, compressor **geomfile, std::atomic<long long> *geompos, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, unsigned long long mingap, unsigned long long maxgap, long long minextent, unsigned long long mindrop_sequence, const char *prefilter, const char *postfilter, json_object *filter, write_tile_args *arg, atomic_strategy *strategy_out, bool compressed_input, node *shared_nodes_map, size_t nodepos, std::string const &shared_nodes_bloom, std::vector<std::string> const &unidecode_data, long long estimated_complexity, std::set<zxy> &skip_children_out) {
double merge_fraction = 1;
double mingap_fraction = 1;
double maxgap_fraction = 1;
double minextent_fraction = 1;
double mindrop_sequence_fraction = 1;

Expand Down Expand Up @@ -1864,9 +1878,9 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
can_stop_early = false;
continue;
}
} else if (additional[A_DROP_DENSEST_AS_NEEDED]) {
} else if (additional[A_DROP_DENSEST_AS_NEEDED] || additional[A_DROP_SPARSEST_AS_NEEDED]) {
add_sample_to(gaps, sf.gap, gaps_increment, seq);
if (sf.gap < mingap) {
if (sf.gap < mingap || sf.gap > maxgap) {
if (drop_feature_unless_it_can_be_added_to_a_multiplier_cluster(layer, sf, layer_unmaps, multiplier_seq, strategy, drop_rest, arg->attribute_accum)) {
can_stop_early = false;
continue;
Expand Down Expand Up @@ -1895,9 +1909,9 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
drop_rest = true;
continue;
}
} else if (additional[A_COALESCE_DENSEST_AS_NEEDED]) {
} else if (additional[A_COALESCE_DENSEST_AS_NEEDED] || additional[A_COALESCE_SPARSEST_AS_NEEDED]) {
add_sample_to(gaps, sf.gap, gaps_increment, seq);
if (sf.gap < mingap && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX, multiplier_seq)) {
if ((sf.gap < mingap || sf.gap > maxgap) && find_feature_to_accumulate_onto(features, sf, which_serial_feature, layer_unmaps, LLONG_MAX, multiplier_seq)) {
coalesce_geometry(features[which_serial_feature], sf);
features[which_serial_feature].coalesced = true;
coalesced_area += sf.extent;
Expand Down Expand Up @@ -2529,6 +2543,21 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
line_detail++;
continue;
}
} else if (maxgap > 0 && (additional[A_DROP_SPARSEST_AS_NEEDED] || additional[A_COALESCE_SPARSEST_AS_NEEDED])) {
maxgap_fraction = maxgap_fraction * scaled_max_tile_features / totalsize * 0.80;
unsigned long long m = choose_maxgap(gaps, maxgap_fraction, maxgap);
if (m != maxgap) {
maxgap = m;
if (maxgap < arg->maxgap_out) {
arg->maxgap_out = maxgap;
arg->still_dropping = true;
}
if (!quiet) {
fprintf(stderr, "Going to try keeping the densest %0.2f%% of the features to make it fit\n", maxgap_fraction * 100.0);
}
line_detail++;
continue;
}
} else if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
minextent_fraction = minextent_fraction * scaled_max_tile_features / totalsize * 0.75;
long long m = choose_minextent(extents, minextent_fraction, minextent);
Expand Down Expand Up @@ -2641,6 +2670,21 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
line_detail++;
continue;
}
} else if (maxgap > 0 && (additional[A_DROP_SPARSEST_AS_NEEDED] || additional[A_COALESCE_SPARSEST_AS_NEEDED])) {
maxgap_fraction = maxgap_fraction * scaled_max_tile_size / (kept_adjust * compressed.size()) * 0.80;
unsigned long long m = choose_maxgap(gaps, maxgap_fraction, maxgap);
if (m != maxgap) {
maxgap = m;
if (maxgap < arg->maxgap_out) {
arg->maxgap_out = maxgap;
arg->still_dropping = true;
}
if (!quiet) {
fprintf(stderr, "Going to try keeping the densest %0.2f%% of the features to make it fit\n", maxgap_fraction * 100.0);
}
line_detail++;
continue;
}
} else if (additional[A_DROP_SMALLEST_AS_NEEDED] || additional[A_COALESCE_SMALLEST_AS_NEEDED]) {
minextent_fraction = minextent_fraction * scaled_max_tile_size / (kept_adjust * compressed.size()) * 0.75;
long long m = choose_minextent(extents, minextent_fraction, minextent);
Expand Down Expand Up @@ -2810,7 +2854,7 @@ exit(EXIT_IMPOSSIBLE);
skip_tile(&dc, &geompos, arg->compressed);
len = 1;
} else {
len = write_tile(&dc, &geompos, arg->global_stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->geompos, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->minextent, arg->mindrop_sequence, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->shared_nodes_map, arg->nodepos, *(arg->shared_nodes_bloom), (*arg->unidecode_data), estimated_complexity, arg->skip_children_out);
len = write_tile(&dc, &geompos, arg->global_stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->outdb, arg->outdir, arg->buffer, arg->fname, arg->geomfile, arg->geompos, arg->minzoom, arg->maxzoom, arg->todo, arg->along, geompos, arg->gamma, arg->child_shards, arg->pool_off, arg->initial_x, arg->initial_y, arg->running, arg->simplification, arg->layermaps, arg->layer_unmaps, arg->tiling_seg, arg->pass, arg->mingap, arg->maxgap, arg->minextent, arg->mindrop_sequence, arg->prefilter, arg->postfilter, arg->filter, arg, arg->strategy, arg->compressed, arg->shared_nodes_map, arg->nodepos, *(arg->shared_nodes_bloom), (*arg->unidecode_data), estimated_complexity, arg->skip_children_out);
}

if (pthread_mutex_lock(&var_lock) != 0) {
Expand Down Expand Up @@ -2987,6 +3031,7 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::

double zoom_gamma = gamma;
unsigned long long zoom_mingap = 0;
unsigned long long zoom_maxgap = ULLONG_MAX;
long long zoom_minextent = 0;
unsigned long long zoom_mindrop_sequence = 0;
size_t zoom_tile_size = 0;
Expand Down Expand Up @@ -3020,7 +3065,9 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::
args[thread].gamma = zoom_gamma;
args[thread].gamma_out = zoom_gamma;
args[thread].mingap = zoom_mingap;
args[thread].maxgap = zoom_maxgap;
args[thread].mingap_out = zoom_mingap;
args[thread].maxgap_out = zoom_maxgap;
args[thread].minextent = zoom_minextent;
args[thread].minextent_out = zoom_minextent;
args[thread].mindrop_sequence = zoom_mindrop_sequence;
Expand Down Expand Up @@ -3103,6 +3150,10 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *global_stringpool, std::
zoom_mingap = args[thread].mingap_out;
again = true;
}
if (args[thread].maxgap_out < zoom_maxgap) {
zoom_maxgap = args[thread].maxgap_out;
again = true;
}
if (args[thread].minextent_out > zoom_minextent) {
zoom_minextent = args[thread].minextent_out;
again = true;
Expand Down

0 comments on commit 459aef1

Please sign in to comment.