Skip to content

Commit

Permalink
Once features can't possibly fit in a tile, stop trying (#9)
Browse files Browse the repository at this point in the history 
* Stop adding features to a tile if it can't possibly work

* Add --integer and --fraction options to tippecanoe-decode

* Carry the strategies field from tileset metadata through tile-join

* Update changelog

* Assign different codes to different kinds of error exits
  • Loading branch information
@e-n-f
e-n-f authored Sep 24, 2022
1 parent ed31b9a commit af1a7ed
Show file tree
Hide file tree
Showing 44 changed files with 783 additions and 489 deletions.
8 changes: 7 additions & 1 deletion CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
## 2.6.2

* Stop adding features to a tile if it can't possibly work, to limit memory use
* Add --integer and --fraction options to tippecanoe-decode
* Carry `strategies` field from tileset metadata through tile-join

## 2.6.1

Upgrade protozero to version 1.7.1
* Upgrade protozero to version 1.7.1

## 2.6.0

Expand Down
4 changes: 4 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,10 +160,14 @@ decode-test:
mkdir -p tests/muni/decode
./tippecanoe -q -z11 -Z11 -f -o tests/muni/decode/multi.mbtiles tests/muni/*.json
./tippecanoe-decode -x generator -l subway tests/muni/decode/multi.mbtiles > tests/muni/decode/multi.mbtiles.json.check
./tippecanoe-decode -x generator -l subway --integer tests/muni/decode/multi.mbtiles > tests/muni/decode/multi.mbtiles.integer.json.check
./tippecanoe-decode -x generator -l subway --fraction tests/muni/decode/multi.mbtiles > tests/muni/decode/multi.mbtiles.fraction.json.check
./tippecanoe-decode -x generator -c tests/muni/decode/multi.mbtiles > tests/muni/decode/multi.mbtiles.pipeline.json.check
./tippecanoe-decode -x generator tests/muni/decode/multi.mbtiles 11 327 791 > tests/muni/decode/multi.mbtiles.onetile.json.check
./tippecanoe-decode -x generator --stats tests/muni/decode/multi.mbtiles > tests/muni/decode/multi.mbtiles.stats.json.check
cmp tests/muni/decode/multi.mbtiles.json.check tests/muni/decode/multi.mbtiles.json
cmp tests/muni/decode/multi.mbtiles.integer.json.check tests/muni/decode/multi.mbtiles.integer.json
cmp tests/muni/decode/multi.mbtiles.fraction.json.check tests/muni/decode/multi.mbtiles.fraction.json
cmp tests/muni/decode/multi.mbtiles.pipeline.json.check tests/muni/decode/multi.mbtiles.pipeline.json
cmp tests/muni/decode/multi.mbtiles.onetile.json.check tests/muni/decode/multi.mbtiles.onetile.json
cmp tests/muni/decode/multi.mbtiles.stats.json.check tests/muni/decode/multi.mbtiles.stats.json
Expand Down
2 changes: 2 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -880,6 +880,8 @@ resolutions.
* `-c` or `--tag-layer-and-zoom`: Include each feature's layer and zoom level as part of its `tippecanoe` object rather than as a FeatureCollection wrapper
* `-S` or `--stats`: Just report statistics about each tile's size and the number of features in it, as a JSON structure.
* `-f` or `--force`: Decode tiles even if polygon ring order or closure problems are detected
* `-I` or `--integer`: Report coordinates in integer tile coordinates
* `-F` or `--fraction`: Report coordinates as a fraction of the tile extent

tippecanoe-json-tool
====================
Expand Down
9 changes: 5 additions & 4 deletions csv.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#include "csv.hpp"
#include "text.hpp"
#include "errors.hpp"

std::vector<std::string> csv_split(const char *s) {
std::vector<std::string> ret;
Expand Down Expand Up @@ -68,15 +69,15 @@ void readcsv(const char *fn, std::vector<std::string> &header, std::map<std::str
FILE *f = fopen(fn, "r");
if (f == NULL) {
perror(fn);
exit(EXIT_FAILURE);
exit(EXIT_OPEN);
}

std::string s;
if ((s = csv_getline(f)).size() > 0) {
std::string err = check_utf8(s);
if (err != "") {
fprintf(stderr, "%s: %s\n", fn, err.c_str());
exit(EXIT_FAILURE);
exit(EXIT_UTF8);
}

header = csv_split(s.c_str());
Expand All @@ -89,7 +90,7 @@ void readcsv(const char *fn, std::vector<std::string> &header, std::map<std::str
std::string err = check_utf8(s);
if (err != "") {
fprintf(stderr, "%s: %s\n", fn, err.c_str());
exit(EXIT_FAILURE);
exit(EXIT_UTF8);
}

std::vector<std::string> line = csv_split(s.c_str());
Expand All @@ -105,7 +106,7 @@ void readcsv(const char *fn, std::vector<std::string> &header, std::map<std::str

if (fclose(f) != 0) {
perror("fclose");
exit(EXIT_FAILURE);
exit(EXIT_CLOSE);
}
}

Expand Down
72 changes: 45 additions & 27 deletions decode.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include "write_json.hpp"
#include "jsonpull/jsonpull.h"
#include "dirtiles.hpp"
#include "errors.hpp"

int minzoom = 0;
int maxzoom = 32;
Expand Down Expand Up @@ -85,18 +86,18 @@ void do_stats(mvt_tile &tile, size_t size, bool compressed, int z, unsigned x, u
state.json_write_newline();
}

void handle(std::string message, int z, unsigned x, unsigned y, std::set<std::string> const &to_decode, bool pipeline, bool stats, json_writer &state) {
void handle(std::string message, int z, unsigned x, unsigned y, std::set<std::string> const &to_decode, bool pipeline, bool stats, json_writer &state, int coordinate_mode) {
mvt_tile tile;
bool was_compressed;

try {
if (!tile.decode(message, was_compressed)) {
fprintf(stderr, "Couldn't parse tile %d/%u/%u\n", z, x, y);
exit(EXIT_FAILURE);
exit(EXIT_MVT);
}
} catch (std::exception const &e) {
fprintf(stderr, "PBF decoding error in tile %d/%u/%u\n", z, x, y);
exit(EXIT_FAILURE);
exit(EXIT_PROTOBUF);
}

if (stats) {
Expand Down Expand Up @@ -159,7 +160,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, std::set<std::st

if (layer.extent <= 0) {
fprintf(stderr, "Impossible layer extent %lld in mbtiles\n", layer.extent);
exit(EXIT_FAILURE);
exit(EXIT_IMPOSSIBLE);
}

if (to_decode.size() != 0 && !to_decode.count(layer.name)) {
Expand Down Expand Up @@ -202,10 +203,16 @@ void handle(std::string message, int z, unsigned x, unsigned y, std::set<std::st
// X and Y are unsigned, so no need to check <0
if (x > (1ULL << z) || y > (1ULL << z)) {
fprintf(stderr, "Impossible tile %d/%u/%u\n", z, x, y);
exit(EXIT_FAILURE);
exit(EXIT_IMPOSSIBLE);
}

layer_to_geojson(layer, z, x, y, !pipeline, pipeline, pipeline, false, 0, 0, 0, !force, state);
double scale = 0;
if (coordinate_mode == 1) { // fraction
scale = layer.extent;
} else if (coordinate_mode == 2) { // integer
scale = 1;
}
layer_to_geojson(layer, z, x, y, !pipeline, pipeline, pipeline, false, 0, 0, 0, !force, state, scale);

if (!pipeline) {
if (true) {
Expand All @@ -223,7 +230,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, std::set<std::st
}
}

void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> const &to_decode, bool pipeline, bool stats, std::set<std::string> const &exclude_meta) {
void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> const &to_decode, bool pipeline, bool stats, std::set<std::string> const &exclude_meta, int coordinate_mode) {
sqlite3 *db = NULL;
bool isdir = false;
int oz = z;
Expand All @@ -240,12 +247,12 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
if (strcmp(map, "SQLite format 3") != 0) {
if (z >= 0) {
std::string s = std::string(map, st.st_size);
handle(s, z, x, y, to_decode, pipeline, stats, state);
handle(s, z, x, y, to_decode, pipeline, stats, state, coordinate_mode);
munmap(map, st.st_size);
return;
} else {
fprintf(stderr, "Must specify zoom/x/y to decode a single pbf file\n");
exit(EXIT_FAILURE);
exit(EXIT_ARGS);
}
}
}
Expand All @@ -256,7 +263,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
}
if (close(fd) != 0) {
perror("close");
exit(EXIT_FAILURE);
exit(EXIT_CLOSE);
}
} else {
perror(fname);
Expand All @@ -272,13 +279,13 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
} else {
if (sqlite3_open(fname, &db) != SQLITE_OK) {
fprintf(stderr, "%s: %s\n", fname, sqlite3_errmsg(db));
exit(EXIT_FAILURE);
exit(EXIT_OPEN);
}

char *err = NULL;
if (sqlite3_exec(db, "PRAGMA integrity_check;", NULL, NULL, &err) != SQLITE_OK) {
fprintf(stderr, "%s: integrity_check: %s\n", fname, err);
exit(EXIT_FAILURE);
exit(EXIT_SQLITE);
}
}

Expand All @@ -299,7 +306,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
sqlite3_stmt *stmt2;
if (sqlite3_prepare_v2(db, sql2, -1, &stmt2, NULL) != SQLITE_OK) {
fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db));
exit(EXIT_FAILURE);
exit(EXIT_SQLITE);
}

while (sqlite3_step(stmt2) == SQLITE_ROW) {
Expand All @@ -308,7 +315,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co

if (name == NULL || value == NULL) {
fprintf(stderr, "Corrupt mbtiles file: null metadata\n");
exit(EXIT_FAILURE);
exit(EXIT_SQLITE);
}

if (exclude_meta.count((char *) name) == 0) {
Expand Down Expand Up @@ -361,7 +368,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
FILE *f = fopen(fn.c_str(), "rb");
if (f == NULL) {
perror(fn.c_str());
exit(EXIT_FAILURE);
exit(EXIT_OPEN);
}

std::string s;
Expand All @@ -372,14 +379,14 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
}
fclose(f);

handle(s, tiles[i].z, tiles[i].x, tiles[i].y, to_decode, pipeline, stats, state);
handle(s, tiles[i].z, tiles[i].x, tiles[i].y, to_decode, pipeline, stats, state, coordinate_mode);
}
} else {
const char *sql = "SELECT tile_data, zoom_level, tile_column, tile_row from tiles where zoom_level between ? and ? order by zoom_level, tile_column, tile_row;";
sqlite3_stmt *stmt;
if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {
fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db));
exit(EXIT_FAILURE);
exit(EXIT_SQLITE);
}

sqlite3_bind_int(stmt, 1, minzoom);
Expand Down Expand Up @@ -407,18 +414,18 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co

if (tz < 0 || tz >= 32) {
fprintf(stderr, "Impossible zoom level %d in mbtiles\n", tz);
exit(EXIT_FAILURE);
exit(EXIT_IMPOSSIBLE);
}

ty = (1LL << tz) - 1 - ty;
const char *s = (const char *) sqlite3_column_blob(stmt, 0);

if (s == NULL) {
fprintf(stderr, "Corrupt mbtiles file: null entry in tiles table\n");
exit(EXIT_FAILURE);
exit(EXIT_SQLITE);
}

handle(std::string(s, len), tz, tx, ty, to_decode, pipeline, stats, state);
handle(std::string(s, len), tz, tx, ty, to_decode, pipeline, stats, state, coordinate_mode);
}

sqlite3_finalize(stmt);
Expand All @@ -443,7 +450,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
sqlite3_stmt *stmt;
if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {
fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db));
exit(EXIT_FAILURE);
exit(EXIT_SQLITE);
}

sqlite3_bind_int(stmt, 1, z);
Expand All @@ -456,14 +463,14 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co

if (s == NULL) {
fprintf(stderr, "Corrupt mbtiles file: null entry in tiles table\n");
exit(EXIT_FAILURE);
exit(EXIT_SQLITE);
}

if (z != oz) {
fprintf(stderr, "%s: Warning: using tile %d/%u/%u instead of %d/%u/%u\n", fname, z, x, y, oz, ox, oy);
}

handle(std::string(s, len), z, x, y, to_decode, pipeline, stats, state);
handle(std::string(s, len), z, x, y, to_decode, pipeline, stats, state, coordinate_mode);
handled = 1;
}

Expand All @@ -477,13 +484,13 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co

if (sqlite3_close(db) != SQLITE_OK) {
fprintf(stderr, "%s: could not close database: %s\n", fname, sqlite3_errmsg(db));
exit(EXIT_FAILURE);
exit(EXIT_CLOSE);
}
}

void usage(char **argv) {
fprintf(stderr, "Usage: %s [-s projection] [-Z minzoom] [-z maxzoom] [-l layer ...] file.mbtiles [zoom x y]\n", argv[0]);
exit(EXIT_FAILURE);
exit(EXIT_ARGS);
}

int main(int argc, char **argv) {
Expand All @@ -494,9 +501,12 @@ int main(int argc, char **argv) {
bool pipeline = false;
bool stats = false;
std::set<std::string> exclude_meta;
int coordinate_mode = 0;

struct option long_options[] = {
{"projection", required_argument, 0, 's'},
{"fractional-coordinates", no_argument, 0, 'F'},
{"integer-coordinates", no_argument, 0, 'I'},
{"maximum-zoom", required_argument, 0, 'z'},
{"minimum-zoom", required_argument, 0, 'Z'},
{"layer", required_argument, 0, 'l'},
Expand Down Expand Up @@ -527,6 +537,14 @@ int main(int argc, char **argv) {
set_projection_or_exit(optarg);
break;

case 'F':
coordinate_mode = 1;
break;

case 'I':
coordinate_mode = 2;
break;

case 'z':
maxzoom = atoi(optarg);
break;
Expand Down Expand Up @@ -561,9 +579,9 @@ int main(int argc, char **argv) {
}

if (argc == optind + 4) {
decode(argv[optind], atoi(argv[optind + 1]), atoi(argv[optind + 2]), atoi(argv[optind + 3]), to_decode, pipeline, stats, exclude_meta);
decode(argv[optind], atoi(argv[optind + 1]), atoi(argv[optind + 2]), atoi(argv[optind + 3]), to_decode, pipeline, stats, exclude_meta, coordinate_mode);
} else if (argc == optind + 1) {
decode(argv[optind], -1, -1, -1, to_decode, pipeline, stats, exclude_meta);
decode(argv[optind], -1, -1, -1, to_decode, pipeline, stats, exclude_meta, coordinate_mode);
} else {
usage(argv);
}
Expand Down
Loading

0 comments on commit af1a7ed

Please sign in to comment.