Skip to content

Commit 259ec79

Browse files
e-n-fe-n-f
authored
Merge pull request #42 from mapbox/tilestats
Include tilestats in metadata
2 parents f0d828f + 8c12cc2 commit 259ec79

File tree

9 files changed

+589
-82
lines changed

9 files changed

+589
-82
lines changed

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ tile-count-create: tippecanoe/projection.o create.o header.o serial.o merge.o js
4040
tile-count-decode: tippecanoe/projection.o decode.o header.o serial.o
4141
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread
4242

43-
tile-count-tile: tippecanoe/projection.o tile.o header.o serial.o tippecanoe/mbtiles.o tippecanoe/mvt.o
43+
tile-count-tile: tippecanoe/projection.o tile.o header.o serial.o tippecanoe/mbtiles.o tippecanoe/mvt.o tippecanoe/text.o
4444
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread -lpng
4545

4646
tile-count-merge: mergetool.o header.o serial.o merge.o

‎tile.cpp‎

Lines changed: 68 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ struct tiler {
8989
size_t shard;
9090
size_t cpus;
9191
std::string layername;
92+
std::map<std::string, layermap_entry> *layermap;
9293
};
9394

9495
void gather_quantile(tile const &tile, int detail, long long &max) {
@@ -113,7 +114,7 @@ static void fail(png_structp png_ptr, png_const_charp error_msg) {
113114
exit(EXIT_FAILURE);
114115
}
115116

116-
void make_tile(sqlite3 *outdb, tile &otile, int z, int detail, long long zoom_max, std::string const &layername) {
117+
void make_tile(sqlite3 *outdb, tile &otile, int z, int detail, long long zoom_max, std::string const &layername, std::map<std::string, layermap_entry> *layermap) {
117118
long long thresh = first_count;
118119
bool again = true;
119120
tile tile = otile;
@@ -261,13 +262,27 @@ void make_tile(sqlite3 *outdb, tile &otile, int z, int detail, long long zoom_ma
261262
val.type = mvt_uint;
262263
val.numeric_value.uint_value = normalized[y * (1 << detail) + x];
263264
layer.tag(feature, "density", val);
265+
266+
type_and_string attrib;
267+
attrib.type = mvt_double;
268+
attrib.string = std::to_string(val.numeric_value.uint_value);
269+
270+
auto fk = layermap->find(layername);
271+
add_to_file_keys(fk->second.file_keys, "density", attrib);
264272
}
265273

266274
if (include_count) {
267275
mvt_value val;
268276
val.type = mvt_uint;
269277
val.numeric_value.uint_value = tile.count[y * (1 << detail) + x];
270278
layer.tag(feature, "count", val);
279+
280+
type_and_string attrib;
281+
attrib.type = mvt_double;
282+
attrib.string = std::to_string(val.numeric_value.uint_value);
283+
284+
auto fk = layermap->find(layername);
285+
add_to_file_keys(fk->second.file_keys, "count", attrib);
271286
}
272287

273288
layer.features.push_back(feature);
@@ -312,13 +327,27 @@ void make_tile(sqlite3 *outdb, tile &otile, int z, int detail, long long zoom_ma
312327
val.type = mvt_uint;
313328
val.numeric_value.uint_value = i;
314329
layer.tag(features[i], "density", val);
330+
331+
type_and_string attrib;
332+
attrib.type = mvt_double;
333+
attrib.string = std::to_string(val.numeric_value.uint_value);
334+
335+
auto fk = layermap->find(layername);
336+
add_to_file_keys(fk->second.file_keys, "density", attrib);
315337
}
316338

317339
if (include_count) {
318340
mvt_value val;
319341
val.type = mvt_uint;
320342
val.numeric_value.uint_value = count;
321343
layer.tag(features[i], "count", val);
344+
345+
type_and_string attrib;
346+
attrib.type = mvt_double;
347+
attrib.string = std::to_string(val.numeric_value.uint_value);
348+
349+
auto fk = layermap->find(layername);
350+
add_to_file_keys(fk->second.file_keys, "count", attrib);
322351
}
323352

324353
layer.features.push_back(features[i]);
@@ -330,7 +359,8 @@ void make_tile(sqlite3 *outdb, tile &otile, int z, int detail, long long zoom_ma
330359
mvt_tile mvt;
331360
mvt.layers.push_back(layer);
332361

333-
compressed = mvt.encode();
362+
std::string pbf = mvt.encode();
363+
compress(pbf, compressed);
334364
}
335365
}
336366

@@ -508,7 +538,7 @@ void *run_tile(void *p) {
508538
if (t->pass == 0) {
509539
gather_quantile(t->tiles[z], t->detail, t->max[z]);
510540
} else {
511-
make_tile(t->outdb, t->tiles[z], z, t->detail, t->zoom_max[z], t->layername);
541+
make_tile(t->outdb, t->tiles[z], z, t->detail, t->zoom_max[z], t->layername, t->layermap);
512542
}
513543
} else {
514544
t->partial_tiles.push_back(t->tiles[z]);
@@ -546,7 +576,7 @@ void *run_tile(void *p) {
546576
if (t->pass == 0) {
547577
gather_quantile(t->tiles[z], t->detail, t->max[z]);
548578
} else {
549-
make_tile(t->outdb, t->tiles[z], z, t->detail, t->zoom_max[z], t->layername);
579+
make_tile(t->outdb, t->tiles[z], z, t->detail, t->zoom_max[z], t->layername, t->layermap);
550580
}
551581
} else {
552582
t->partial_tiles.push_back(t->tiles[z]);
@@ -643,6 +673,7 @@ struct tile_reader {
643673
double density_gamma;
644674
std::vector<long long> max_density;
645675
std::vector<long long> global_density;
676+
std::map<std::string, layermap_entry> *layermap;
646677

647678
int y() {
648679
return (1LL << zoom) - 1 - sorty;
@@ -740,7 +771,7 @@ void *retile(void *v) {
740771

741772
if (!t.active || t.z != (*queue)[i]->zoom || t.x != (*queue)[i]->x || t.y != (*queue)[i]->y() || t.count.size() != width * height) {
742773
if (t.active) {
743-
make_tile((*queue)[i]->outdb, t, t.z, log(sqrt(t.count.size())) / log(2), (*queue)[i]->global_density[t.z], (*queue)[i]->layername);
774+
make_tile((*queue)[i]->outdb, t, t.z, log(sqrt(t.count.size())) / log(2), (*queue)[i]->global_density[t.z], (*queue)[i]->layername, (*queue)[i]->layermap);
744775
}
745776

746777
t.active = true;
@@ -783,7 +814,9 @@ void *retile(void *v) {
783814
mvt_tile tile;
784815

785816
try {
786-
if (!tile.decode((*queue)[i]->data)) {
817+
bool was_compressed;
818+
819+
if (!tile.decode((*queue)[i]->data, was_compressed)) {
787820
fprintf(stderr, "Couldn't parse tile\n");
788821
exit(EXIT_FAILURE);
789822
}
@@ -802,7 +835,7 @@ void *retile(void *v) {
802835

803836
if (!t.active || t.z != (*queue)[i]->zoom || t.x != (*queue)[i]->x || t.y != (*queue)[i]->y() || t.count.size() != extent * extent) {
804837
if (t.active) {
805-
make_tile((*queue)[i]->outdb, t, t.z, log(sqrt(t.count.size())) / log(2), (*queue)[i]->global_density[t.z], (*queue)[i]->layername);
838+
make_tile((*queue)[i]->outdb, t, t.z, log(sqrt(t.count.size())) / log(2), (*queue)[i]->global_density[t.z], (*queue)[i]->layername, (*queue)[i]->layermap);
806839
}
807840

808841
t.active = true;
@@ -878,18 +911,19 @@ void *retile(void *v) {
878911
}
879912

880913
if (t.active) {
881-
make_tile((*queue)[0]->outdb, t, t.z, log(sqrt(t.count.size())) / log(2), (*queue)[0]->global_density[t.z], (*queue)[0]->layername);
914+
make_tile((*queue)[0]->outdb, t, t.z, log(sqrt(t.count.size())) / log(2), (*queue)[0]->global_density[t.z], (*queue)[0]->layername, (*queue)[0]->layermap);
882915
}
883916

884917
return NULL;
885918
}
886919

887-
void merge(std::vector<tile_reader> &r, size_t cpus) {
920+
void merge(std::vector<tile_reader> &r, size_t cpus, std::vector<std::map<std::string, layermap_entry>> &layermaps) {
888921
std::vector<std::vector<tile_reader *>> queues;
889922
queues.resize(cpus);
890923

891924
size_t o = 0;
892925
for (size_t i = 0; i < r.size(); i++) {
926+
r[i].layermap = &layermaps[o];
893927
queues[o].push_back(&r[i]);
894928

895929
if (i + 1 < r.size() && (r[i].x != r[i + 1].x || r[i].y() != r[i + 1].y() || r[i].zoom != r[i + 1].zoom)) {
@@ -933,7 +967,7 @@ std::vector<long long> parse_max_density(const unsigned char *v) {
933967
return out;
934968
}
935969

936-
void merge_tiles(char **fnames, size_t n, size_t cpus, sqlite3 *outdb, int zooms, std::vector<long long> &zoom_max, double &midlat, double &midlon, double &minlat, double &minlon, double &maxlat, double &maxlon, std::string const &layername) {
970+
void merge_tiles(char **fnames, size_t n, size_t cpus, sqlite3 *outdb, int zooms, std::vector<long long> &zoom_max, double &midlat, double &midlon, double &minlat, double &minlon, double &maxlat, double &maxlon, std::string const &layername, std::vector<std::map<std::string, layermap_entry>> &layermaps) {
937971
std::vector<tile_reader> readers;
938972
size_t total_rows = 0;
939973
size_t seq = 0;
@@ -1094,7 +1128,7 @@ void merge_tiles(char **fnames, size_t n, size_t cpus, sqlite3 *outdb, int zooms
10941128
if (to_merge.size() > 50 * cpus) {
10951129
tile_reader &last = to_merge[to_merge.size() - 1];
10961130
if (r.x != last.x || r.y() != last.y() || r.zoom != last.zoom) {
1097-
merge(to_merge, cpus);
1131+
merge(to_merge, cpus, layermaps);
10981132
to_merge.clear();
10991133
}
11001134
}
@@ -1136,7 +1170,7 @@ void merge_tiles(char **fnames, size_t n, size_t cpus, sqlite3 *outdb, int zooms
11361170
}
11371171
}
11381172

1139-
merge(to_merge, cpus);
1173+
merge(to_merge, cpus, layermaps);
11401174
}
11411175

11421176
int main(int argc, char **argv) {
@@ -1173,6 +1207,10 @@ int main(int argc, char **argv) {
11731207

11741208
case 'p':
11751209
cpus = atoi(optarg);
1210+
if (cpus <= 0) {
1211+
fprintf(stderr, "%s: cpu count %s must be positive\n", argv[0], optarg);
1212+
exit(EXIT_FAILURE);
1213+
}
11761214
break;
11771215

11781216
case 'd':
@@ -1282,6 +1320,18 @@ int main(int argc, char **argv) {
12821320
std::vector<long long> zoom_max;
12831321
size_t zooms = 0;
12841322

1323+
std::vector<std::map<std::string, layermap_entry>> layermaps;
1324+
for (size_t cpu = 0; cpu < cpus; cpu++) {
1325+
layermap_entry lme(0);
1326+
lme.minzoom = 0;
1327+
lme.maxzoom = zooms - 1;
1328+
1329+
std::map<std::string, layermap_entry> m;
1330+
m.insert(std::pair<std::string, layermap_entry>(layername, lme));
1331+
1332+
layermaps.push_back(m);
1333+
}
1334+
12851335
// Try opening input as mbtiles for retiling.
12861336
// If that doesn't work, it must be new counts.
12871337

@@ -1371,6 +1421,7 @@ int main(int argc, char **argv) {
13711421
tilers[j].pass = pass;
13721422
tilers[j].zoom_max = zoom_max;
13731423
tilers[j].layername = layername;
1424+
tilers[j].layermap = &layermaps[j];
13741425
}
13751426

13761427
size_t records = (st.st_size - HEADER_LEN) / RECORD_BYTES;
@@ -1426,7 +1477,7 @@ int main(int argc, char **argv) {
14261477
if (pass == 0) {
14271478
gather_quantile(a->second, detail, tilers[0].max[a->second.z]);
14281479
} else {
1429-
make_tile(outdb, a->second, a->second.z, detail, zoom_max[a->second.z], layername);
1480+
make_tile(outdb, a->second, a->second.z, detail, zoom_max[a->second.z], layername, &layermaps[0]);
14301481
}
14311482
}
14321483

@@ -1475,36 +1526,14 @@ int main(int argc, char **argv) {
14751526
}
14761527
} else {
14771528
fprintf(stderr, "going to merge %zu zoom levels\n", zooms);
1478-
merge_tiles(argv + optind, argc - optind, cpus, outdb, zooms, zoom_max, midlat, midlon, minlat, minlon, maxlat, maxlon, layername);
1529+
merge_tiles(argv + optind, argc - optind, cpus, outdb, zooms, zoom_max, midlat, midlon, minlat, minlon, maxlat, maxlon, layername, layermaps);
14791530
}
14801531

1481-
layermap_entry lme(0);
1482-
1483-
if (include_count) {
1484-
type_and_string tas;
1485-
tas.type = mvt_double;
1486-
tas.string = "count";
1487-
lme.file_keys.insert(tas);
1488-
}
1489-
1490-
if (include_density) {
1491-
type_and_string tas2;
1492-
tas2.type = mvt_double;
1493-
tas2.string = "density";
1494-
lme.file_keys.insert(tas2);
1495-
}
1496-
1497-
lme.minzoom = 0;
1498-
lme.maxzoom = zooms - 1;
1499-
1500-
std::map<std::string, layermap_entry> lm;
1501-
if (!bitmap) {
1502-
lm.insert(std::pair<std::string, layermap_entry>(layername, lme));
1503-
}
1532+
std::map<std::string, layermap_entry> lm = merge_layermaps(layermaps);
15041533

1505-
mbtiles_write_metadata(outdb, outfile, 0, zooms - 1, minlat, minlon, maxlat, maxlon, midlat, midlon, false, "", lm, !bitmap);
1534+
mbtiles_write_metadata(outdb, NULL, outfile, 0, zooms - 1, minlat, minlon, maxlat, maxlon, midlat, midlon, false, "", lm, !bitmap, outfile);
15061535

15071536
write_meta(zoom_max, outdb);
15081537

1509-
mbtiles_close(outdb, argv);
1538+
mbtiles_close(outdb, argv[0]);
15101539
}

0 commit comments

Comments
 (0)