stats for serialtiles

bin/mapbox-tile-copy.js

@@ -66,6 +66,7 @@ if (isNumeric(argv.retry)) options.retry = parseInt(argv.retry, 10);
 if (isNumeric(argv.timeout)) options.timeout = parseInt(argv.timeout, 10);
 if (argv.bundle === 'true') options.bundle = true;
 if (argv['bypass-validation'] === 'true') options.bypassValidation = true;
+if (process.env.BRIDGE_LOG_MAX_VTILE_BYTES_COMPRESSED) options.tileSizeStats = true;
 
 if (!dsturi) {
   console.error('You must provide a valid s3:// or file:// url');

lib/serialtilescopy.js

@@ -1,4 +1,5 @@
 var fs = require('fs');
+var os = require('os');
 var util = require('util');
 var TileStatStream = require('tile-stat-stream');
 var tilelive = require('@mapbox/tilelive');
@@ -14,6 +15,10 @@ function serialtiles(srcUri, s3urlTemplate, options, callback) {
     options = {};
   }
 
+  if (options.tileSizeStats) {
+    var tileSizeStats = { total: 0, count: 0, max: 0 };
+  }
+
   var statStream = new TileStatStream();
 
   var max_tilesize = options.limits && options.limits.max_tilesize ?
@@ -57,6 +62,13 @@ function serialtiles(srcUri, s3urlTemplate, options, callback) {
       .on('tile', function(tile) {
         if (!tile.buffer) return;
         stats.done++;
+        if (options.tileSizeStats) {
+          tileSizeStats.count++;
+          tileSizeStats.total = tileSizeStats.total + (tile.buffer.length * 0.001);
+          if (tileSizeStats.max < tile.buffer.length) {
+            tileSizeStats.max = tile.buffer.length;
+        }
+        }
         if (tile.buffer.length <= max_tilesize) return;
         var err = new Error(util.format('Tile exceeds maximum size of %sk at z %s. Reduce the detail of data at this zoom level or omit it by adjusting your minzoom.', Math.round(max_tilesize / 1024), tile.z));
         err.code = 'EINVALID';
@@ -88,6 +100,16 @@ function serialtiles(srcUri, s3urlTemplate, options, callback) {
     once = true;
 
     if (err) source.unpipe();
+
+    if (options.tileSizeStats) {
+      // dump file to tmp dir
+      tileSizeStats.avg = tileSizeStats.total / tileSizeStats.count;
+      if (tileSizeStats.count > 0) {
+        var file = os.tmpdir() + '/tilelive-bridge-stats.json';
+        fs.writeFileSync(file, JSON.stringify(tileSizeStats));
+      }
+    }
+
     if (options.stats) {
       callback(err, statStream.getStatistics());
     } else {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mapbox/mapbox-tile-copy",
-  "version": "7.3.2",
+  "version": "7.4.0",
   "description": "From geodata files to tiles on S3",
   "main": "index.js",
   "scripts": {

readme.md

@@ -64,6 +64,11 @@ The following example will distribute tiles to the second part out of 4 total pa
 $ mapbox-tile-copy ~/data/my-tiles.mbtiles s3://my-bucket/parallel/{z}/{x}/{y} --part 1 --parts 4
 ```
 
+Collect tile size statistics and dump to your local tmp dir named `/tmp/<tmpdirpath>/tilelive-bridge-stats.json`
+```
+$ BRIDGE_LOG_MAX_VTILE_BYTES_COMPRESSED=1 mapbox-tile-copy ~/data/my-tiles.mbtiles s3://my-bucket/folder/mbtiles/{z}/{x}/{y}
+```
+
 ## Supported file types
 
 - .mbtiles