diff --git a/binding.gyp b/binding.gyp index b9f1bd4879..20ac2ffe40 100644 --- a/binding.gyp +++ b/binding.gyp @@ -99,6 +99,7 @@ 'mapnik-json.lib', ' #include +#include + namespace detail { struct p2p_distance @@ -2397,24 +2399,166 @@ void VectorTile::EIO_AfterSetData(uv_work_t* req) delete closure; } +#define MOD_GZIP_ZLIB_WINDOWSIZE 15 +#define MOD_GZIP_ZLIB_CFACTOR 9 + +std::string VectorTile::_gzip_compress(const unsigned char * str, const int len, + int compressionlevel = Z_BEST_COMPRESSION, + int strategy = Z_DEFAULT_STRATEGY) +{ + + + z_stream zs; + memset(&zs, 0, sizeof(zs)); + + if (deflateInit2(&zs, compressionlevel, Z_DEFLATED, + MOD_GZIP_ZLIB_WINDOWSIZE + 16, + MOD_GZIP_ZLIB_CFACTOR, + strategy) != Z_OK) + throw(std::runtime_error("deflateInit failed while compressing.")); + + zs.next_in = (unsigned char *)str; + zs.avail_in = len; + + int ret; + char outbuffer[32768]; + std::string outstring; + + // retrieve the compressed bytes blockwise + do { + zs.next_out = reinterpret_cast(outbuffer); + zs.avail_out = sizeof(outbuffer); + + ret = deflate(&zs, Z_FINISH); + + if (outstring.size() < zs.total_out) { + // append the block to the output string + outstring.append(outbuffer, + zs.total_out - outstring.size()); + } + } while (ret == Z_OK); + + deflateEnd(&zs); + + if (ret != Z_STREAM_END) { // an error occurred that was not EOF + std::ostringstream oss; + oss << "Exception during zlib compression: (" << ret << ") " << zs.msg; + throw(std::runtime_error(oss.str())); + } + + return outstring; +} + + NAN_METHOD(VectorTile::getData) { NanScope(); + NanReturnValue(_getData(args)); +} + + + +Local VectorTile::_getData(_NAN_METHOD_ARGS) +{ + NanEscapableScope(); VectorTile* d = node::ObjectWrap::Unwrap(args.Holder()); + + Local options = NanNew(); + bool gzip = false; + int level = 9; + int strategy = Z_DEFAULT_STRATEGY; + + if (args.Length() > 0) + { + // options object + if (!args[0]->IsObject()) + { + throw std::runtime_error("first argument must be an options object"); + } + + options = args[0]->ToObject(); + + if (options->Has(NanNew("compression"))) + { + Local param_val = options->Get(NanNew("compression")); + if (!param_val->IsString()) + { + throw std::runtime_error("option 'compression' must be a string, either 'gzip', or 'none' (default)"); + } + gzip = std::string("gzip") == (TOSTR(param_val->ToString())); + } + + if (options->Has(NanNew("level"))) + { + Local param_val = options->Get(NanNew("level")); + if (!param_val->IsNumber()) + { + throw std::runtime_error("option 'level' must be an integer between 0 (no compression) and 9 (best compression) inclusive"); + } + level = param_val->IntegerValue(); + if (level < 0 || level > 9) + { + throw std::runtime_error("option 'level' must be an integer between 0 (no compression) and 9 (best compression) inclusive"); + } + } + if (options->Has(NanNew("strategy"))) + { + Local param_val = options->Get(NanNew("level")); + if (!param_val->IsString()) + { + throw std::runtime_error("option 'strategy' must be one of the following strings: FILTERED, HUFFMAN_ONLY, RLE, FIXED, DEFAULT"); + } + else if (std::string("FILTERED") == TOSTR(param_val->ToString())) + { + strategy = Z_FILTERED; + } + else if (std::string("HUFFMAN_ONLY") == TOSTR(param_val->ToString())) + { + strategy = Z_HUFFMAN_ONLY; + } + else if (std::string("RLE") == TOSTR(param_val->ToString())) + { + strategy = Z_RLE; + } + else if (std::string("FIXED") == TOSTR(param_val->ToString())) + { + strategy = Z_FIXED; + } + else if (std::string("DEFAULT") == TOSTR(param_val->ToString())) + { + strategy = Z_DEFAULT_STRATEGY; + } + else + { + throw std::runtime_error("option 'strategy' must be one of the following strings: FILTERED, HUFFMAN_ONLY, RLE, FIXED, DEFAULT"); + } + } + } try { // shortcut: return raw data and avoid trip through proto object std::size_t raw_size = d->buffer_.size(); if (raw_size > 0 && (d->byte_size_ < 0 || static_cast(d->byte_size_) <= raw_size)) { + if (gzip) + { + std::string compressed_data = _gzip_compress((unsigned char *)d->buffer_.data(), raw_size, level); + if (compressed_data.size() >= node::Buffer::kMaxLength) { + std::ostringstream s; + s << "Compressed data is too large to convert to a node::Buffer "; + s << "(" << compressed_data.size() << " compressed bytes >= node::Buffer::kMaxLength)"; + throw std::runtime_error(s.str()); + } + return NanEscapeScope(NanNewBufferHandle(compressed_data.c_str(), compressed_data.size())); + } if (raw_size >= node::Buffer::kMaxLength) { std::ostringstream s; s << "Data is too large to convert to a node::Buffer "; s << "(" << raw_size << " raw bytes >= node::Buffer::kMaxLength)"; throw std::runtime_error(s.str()); } - NanReturnValue(NanNewBufferHandle((char*)d->buffer_.data(),raw_size)); + return NanEscapeScope(NanNewBufferHandle((char*)d->buffer_.data(),raw_size)); } else { if (d->byte_size_ <= 0) { - NanReturnValue(NanNewBufferHandle(0)); + return NanEscapeScope(NanNewBufferHandle(0)); } else { // NOTE: tiledata.ByteSize() must be called // after each modification of tiledata otherwise the @@ -2426,6 +2570,7 @@ NAN_METHOD(VectorTile::getData) s << "(" << d->byte_size_ << " cached bytes >= node::Buffer::kMaxLength)"; throw std::runtime_error(s.str()); } + // TODO: possible memory leak here if gzip is used???? Local retbuf = NanNewBufferHandle(d->byte_size_); // TODO - consider wrapping in fastbuffer: https://gist.github.com/drewish/2732711 // http://www.samcday.com.au/blog/2011/03/03/creating-a-proper-buffer-in-a-node-c-addon/ @@ -2435,14 +2580,27 @@ NAN_METHOD(VectorTile::getData) if (end - start != d->byte_size_) { throw std::runtime_error("serialization failed, possible race condition"); } - NanReturnValue(retbuf); + if (gzip) + { + std::string compressed_data = _gzip_compress(start, d->byte_size_, level); + if (compressed_data.size() >= node::Buffer::kMaxLength) { + std::ostringstream s; + s << "Compressed data is too large to convert to a node::Buffer "; + s << "(" << compressed_data.size() << " compressed bytes >= node::Buffer::kMaxLength)"; + throw std::runtime_error(s.str()); + } + return NanEscapeScope(NanNewBufferHandle(compressed_data.c_str(), compressed_data.size())); + } + return NanEscapeScope(retbuf); } } - } catch (std::exception const& ex) { + } + catch (std::exception const& ex) + { NanThrowError(ex.what()); - NanReturnUndefined(); + return NanEscapeScope(NanUndefined()); } - NanReturnUndefined(); + return NanEscapeScope(NanUndefined()); } using surface_type = mapnik::util::variant; diff --git a/src/mapnik_vector_tile.hpp b/src/mapnik_vector_tile.hpp index d2c535798e..7415d1ce3b 100644 --- a/src/mapnik_vector_tile.hpp +++ b/src/mapnik_vector_tile.hpp @@ -44,6 +44,7 @@ class VectorTile: public node::ObjectWrap { static Persistent constructor; static void Initialize(Handle target); static NAN_METHOD(New); + static Local _getData(_NAN_METHOD_ARGS); static NAN_METHOD(getData); static NAN_METHOD(render); static NAN_METHOD(toJSON); @@ -148,6 +149,7 @@ class VectorTile: public node::ObjectWrap { parsing_status status_; private: ~VectorTile(); + static std::string _gzip_compress(const unsigned char * str, const int len, int compressionlevel, int strategy); vector_tile::Tile tiledata_; unsigned width_; unsigned height_; diff --git a/test/vector-tile.compression.test.js b/test/vector-tile.compression.test.js new file mode 100644 index 0000000000..9ca8e6480b --- /dev/null +++ b/test/vector-tile.compression.test.js @@ -0,0 +1,62 @@ +"use strict"; + +var zlib = require('zlib'); +var mapnik = require('../'); +var assert = require('assert'); +var fs = require('fs'); +var path = require('path'); +var mercator = new(require('sphericalmercator'))(); +var existsSync = require('fs').existsSync || require('path').existsSync; +var overwrite_expected_data = false; +//var SegfaultHandler = require('segfault-handler'); +//SegfaultHandler.registerHandler(); + +mapnik.register_datasource(path.join(mapnik.settings.paths.input_plugins,'geojson.input')); + +var trunc_6 = function(key, val) { + return val.toFixed ? Number(val.toFixed(6)) : val; +}; + +function deepEqualTrunc(json1,json2) { + return assert.deepEqual(JSON.stringify(json1,trunc_6),JSON.stringify(json2,trunc_6)); +} + +mapnik.register_datasource(path.join(mapnik.settings.paths.input_plugins,'shape.input')); +mapnik.register_datasource(path.join(mapnik.settings.paths.input_plugins,'gdal.input')); + +describe('mapnik.VectorTile compression', function() { + + it('should be able to create a vector tile from geojson', function(done) { + var vtile = new mapnik.VectorTile(0,0,0); + var geojson = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + -122, + 48 + ] + }, + "properties": { + "name": "geojson data" + } + } + ] + }; + vtile.addGeoJSON(JSON.stringify(geojson),"layer-name"); + + assert.equal(vtile.getData().length, 58); + assert.equal(vtile.getData({ compression: 'gzip'}).length,76); + + // Actually test decompressing, but only available on NodeJS >= 0.12.4 + if (typeof zlib.gunzipSync != 'undefined') { + assert.equal(zlib.gunzipSync(vtile.getData({ compression: 'gzip'})).length,58); + } + + done(); + }); + +});