From c32f552ad0a4dd969ba43d8d8776b92d552ae1c7 Mon Sep 17 00:00:00 2001 From: delvedor Date: Thu, 22 Jul 2021 15:30:54 +0200 Subject: [PATCH 1/4] Support mapbox vector tile content type --- lib/Transport.d.ts | 4 ++-- lib/Transport.js | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/Transport.d.ts b/lib/Transport.d.ts index 912dd96da..2313bf23f 100644 --- a/lib/Transport.d.ts +++ b/lib/Transport.d.ts @@ -155,8 +155,8 @@ export default class Transport { _nextSniff: number; _isSniffing: boolean; constructor(opts: TransportOptions); - request(params: TransportRequestParams, options?: TransportRequestOptions): TransportRequestPromise; - request(params: TransportRequestParams, options?: TransportRequestOptions, callback?: (err: ApiError, result: ApiResponse) => void): TransportRequestCallback; + request, TContext = Context>(params: TransportRequestParams, options?: TransportRequestOptions): TransportRequestPromise>; + request, TContext = Context>(params: TransportRequestParams, options?: TransportRequestOptions, callback?: (err: ApiError, result: ApiResponse) => void): TransportRequestCallback; getConnection(opts: TransportGetConnectionOptions): Connection | null; sniff(opts?: TransportSniffOptions, callback?: (...args: any[]) => void): void; } diff --git a/lib/Transport.js b/lib/Transport.js index 6abcb723f..c1988ba56 100644 --- a/lib/Transport.js +++ b/lib/Transport.js @@ -237,6 +237,7 @@ class Transport { const contentEncoding = (result.headers['content-encoding'] || '').toLowerCase() const isCompressed = contentEncoding.indexOf('gzip') > -1 || contentEncoding.indexOf('deflate') > -1 + const isVectorTile = (result.headers['content-type'] || '').indexOf('application/vnd.mapbox-vector-tile') > -1 /* istanbul ignore else */ if (result.headers['content-length'] !== undefined) { @@ -255,8 +256,9 @@ class Transport { } // if the response is compressed, we must handle it // as buffer for allowing decompression later - let payload = isCompressed ? [] : '' - const onData = isCompressed + // while if it's a vector tile, we should return it as buffer + let payload = isCompressed || isVectorTile ? [] : '' + const onData = isCompressed || isVectorTile ? chunk => { payload.push(chunk) } : chunk => { payload += chunk } const onEnd = err => { @@ -272,7 +274,7 @@ class Transport { if (isCompressed) { unzip(Buffer.concat(payload), onBody) } else { - onBody(null, payload) + onBody(null, isVectorTile ? Buffer.concat(payload) : payload) } } @@ -281,7 +283,7 @@ class Transport { onEnd(new Error('Response aborted while reading the body')) } - if (!isCompressed) { + if (!isCompressed && !isVectorTile) { response.setEncoding('utf8') } @@ -297,7 +299,9 @@ class Transport { this.emit('response', err, result) return callback(err, result) } - if (Buffer.isBuffer(payload)) { + + const isVectorTile = (result.headers['content-type'] || '').indexOf('application/vnd.mapbox-vector-tile') > -1 + if (Buffer.isBuffer(payload) && !isVectorTile) { payload = payload.toString() } const isHead = params.method === 'HEAD' From f72b81ef543f4265daa14da642d41e4b36cbe083 Mon Sep 17 00:00:00 2001 From: delvedor Date: Thu, 22 Jul 2021 15:31:04 +0200 Subject: [PATCH 2/4] Updated test --- test/unit/transport.test.js | 69 +++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/test/unit/transport.test.js b/test/unit/transport.test.js index c617bc3af..9c7658dca 100644 --- a/test/unit/transport.test.js +++ b/test/unit/transport.test.js @@ -2689,3 +2689,72 @@ test('The callback with a sync error should be called in the next tick - ndjson' t.type(transportReturn.catch, 'function') t.type(transportReturn.abort, 'function') }) + +test('Support mapbox vector tile', t => { + t.plan(2) + function handler (req, res) { + res.setHeader('Content-Type', 'application/vnd.mapbox-vector-tile') + res.end(Buffer.from('vector tile stuff')) + } + + buildServer(handler, ({ port }, server) => { + const pool = new ConnectionPool({ Connection }) + pool.addConnection(`http://localhost:${port}`) + + const transport = new Transport({ + emit: () => {}, + connectionPool: pool, + serializer: new Serializer(), + maxRetries: 3, + requestTimeout: 30000, + sniffInterval: false, + sniffOnStart: false + }) + skipProductCheck(transport) + + transport.request({ + method: 'GET', + path: '/hello' + }, (err, { body }) => { + t.error(err) + t.same(body.toString(), Buffer.from('vector tile stuff').toString()) + server.stop() + }) + }) +}) + +test('Compressed mapbox vector tile', t => { + t.plan(2) + function handler (req, res) { + const body = gzipSync(Buffer.from('vector tile stuff')) + res.setHeader('Content-Type', 'application/vnd.mapbox-vector-tile') + res.setHeader('Content-Encoding', 'gzip') + res.setHeader('Content-Length', Buffer.byteLength(body)) + res.end(body) + } + + buildServer(handler, ({ port }, server) => { + const pool = new ConnectionPool({ Connection }) + pool.addConnection(`http://localhost:${port}`) + + const transport = new Transport({ + emit: () => {}, + connectionPool: pool, + serializer: new Serializer(), + maxRetries: 3, + requestTimeout: 30000, + sniffInterval: false, + sniffOnStart: false + }) + skipProductCheck(transport) + + transport.request({ + method: 'GET', + path: '/hello' + }, (err, { body }) => { + t.error(err) + t.same(body.toString(), Buffer.from('vector tile stuff').toString()) + server.stop() + }) + }) +}) From 507e1ec9a81abf79ad0abfcd9981683fc0fd2c79 Mon Sep 17 00:00:00 2001 From: delvedor Date: Fri, 23 Jul 2021 17:20:06 +0200 Subject: [PATCH 3/4] Use real mvt payload --- test/unit/transport.test.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/unit/transport.test.js b/test/unit/transport.test.js index 9c7658dca..e1a6aaa9f 100644 --- a/test/unit/transport.test.js +++ b/test/unit/transport.test.js @@ -2692,9 +2692,11 @@ test('The callback with a sync error should be called in the next tick - ndjson' test('Support mapbox vector tile', t => { t.plan(2) + const mvtContent = 'GoMCCgRtZXRhEikSFAAAAQACAQMBBAAFAgYDBwAIBAkAGAMiDwkAgEAagEAAAP8//z8ADxoOX3NoYXJkcy5mYWlsZWQaD19zaGFyZHMuc2tpcHBlZBoSX3NoYXJkcy5zdWNjZXNzZnVsGg1fc2hhcmRzLnRvdGFsGhlhZ2dyZWdhdGlvbnMuX2NvdW50LmNvdW50GhdhZ2dyZWdhdGlvbnMuX2NvdW50LnN1bRoTaGl0cy50b3RhbC5yZWxhdGlvbhoQaGl0cy50b3RhbC52YWx1ZRoJdGltZWRfb3V0GgR0b29rIgIwACICMAIiCRkAAAAAAAAAACIECgJlcSICOAAogCB4Ag==' + function handler (req, res) { res.setHeader('Content-Type', 'application/vnd.mapbox-vector-tile') - res.end(Buffer.from('vector tile stuff')) + res.end(Buffer.from(mvtContent, 'base64')) } buildServer(handler, ({ port }, server) => { @@ -2717,7 +2719,7 @@ test('Support mapbox vector tile', t => { path: '/hello' }, (err, { body }) => { t.error(err) - t.same(body.toString(), Buffer.from('vector tile stuff').toString()) + t.same(body.toString('base64'), Buffer.from(mvtContent, 'base64').toString('base64')) server.stop() }) }) @@ -2725,8 +2727,10 @@ test('Support mapbox vector tile', t => { test('Compressed mapbox vector tile', t => { t.plan(2) + const mvtContent = 'GoMCCgRtZXRhEikSFAAAAQACAQMBBAAFAgYDBwAIBAkAGAMiDwkAgEAagEAAAP8//z8ADxoOX3NoYXJkcy5mYWlsZWQaD19zaGFyZHMuc2tpcHBlZBoSX3NoYXJkcy5zdWNjZXNzZnVsGg1fc2hhcmRzLnRvdGFsGhlhZ2dyZWdhdGlvbnMuX2NvdW50LmNvdW50GhdhZ2dyZWdhdGlvbnMuX2NvdW50LnN1bRoTaGl0cy50b3RhbC5yZWxhdGlvbhoQaGl0cy50b3RhbC52YWx1ZRoJdGltZWRfb3V0GgR0b29rIgIwACICMAIiCRkAAAAAAAAAACIECgJlcSICOAAogCB4Ag==' + function handler (req, res) { - const body = gzipSync(Buffer.from('vector tile stuff')) + const body = gzipSync(Buffer.from(mvtContent, 'base64')) res.setHeader('Content-Type', 'application/vnd.mapbox-vector-tile') res.setHeader('Content-Encoding', 'gzip') res.setHeader('Content-Length', Buffer.byteLength(body)) @@ -2753,7 +2757,7 @@ test('Compressed mapbox vector tile', t => { path: '/hello' }, (err, { body }) => { t.error(err) - t.same(body.toString(), Buffer.from('vector tile stuff').toString()) + t.same(body.toString('base64'), Buffer.from(mvtContent, 'base64').toString('base64')) server.stop() }) }) From 1316da464b5045a4e0ab08d93ab737ee6da9db04 Mon Sep 17 00:00:00 2001 From: delvedor Date: Fri, 23 Jul 2021 17:22:59 +0200 Subject: [PATCH 4/4] Updated docs --- docs/transport.asciidoc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/transport.asciidoc b/docs/transport.asciidoc index f4fc722ce..308e7098e 100644 --- a/docs/transport.asciidoc +++ b/docs/transport.asciidoc @@ -32,3 +32,10 @@ class MyTransport extends Transport { } ---- +==== Supported content types + +- `application/json`, in this case the transport will return a plain JavaScript object +- `text/plain`, in this case the transport will return a plain string +- `application/vnd.mapbox-vector-tile`, in this case the transport will return a Buffer +- `application/vnd.elasticsearch+json`, in this case the transport will return a plain JavaScript object +