Mapbox Vector Tiles loader (#624)

docs/specifications/category-gis.md

@@ -6,10 +6,11 @@ Several geospatial formats return data in the form of lists of lng/lat encoded g
 
 ## Geospatial Category Loaders
 
-| Loader                                                   | Notes |
-| -------------------------------------------------------- | ----- |
-| [`KMLLoader`](modules/kml/docs/api-reference/kml-loader) |       |
-| [`WKTLoader`](modules/wkt/docs/api-reference/wkt-loader) |       |
+| Loader                                                                                              | Notes |
+| --------------------------------------------------------------------------------------------------- | ----- |
+| [`KMLLoader`](modules/kml/docs/api-reference/kml-loader)                                            |       |
+| [`WKTLoader`](modules/wkt/docs/api-reference/wkt-loader)                                            |       |
+| [`MapboxVectorTileLoader`](modules/mapbox-vector-tile/docs/api-reference/mapbox-vector-tile-loader) |       |
 
 ## Data Format
 

modules/mapbox-vector-tile/README.md

@@ -0,0 +1,5 @@
+# @loaders.gl/mapbox-vector-tile
+
+This module contains a geometry loader for Mapbox Vector Tiles (MVT).
+
+[loaders.gl](https://loaders.gl/docs) is a collection of framework independent visualization-focused loaders (parsers).

modules/mapbox-vector-tile/docs/README.md

@@ -0,0 +1,20 @@
+# @loaders.gl/mapbox-vector-tile
+
+The `@loaders.gl/mapbox-vector-tile` module handles the [Mapbox Vector Tile](https://github.com/mapbox/vector-tile-spec) format, a protobuf encoded format that defines geospatial geometries.
+
+## Installation
+
+```bash
+npm install @loaders.gl/mapbox-vector-tile
+npm install @loaders.gl/core
+```
+
+## Loaders and Writers
+
+| Loader                                                                                               |
+| ---------------------------------------------------------------------------------------------------- |
+| [`MapboxVectorTileLoader`](modules/mapbox-vector-tiles/docs/api-reference/mapbox-vector-tile-loader) |
+
+## Attribution
+
+The `MapboxVectorTileLoader` uses [`@mapbox/vector-tile`](https://github.com/mapbox/vector-tile-js) module under the BSD-3-Clause.

modules/mapbox-vector-tile/docs/api-reference/mapbox-vector-tile-loader.md

@@ -0,0 +1,28 @@
+# MapboxVectorTileLoader
+
+Loader for the [Mapbox Vector Tile](https://docs.mapbox.com/vector-tiles/specification/) format for representation of geometry.
+
+| Loader         | Characteristic                                                            |
+| -------------- | ------------------------------------------------------------------------- |
+| File Extension | `.mvt`,                                                                   |
+| File Type      | Binary                                                                    |
+| File Format    | [Mapbox Vector Tile](https://docs.mapbox.com/vector-tiles/specification/) |
+| Data Format    | [Geometry](/docs/specifications/category-gis)                             |
+| Supported APIs | `load`, `parse`, `parseSync`                                              |
+
+## Usage
+
+```js
+import {MapboxVectorTileLoader} from '@loaders.gl/mapbox-vector-tile';
+import {load} from '@loaders.gl/core';
+
+const data = await load(url, MapboxVectorTileLoader);
+```
+
+## Options
+
+N/A
+
+## Attribution
+
+The `MapboxVectorTileLoader` uses [`@mapbox/vector-tile`](https://github.com/mapbox/vector-tile-js) module under the BSD-3-Clause.

modules/mapbox-vector-tile/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@loaders.gl/mapbox-vector-tile",
+  "description": "Loader for Mapbox Vector Tiles",
+  "version": "2.1.0-alpha.1",
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/uber-web/loaders.gl"
+  },
+  "keywords": [
+    "geometry",
+    "loader",
+    "parser",
+    "MVT",
+    "Mapbox Vector Tiles"
+  ],
+  "main": "dist/es5/index.js",
+  "module": "dist/esm/index.js",
+  "esnext": "dist/es6/index.js",
+  "sideEffects": false,
+  "files": [
+    "src",
+    "dist",
+    "README.md"
+  ],
+  "scripts": {
+    "pre-build": "npm run build-bundle && npm run build-bundle -- --env.dev",
+    "build-bundle": "webpack --display=minimal --config ../../scripts/bundle.config.js"
+  },
+  "devDependencies": {
+    "@mapbox/vector-tile": "^1.3.1",
+    "pbf": "^3.2.1"
+  }
+}

modules/mapbox-vector-tile/src/bundle.js

@@ -0,0 +1,7 @@
+/* global window, global */
+const moduleExports = require('./index');
+
+const _global = typeof window === 'undefined' ? global : window;
+_global.loaders = _global.loaders || {};
+
+module.exports = Object.assign(_global.loaders, moduleExports);

modules/mapbox-vector-tile/src/index.js

@@ -0,0 +1 @@
+export {MapboxVectorTileLoader} from './mvt-loader';

modules/mapbox-vector-tile/src/lib/parse-mvt.js

@@ -0,0 +1,35 @@
+import {VectorTile} from '@mapbox/vector-tile';
+import Protobuf from 'pbf';
+
+/*
+  * Parse MVT arrayBuffer and return GeoJSON.
+  *
+  * @param {arrayBuffer} _ A MVT arrayBuffer
+  * @return {?Object} A GeoJSON geometry object
+  */
+export default function parseMVT(input, options) {
+  if (input.byteLength === 0) {
+    return [];
+  }
+
+  const tile = new VectorTile(new Protobuf(input));
+  const features = [];
+
+  for (const layerName in tile.layers) {
+    const vectorTileLayer = tile.layers[layerName];
+
+    for (let i = 0; i < vectorTileLayer.length; i++) {
+      const vectorTileFeature = vectorTileLayer.feature(i);
+
+      const feature = vectorTileFeature.toGeoJSON(
+        options.tileProperties.x,
+        options.tileProperties.y,
+        options.tileProperties.z
+      );
+
+      features.push(feature);
+    }
+  }
+
+  return features;
+}

modules/mapbox-vector-tile/src/mvt-loader.js

@@ -0,0 +1,22 @@
+/* global __VERSION__ */ // __VERSION__ is injected by babel-plugin-version-inline
+import parseMVT from './lib/parse-mvt';
+
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+export const MapboxVectorTileLoader = {
+  id: 'mvt',
+  name: 'Mapbox Vector Tile',
+  version: VERSION,
+  extensions: ['mvt'],
+  mimeType: 'application/x-protobuf',
+  category: 'geometry',
+  parse: async (arrayBuffer, options) => parseMVT(arrayBuffer, options),
+  parseSync: parseMVT,
+  binary: true,
+  testText: null,
+  options: {
+    mvt: {
+      tileProperties: {}
+    }
+  }
+};

modules/mapbox-vector-tile/test/data/README.md

@@ -0,0 +1,3 @@
+# Data Attribution
+
+- gadm_usa_3-0-3.mvt from GADM which is free for non-comercial use

modules/mapbox-vector-tile/test/index.js

@@ -0,0 +1,3 @@
+import './mvt/mvt-points.spec';
+import './mvt/mvt-lines.spec';
+import './mvt/mvt-polygons.spec';

modules/mapbox-vector-tile/test/mvt/mvt-lines.spec.js

@@ -0,0 +1,26 @@
+/* eslint-disable */
+
+import test from 'tape-promise/tape';
+import {MVTLoader} from '@loaders.gl/mapbox-vector-tile';
+import {fetchFile, parse} from '@loaders.gl/core';
+
+import decodedGeoJSON from '../results/decoded_mvt_lines.json';
+
+const MVT_LINES_DATA_URL = '@loaders.gl/mapbox-vector-tile/test/data/lines_generated_5-16-11.mvt';
+
+test('Lines MVT', async t => {
+  const response = await fetchFile(MVT_LINES_DATA_URL);
+  const mvtArrayBuffer = await response.arrayBuffer();
+
+  const loaderOptions = {
+    tileProperties: {
+      x: 16,
+      y: 11,
+      z: 5
+    }
+  };
+
+  t.deepEqual(parse(mvtArrayBuffer, MVTLoader, loaderOptions), decodedGeoJSON);
+
+  t.end();
+});

modules/mapbox-vector-tile/test/mvt/mvt-points.spec.js

@@ -0,0 +1,26 @@
+/* eslint-disable */
+
+import test from 'tape-promise/tape';
+import {MVTLoader} from '@loaders.gl/mapbox-vector-tile';
+import {fetchFile, parse} from '@loaders.gl/core';
+
+import decodedGeoJSON from '../results/decoded_mvt_points.json';
+
+const MVT_POINTS_DATA_URL = '@loaders.gl/mapbox-vector-tile/test/data/points_generated_2-0-1.mvt';
+
+test('Point MVT', async t => {
+  const response = await fetchFile(MVT_POINTS_DATA_URL);
+  const mvtArrayBuffer = await response.arrayBuffer();
+
+  const loaderOptions = {
+    tileProperties: {
+      x: 0,
+      y: 1,
+      z: 2
+    }
+  };
+
+  t.deepEqual(parse(mvtArrayBuffer, MVTLoader, loaderOptions), decodedGeoJSON);
+
+  t.end();
+});

modules/mapbox-vector-tile/test/mvt/mvt-polygons.spec.js

@@ -0,0 +1,26 @@
+/* eslint-disable */
+
+import test from 'tape-promise/tape';
+import {MVTLoader} from '@loaders.gl/mapbox-vector-tile';
+import {fetchFile, parse} from '@loaders.gl/core';
+
+import decodedGeoJSON from '../results/decoded_mvt_polygons.json';
+
+const MVT_POLYGONS_DATA_URL = '@loaders.gl/mapbox-vector-tile/test/data/gadm_usa_3-0-3.mvt';
+
+test('Polygons MVT', async t => {
+  const response = await fetchFile(MVT_POLYGONS_DATA_URL);
+  const mvtArrayBuffer = await response.arrayBuffer();
+
+  const loaderOptions = {
+    tileProperties: {
+      x: 0,
+      y: 3,
+      z: 3
+    }
+  };
+
+  t.deepEqual(parse(mvtArrayBuffer, MVTLoader, loaderOptions), decodedGeoJSON);
+
+  t.end();
+});

test/aliases.js

@@ -35,6 +35,10 @@ const makeAliases = () => ({
   '@loaders.gl/kml/test': path.resolve(__dirname, '../modules/kml/test'),
   '@loaders.gl/i3s/test': path.resolve(__dirname, '../modules/i3s/test'),
   '@loaders.gl/las/test': path.resolve(__dirname, '../modules/las/test'),
+  '@loaders.gl/mapbox-vector-tile/test': path.resolve(
+    __dirname,
+    '../modules/mapbox-vector-tile/test'
+  ),
   '@loaders.gl/obj/test': path.resolve(__dirname, '../modules/obj/test'),
   '@loaders.gl/pcd/test': path.resolve(__dirname, '../modules/pcd/test'),
   '@loaders.gl/ply/test': path.resolve(__dirname, '../modules/ply/test'),

test/modules.js

@@ -35,6 +35,7 @@ require('@loaders.gl/potree/test');
 // Geospatial Formats
 require('@loaders.gl/kml/test');
 require('@loaders.gl/wkt/test');
+require('@loaders.gl/mapbox-vector-tile/test');
 
 // Table Formats
 require('@loaders.gl/tables/test');

yarn.lock

@@ -1529,6 +1529,18 @@
     "@luma.gl/gltools" "8.0.1"
     probe.gl "^3.2.0"
 
+"@mapbox/point-geometry@~0.1.0":
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/@mapbox/point-geometry/-/point-geometry-0.1.0.tgz#8a83f9335c7860effa2eeeca254332aa0aeed8f2"
+  integrity sha1-ioP5M1x4YO/6Lu7KJUMyqgru2PI=
+
+"@mapbox/vector-tile@^1.3.1":
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/@mapbox/vector-tile/-/vector-tile-1.3.1.tgz#d3a74c90402d06e89ec66de49ec817ff53409666"
+  integrity sha512-MCEddb8u44/xfQ3oD+Srl/tNcQoqTw3goGk2oLsrFxOTc3dUp+kAnby3PvAeeBYSMSjSPD1nd1AJA6W49WnoUw==
+  dependencies:
+    "@mapbox/point-geometry" "~0.1.0"
+
 "@math.gl/core@3.1.2":
   version "3.1.2"
   resolved "https://registry.yarnpkg.com/@math.gl/core/-/core-3.1.2.tgz#226472efe83388ca454ee7ab1912358c299f8bc1"
@@ -5207,7 +5219,7 @@ iconv-lite@0.4, iconv-lite@0.4.24, iconv-lite@^0.4.24, iconv-lite@~0.4.13:
   dependencies:
     safer-buffer ">= 2.1.2 < 3"
 
-ieee754@^1.1.4:
+ieee754@^1.1.12, ieee754@^1.1.4:
   version "1.1.13"
   resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84"
   integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==
@@ -7491,6 +7503,14 @@ path-type@^3.0.0:
   dependencies:
     pify "^3.0.0"
 
+pbf@^3.2.1:
+  version "3.2.1"
+  resolved "https://registry.yarnpkg.com/pbf/-/pbf-3.2.1.tgz#b4c1b9e72af966cd82c6531691115cc0409ffe2a"
+  integrity sha512-ClrV7pNOn7rtmoQVF4TS1vyU0WhYRnP92fzbfF75jAIwpnzdJXf8iTd4CMEqO4yUenH6NDqLiwjqlh6QgZzgLQ==
+  dependencies:
+    ieee754 "^1.1.12"
+    resolve-protobuf-schema "^2.1.0"
+
 pbkdf2@^3.0.3:
   version "3.0.17"
   resolved "https://registry.yarnpkg.com/pbkdf2/-/pbkdf2-3.0.17.tgz#976c206530617b14ebb32114239f7b09336e93a6"
@@ -7731,6 +7751,11 @@ proto-list@~1.2.1:
   resolved "https://registry.yarnpkg.com/proto-list/-/proto-list-1.2.4.tgz#212d5bfe1318306a420f6402b8e26ff39647a849"
   integrity sha1-IS1b/hMYMGpCD2QCuOJv85ZHqEk=
 
+protocol-buffers-schema@^3.3.1:
+  version "3.3.2"
+  resolved "https://registry.yarnpkg.com/protocol-buffers-schema/-/protocol-buffers-schema-3.3.2.tgz#00434f608b4e8df54c59e070efeefc37fb4bb859"
+  integrity sha512-Xdayp8sB/mU+sUV4G7ws8xtYMGdQnxbeIfLjyO9TZZRJdztBGhlmbI5x1qcY4TG5hBkIKGnc28i7nXxaugu88w==
+
 protocols@^1.1.0, protocols@^1.4.0:
   version "1.4.7"
   resolved "https://registry.yarnpkg.com/protocols/-/protocols-1.4.7.tgz#95f788a4f0e979b291ffefcf5636ad113d037d32"
@@ -8275,6 +8300,13 @@ resolve-from@^4.0.0:
   resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6"
   integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==
 
+resolve-protobuf-schema@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/resolve-protobuf-schema/-/resolve-protobuf-schema-2.1.0.tgz#9ca9a9e69cf192bbdaf1006ec1973948aa4a3758"
+  integrity sha512-kI5ffTiZWmJaS/huM8wZfEMer1eRd7oJQhDuxeCLe3t7N7mX3z94CN0xPxBQxFYQTSNz9T0i+v6inKqSdK8xrQ==
+  dependencies:
+    protocol-buffers-schema "^3.3.1"
+
 resolve-url@^0.2.1:
   version "0.2.1"
   resolved "https://registry.yarnpkg.com/resolve-url/-/resolve-url-0.2.1.tgz#2c637fe77c893afd2a663fe21aa9080068e2052a"