From 5c8d7a570462c0ee547be312d6770cbdaebe7d30 Mon Sep 17 00:00:00 2001 From: missinglink Date: Wed, 9 May 2018 18:19:46 +0200 Subject: [PATCH] protobuf: version one --- codec/codec.js | 182 +++++++++++++++++++++++++++++++++++++ codec/proto/example.proto | 12 +++ codec/proto/model.v1.proto | 47 ++++++++++ codec/stream.js | 0 package.json | 5 +- test/codec/codec.js | 140 ++++++++++++++++++++++++++++ test/codec/stream.js | 0 test/run.js | 1 + 8 files changed, 386 insertions(+), 1 deletion(-) create mode 100644 codec/codec.js create mode 100644 codec/proto/example.proto create mode 100644 codec/proto/model.v1.proto create mode 100644 codec/stream.js create mode 100644 test/codec/codec.js create mode 100644 test/codec/stream.js diff --git a/codec/codec.js b/codec/codec.js new file mode 100644 index 0000000..c0a610b --- /dev/null +++ b/codec/codec.js @@ -0,0 +1,182 @@ + +const fs = require('fs'); +const path = require('path'); +const _ = require('lodash'); +const protobuf = require('protocol-buffers'); +const json = require('json-protobuf-encoding'); +const wkx = require('wkx'); +const Document = require('../Document'); + +// example codec +module.exports.example = load('example.proto'); + +// // AFAIK protobuf cannot encode a map of arrays of strings without +// // adding an additional property (which here is named '_') +// const StringList = { +// encode: (l) => { return { _: l.map(v => null === v ? '' : v) }; }, +// decode: (l) => l._.map(v => '' === v ? null : v) +// }; + +// this codec 'flattens' (ie casts) single element arrays to a scalar string +// const StringListFlatten = { +// encode: (l) => StringList.encode( Array.isArray(l) ? l : [ l ] ), +// decode: function(l){ +// let d = StringList.decode(l); +// return d.length > 1 ? d : d[0]; +// } +// }; + +// convenience function for operating on maps +// const Map = function(valueCodec){ +// return { +// encode: (m) => _.mapValues(m, valueCodec.encode), +// decode: (m) => _.mapValues(m, valueCodec.decode) +// }; +// }; + +// model codec v1 +// module.exports.v1 = load('model.v1.proto'); +module.exports.v1 = (function(){ + let c = load('model.v1.proto'); + return { + Location: { + encode: (doc) => { + + let geom = { Point: {}, Polygon: {} }; + if(!_.isEmpty(doc.center_point)){ + geom.Point.Centroid = new wkx.Point(doc.center_point.lon, doc.center_point.lat).toWkb(); + } + if(!_.isEmpty(doc.getPolygon())){ + geom.Polygon.Shape = wkx.Geometry.parseGeoJSON( doc.getPolygon() ).toWkb(); + } + if(!_.isEmpty(doc.getBoundingBox())){ + let parsed = JSON.parse( doc.getBoundingBox() ); + geom.Polygon.BoundingBox = wkx.Geometry.parseGeoJSON({ + 'type': 'Polygon', + 'coordinates': [[ + [ parsed.min_lon, parsed.min_lat ], + [ parsed.min_lon, parsed.max_lat ], + [ parsed.max_lon, parsed.max_lat ], + [ parsed.max_lon, parsed.min_lat ], + [ parsed.min_lon, parsed.min_lat ] + ]] + }).toWkb(); + } + + let names = {}; + for( var attr in doc.name ){ + names[attr] = { + Variants: Array.isArray(doc.name[attr]) ? doc.name[attr] : [doc.name[attr]] + }; + } + + let hierarchy = { Levels: {} }; + for( attr in doc.parent ){ + if( !attr.includes('_') ){ + if( !hierarchy.Levels.hasOwnProperty( attr ) ){ + hierarchy.Levels[ attr ] = { Names: {} }; + } + hierarchy.Levels[ attr ].Names.und = { + Variants: doc.parent[attr] + }; + } + else if( /^(.*)_id$/.test(attr) ){ + let idMatch = attr.match(/^(.*)_id$/); + if( !hierarchy.Levels.hasOwnProperty( idMatch[1] ) ){ + hierarchy.Levels[ idMatch[1] ] = { Names: {} }; + } + hierarchy.Levels[ idMatch[1] ].Id = doc.parent[attr][0]; + } + else if( /^(.*)_a$/.test(attr) ){ + let idMatch = attr.match(/^(.*)_a$/); + if( !hierarchy.Levels.hasOwnProperty( idMatch[1] ) ){ + hierarchy.Levels[ idMatch[1] ] = { Names: {} }; + } + hierarchy.Levels[ idMatch[1] ].Names.abbr = { + Variants: doc.parent[attr].map(v => null === v ? '' : v) + }; + } + } + + // console.error(JSON.stringify(hierarchy, null, 2)) + + return c.Location.encode({ + Identity: { + Source: doc.getSource(), + Layer: doc.getLayer(), + SourceId: doc.getSourceId() + }, + Tags: doc._meta || {}, + Names: names, + Hierarchies: { + 'WOF': hierarchy + }, + Address: doc.address_parts, + Categories: doc.category || [], + Stats: { + Population: doc.getPopulation(), + Popularity: doc.getPopularity() + }, + Geometry: geom + }); + }, + decode: (buf) => { + let raw = c.Location.decode(buf); + let doc = new Document(raw.Identity.Source, raw.Identity.Layer, raw.Identity.SourceId); + doc._meta = raw.Tags; + if(!_.isEmpty(raw.Names)){ + doc.name = {}; + doc.phrase = {}; + for( var attr in raw.Names ){ + let n = raw.Names[ attr ]; + doc.name[ attr ] = ( n.Variants.length > 1 ) ? n.Variants : n.Variants[0]; + doc.phrase[ attr ] = ( n.Variants.length > 1 ) ? n.Variants : n.Variants[0]; + } + } + if(!_.isEmpty(raw.Hierarchies)){ + doc.parent = {}; + let hierarchy = raw.Hierarchies.WOF; + for( var level in hierarchy.Levels ){ + doc.parent[ level ] = hierarchy.Levels[level].Names.und.Variants; + doc.parent[ level + '_id' ] = [ hierarchy.Levels[level].Id ]; + doc.parent[ level + '_a' ] = hierarchy.Levels[level].Names.abbr.Variants.map(v => '' === v ? null : v); + } + } + // doc.parent = Map(StringList).decode(raw.Hierarchy); + doc.address_parts = raw.Address; + doc.category = raw.Categories || []; + if(!_.isEmpty(raw.Stats)){ + if(raw.Stats.Population>0){ doc.population = raw.Stats.Population; } + if(raw.Stats.Popularity>0){ doc.popularity = raw.Stats.Popularity; } + } + if(!_.isEmpty(raw.Geometry)){ + if(!_.isEmpty(raw.Geometry.Point.Centroid)){ + let parsed = wkx.Geometry.parse(raw.Geometry.Point.Centroid); + doc.center_point = { lon: parsed.x, lat: parsed.y }; + } + if(!_.isEmpty(raw.Geometry.Polygon.Shape)){ + doc.shape = wkx.Geometry.parse(raw.Geometry.Polygon.Shape).toGeoJSON(); + } + if(!_.isEmpty(raw.Geometry.Polygon.BoundingBox)){ + let parsed = wkx.Geometry.parse( raw.Geometry.Polygon.BoundingBox ); + doc.bounding_box = JSON.stringify({ + min_lat: parsed.exteriorRing[0].y, + max_lat: parsed.exteriorRing[2].y, + min_lon: parsed.exteriorRing[0].x, + max_lon: parsed.exteriorRing[2].x + }); + } + } + return doc; + } + } + }; +})(); + +// generic protobuf loader +function load(protofile){ + return protobuf( + fs.readFileSync(path.join(__dirname, 'proto', protofile)), + { encodings: { json: json() }} + ); +} \ No newline at end of file diff --git a/codec/proto/example.proto b/codec/proto/example.proto new file mode 100644 index 0000000..ca1b281 --- /dev/null +++ b/codec/proto/example.proto @@ -0,0 +1,12 @@ +enum FOO { + BAR = 1; +} + +message Test { + required float num = 1; + required string payload = 2; +} + +message AnotherOne { + repeated FOO list = 1; +} \ No newline at end of file diff --git a/codec/proto/model.v1.proto b/codec/proto/model.v1.proto new file mode 100644 index 0000000..9039364 --- /dev/null +++ b/codec/proto/model.v1.proto @@ -0,0 +1,47 @@ +syntax = "proto3"; + +message Identity { + string Source = 1; + string Layer = 2; + string SourceId = 3; +} + +message Language { + string Code = 1; + string SubTag = 2; + string PrivateTag = 3; +} + +message Name { + Language Language = 1; + repeated string Variants = 2; +} + +message Level { + string Id = 1; + map Names = 2; +} + +message Hierarchy { + map Levels = 1; +} + +message Geometry { + map Point = 1; + map MultiPoint = 2; + map Line = 3; + map MultiLine = 4; + map Polygon = 5; + map MultiPolygon = 6; +} + +message Location { + Identity Identity = 1; + map Tags = 2; + map Names = 3; + map Hierarchies = 4; + map Address = 5; + map Stats = 6; + repeated string Categories = 7; + Geometry Geometry = 8; +} \ No newline at end of file diff --git a/codec/stream.js b/codec/stream.js new file mode 100644 index 0000000..e69de29 diff --git a/package.json b/package.json index 47d9c28..6c2513b 100644 --- a/package.json +++ b/package.json @@ -31,9 +31,12 @@ "npm": ">=1.4.3" }, "dependencies": { + "json-protobuf-encoding": "^1.0.1", "lodash": "^4.6.1", "pelias-config": "2.14.0", - "through2": "^2.0.0" + "protocol-buffers": "^4.0.4", + "through2": "^2.0.0", + "wkx": "^0.4.4" }, "devDependencies": { "event-stream": "^3.3.2", diff --git a/test/codec/codec.js b/test/codec/codec.js new file mode 100644 index 0000000..b0ddca8 --- /dev/null +++ b/test/codec/codec.js @@ -0,0 +1,140 @@ +const codec = require('../../codec/codec.js'); +const Document = require('../../Document.js'); + +module.exports.tests = {}; + +module.exports.tests.interface = function(test) { + test('valid interface', function(t) { + // t.equal(typeof Document, 'function', 'Document is a function'); + + // t.equal(typeof Document.prototype.getId, 'function', 'getId() is a function'); + // t.equal(typeof Document.prototype.setId, 'function', 'setId() is a function'); + + // t.equal(typeof Document.prototype.getType, 'function', 'getType() is a function'); + // t.equal(typeof Document.prototype.setType, 'function', 'setType() is a function'); + + // t.equal(typeof Document.prototype.getSource, 'function', 'getSource() is a function'); + // t.equal(typeof Document.prototype.setSource, 'function', 'setSource() is a function'); + + // t.equal(typeof Document.prototype.getLayer, 'function', 'getLayer() is a function'); + // t.equal(typeof Document.prototype.setLayer, 'function', 'setLayer() is a function'); + + t.end(); + }); +}; + +// --- example codec --- + +module.exports.tests.exampleEncode = function(test) { + test('exampleEncode', function(t) { + + const encoded = new Buffer('\r\x00\x00(B\x12\x0bhello world'); + const decoded = { + num: 42, + payload: 'hello world' + }; + + t.deepEqual(codec.example.Test.encode(decoded), encoded); + t.end(); + }); +}; + +module.exports.tests.exampleDecode = function(test) { + test('exampleDecode', function(t) { + + const encoded = new Buffer('\r\x00\x00(B\x12\x0bhello world'); + const decoded = { + num: 42, + payload: 'hello world' + }; + + t.deepEqual(codec.example.Test.decode(encoded), decoded); + t.end(); + }); +}; + +// --- v1 codec --- + +module.exports.tests.v1Encode = function(test) { + test('v1Encode', function(t) { + + const encoded = new Buffer('\n\x19\n\bmysource\x12\x07mylayer\x1a\x04myid' + + '\x12\n\n\x02id\x12\x04myid\x12\x0f\n\x04type\x12\x07mylayer\x1a\x12\n\x06myprop' + + '\x12\b\x12\x06myname"\x07\n\x03WOF\x12\x002\f\n\nPopulation2\f\n\nPopularityB\x00'); + + var doc = new Document('mysource','mylayer','myid'); + doc.setName('myprop', 'myname'); + + t.deepEqual(codec.v1.Location.encode(doc), encoded); + t.end(); + }); +}; + +module.exports.tests.v1Decode = function(test) { + test('v1Decode', function(t) { + + const encoded = new Buffer('\n\x19\n\bmysource\x12\x07mylayer\x1a\x04myid' + + '\x12\n\n\x02id\x12\x04myid\x12\x0f\n\x04type\x12\x07mylayer\x1a\x12\n\x06myprop' + + '\x12\b\x12\x06myname"\x07\n\x03WOF\x12\x002\f\n\nPopulation2\f\n\nPopularityB\x00'); + + var doc = new Document('mysource','mylayer','myid'); + doc.setName('myprop', 'myname'); + + t.deepEqual(codec.v1.Location.decode(encoded), doc); + t.end(); + }); +}; + +module.exports.tests.v1Symmetry = function(test) { + test('v1Symmetry', function(t) { + + var doc = new Document('mysource','mylayer','myid') + .setMeta( 'author', 'peter' ) + .setName( 'default', 'Hackney City Farm' ) + .setName( 'alt', 'Haggerston City Farm' ) + .setNameAlias( 'alt', 'Haggerston Farm' ) + .addParent( 'country', 'Great Britain', '1001', 'GreatB' ) + .addParent( 'neighbourhood', 'Shoreditch', '2002' ) + .setAddress('name', 'address name') + .setAddress('number', 'address number') + .setAddress('street', 'address street') + .setAddress('zip', 'address zip') + .setAddress('unit', 'address unit') + .addCategory( 'foo' ) + .addCategory( 'bar' ) + .removeCategory( 'foo' ) + .setPopulation(10) + .setPopularity(3) + .setCentroid({ lon: 0.5, lat: 50.1 }) + .setPolygon({ type: 'Point', coordinates: [1, 2] }) + .setBoundingBox({ + upperLeft: { + lat: 13.131313, + lon: 21.212121 + }, + lowerRight: { + lat: 12.121212, + lon: 31.313131 + } + }); + + let encoded = codec.v1.Location.encode(doc); + let decoded = codec.v1.Location.decode(encoded); + + t.deepEqual(decoded, doc); + t.deepEqual(decoded._meta, doc._meta); + t.deepEqual(encoded.length, 614); + t.end(); + }); +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('codec: ' + name, testFunction); + } + + for( var testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; diff --git a/test/codec/stream.js b/test/codec/stream.js new file mode 100644 index 0000000..e69de29 diff --git a/test/run.js b/test/run.js index 038fb9d..2359dce 100644 --- a/test/run.js +++ b/test/run.js @@ -25,6 +25,7 @@ var tests = [ require('./util/transform.js'), require('./util/valid.js'), require('./serialize/test.js'), + require('./codec/codec.js') ]; tests.map(function(t) {