From 522a8ced2df4ea93b40a85f6dbad6ebffd490e9f Mon Sep 17 00:00:00 2001 From: Stephen Hess Date: Fri, 22 Apr 2016 12:57:36 -0400 Subject: [PATCH] modified peliasDocGenerators to accommodate multiple hierarchies --- src/peliasDocGenerators.js | 185 ++++++++++++++++++-------------- test/peliasDocGeneratorsTest.js | 91 ++++++++++------ 2 files changed, 168 insertions(+), 108 deletions(-) diff --git a/src/peliasDocGenerators.js b/src/peliasDocGenerators.js index 53c693c4..61123ff4 100644 --- a/src/peliasDocGenerators.js +++ b/src/peliasDocGenerators.js @@ -1,4 +1,4 @@ -var map_stream = require('through2-map'); +var through2 = require('through2'); var _ = require('lodash'); var iso3166 = require('iso3166-1'); @@ -6,90 +6,119 @@ var Document = require('pelias-model').Document; module.exports = {}; -module.exports.create = function(hierarchy_finder) { - return map_stream.obj(function(record) { - var wofDoc = new Document( 'whosonfirst', record.place_type, record.id ); - - if (record.name) { - wofDoc.setName('default', record.name); - } - wofDoc.setCentroid({ lat: record.lat, lon: record.lon }); +function assignField(hierarchyElement, wofDoc) { + switch (hierarchyElement.place_type) { + case 'locality': + wofDoc.addParent('locality', hierarchyElement.name, hierarchyElement.id.toString()); + break; + case 'borough': + wofDoc.addParent('borough', hierarchyElement.name, hierarchyElement.id.toString()); + break; + case 'localadmin': + wofDoc.addParent('localadmin', hierarchyElement.name, hierarchyElement.id.toString()); + break; + case 'county': + wofDoc.addParent('county', hierarchyElement.name, hierarchyElement.id.toString()); + break; + case 'macrocounty': + wofDoc.addParent('macrocounty', hierarchyElement.name, hierarchyElement.id.toString()); + break; + case 'region': + // if the region has an abbreviation, set it + if (hierarchyElement.abbreviation) { + wofDoc.addParent('region', hierarchyElement.name, hierarchyElement.id.toString(), hierarchyElement.abbreviation); + } else { + wofDoc.addParent('region', hierarchyElement.name, hierarchyElement.id.toString()); + } + break; + case 'macroregion': + wofDoc.addParent('macroregion', hierarchyElement.name, hierarchyElement.id.toString()); + break; + case 'country': + // this is placetype=country, so lookup and set the iso3 from iso2 + if (iso3166.is2(hierarchyElement.iso2)) { + var iso3 = iso3166.to3(hierarchyElement.iso2); - // only set population if available - if (record.population) { - wofDoc.setPopulation(record.population); - } + wofDoc.setAlpha3(iso3); + wofDoc.addParent('country', hierarchyElement.name, hierarchyElement.id.toString(), iso3); - // only set popularity if available - if (record.popularity) { - wofDoc.setPopularity(record.popularity); - } + } else { + wofDoc.addParent('country', hierarchyElement.name, hierarchyElement.id.toString()); - // WOF bbox is defined as: - // lowerLeft.lon, lowerLeft.lat, upperRight.lon, upperRight.lat - // so convert to what ES understands - if (!_.isUndefined(record.bounding_box)) { - var parsedBoundingBox = record.bounding_box.split(',').map(parseFloat); - var marshaledBoundingBoxBox = { - upperLeft: { - lat: parsedBoundingBox[3], - lon: parsedBoundingBox[0] - }, - lowerRight: { - lat: parsedBoundingBox[1], - lon: parsedBoundingBox[2] - } - - }; - wofDoc.setBoundingBox(marshaledBoundingBoxBox); - } + } - // iterate the hierarchy, assigning fields - hierarchy_finder(record).forEach(function(hierarchy_element) { - switch (hierarchy_element.place_type) { - case 'locality': - wofDoc.addParent('locality', hierarchy_element.name, hierarchy_element.id.toString()); - break; - case 'borough': - wofDoc.addParent('borough', hierarchy_element.name, hierarchy_element.id.toString()); - break; - case 'localadmin': - wofDoc.addParent('localadmin', hierarchy_element.name, hierarchy_element.id.toString()); - break; - case 'county': - wofDoc.addParent('county', hierarchy_element.name, hierarchy_element.id.toString()); - break; - case 'macrocounty': - wofDoc.addParent('macrocounty', hierarchy_element.name, hierarchy_element.id.toString()); - break; - case 'region': - if (hierarchy_element.abbreviation) { - wofDoc.addParent('region', hierarchy_element.name, hierarchy_element.id.toString(), hierarchy_element.abbreviation); - } else { - wofDoc.addParent('region', hierarchy_element.name, hierarchy_element.id.toString()); - } - break; - case 'macroregion': - wofDoc.addParent('macroregion', hierarchy_element.name, hierarchy_element.id.toString()); - break; - case 'country': - // this is placetype=country, so lookup and set the iso3 from iso2 - if (iso3166.is2(hierarchy_element.iso2)) { - var iso3 = iso3166.to3(hierarchy_element.iso2); - - wofDoc.setAlpha3(iso3); - wofDoc.addParent('country', hierarchy_element.name, hierarchy_element.id.toString(), iso3); - - } else { - wofDoc.addParent('country', hierarchy_element.name, hierarchy_element.id.toString()); - - } - - break; + break; + } + +} + +// method that extracts the logic for Document creation. `hierarchy` is optional +function setupDocument(record, hierarchy) { + var wofDoc = new Document( 'whosonfirst', record.place_type, record.id ); + + if (record.name) { + wofDoc.setName('default', record.name); + } + wofDoc.setCentroid({ lat: record.lat, lon: record.lon }); + + // only set population if available + if (record.population) { + wofDoc.setPopulation(record.population); + } + + // only set popularity if available + if (record.popularity) { + wofDoc.setPopularity(record.popularity); + } + + // WOF bbox is defined as: + // lowerLeft.lon, lowerLeft.lat, upperRight.lon, upperRight.lat + // so convert to what ES understands + if (!_.isUndefined(record.bounding_box)) { + var parsedBoundingBox = record.bounding_box.split(',').map(parseFloat); + var marshaledBoundingBoxBox = { + upperLeft: { + lat: parsedBoundingBox[3], + lon: parsedBoundingBox[0] + }, + lowerRight: { + lat: parsedBoundingBox[1], + lon: parsedBoundingBox[2] } + + }; + wofDoc.setBoundingBox(marshaledBoundingBoxBox); + } + + // a `hierarchy` is composed of potentially multiple WOF records, so iterate + // and assign fields + if (!_.isUndefined(hierarchy)) { + hierarchy.forEach(function(hierarchyElement) { + assignField(hierarchyElement, wofDoc); }); - return wofDoc; + } + + return wofDoc; + +} + +module.exports.create = function(hierarchy_finder) { + return through2.obj(function(record, enc, next) { + // if there are no hierarchies, then just return the doc as-is + var hierarchies = hierarchy_finder(record); + + if (hierarchies && hierarchies.length > 0) { + hierarchies.forEach(function(hierarchy) { + this.push(setupDocument(record, hierarchy)); + }, this); + + } else { + this.push(setupDocument(record)); + + } + + next(); }); diff --git a/test/peliasDocGeneratorsTest.js b/test/peliasDocGeneratorsTest.js index f447eaa0..d5c3200a 100644 --- a/test/peliasDocGeneratorsTest.js +++ b/test/peliasDocGeneratorsTest.js @@ -92,7 +92,7 @@ tape('create', function(test) { .setName('default', 'name 8') .setCentroid({ lat: 19.191919, lon: 91.919191 }) .addParent( 'locality', 'name 8', '8') - .addParent('borough', 'name 7', '7') + .addParent( 'borough', 'name 7', '7') .addParent( 'localadmin', 'name 6', '6') .addParent( 'county', 'name 5', '5') .addParent( 'macrocounty', 'name 4', '4') @@ -100,19 +100,32 @@ tape('create', function(test) { .addParent( 'macroregion', 'name 2', '2') .addParent( 'country', 'name 1', '1', 'DEU') .setAlpha3( 'DEU' ) - .setBoundingBox({ upperLeft: { lat:60.847893, lon:-13.691314 }, lowerRight: { lat:49.909613 , lon:1.771169 }}) + .setBoundingBox({ upperLeft: { lat:60.847893, lon:-13.691314 }, lowerRight: { lat:49.909613 , lon:1.771169 }}), + new Document( 'whosonfirst', 'locality', '8') + .setName('default', 'name 8') + .setCentroid({ lat: 19.191919, lon: 91.919191 }) + .addParent( 'locality', 'name 8', '8') + .addParent( 'country', 'name 1', '1', 'DEU') + .setAlpha3( 'DEU' ) + .setBoundingBox({ upperLeft: { lat:60.847893, lon:-13.691314 }, lowerRight: { lat:49.909613 , lon:1.771169 }}), ]; var hierarchies_finder = function() { return [ - wofRecords['8'], - wofRecords['7'], - wofRecords['6'], - wofRecords['5'], - wofRecords['4'], - wofRecords['3'], - wofRecords['2'], - wofRecords['1'] + [ + wofRecords['8'], + wofRecords['7'], + wofRecords['6'], + wofRecords['5'], + wofRecords['4'], + wofRecords['3'], + wofRecords['2'], + wofRecords['1'] + ], + [ + wofRecords['8'], + wofRecords['1'] + ] ]; }; @@ -227,7 +240,9 @@ tape('create', function(test) { var hierarchies_finder = function() { return [ - wofRecords['1'] + [ + wofRecords['1'] + ] ]; }; @@ -268,7 +283,9 @@ tape('create', function(test) { var hierarchies_finder = function() { return [ - wofRecords['1'] + [ + wofRecords['1'] + ] ]; }; @@ -335,10 +352,12 @@ tape('create', function(test) { var hierarchies_finder = function() { return [ - wofRecords['4'], - wofRecords['3'], - wofRecords['2'], - wofRecords['1'] + [ + wofRecords['4'], + wofRecords['3'], + wofRecords['2'], + wofRecords['1'] + ] ]; }; @@ -404,10 +423,12 @@ tape('create', function(test) { var hierarchies_finder = function() { return [ - wofRecords['4'], - wofRecords['3'], - wofRecords['2'], - wofRecords['1'] + [ + wofRecords['4'], + wofRecords['3'], + wofRecords['2'], + wofRecords['1'] + ] ]; }; @@ -474,10 +495,12 @@ tape('create', function(test) { var hierarchies_finder = function() { return [ - wofRecords['4'], - wofRecords['3'], - wofRecords['2'], - wofRecords['1'] + [ + wofRecords['4'], + wofRecords['3'], + wofRecords['2'], + wofRecords['1'] + ] ]; }; @@ -520,7 +543,9 @@ tape('create', function(test) { var hierarchies_finder = function() { return [ - wofRecords['1'] + [ + wofRecords['1'] + ] ]; }; @@ -564,7 +589,9 @@ tape('create', function(test) { var hierarchies_finder = function() { return [ - wofRecords['1'] + [ + wofRecords['1'] + ] ]; }; @@ -578,7 +605,7 @@ tape('create', function(test) { }); - test.test('undefined population should not set population in doc', function(t) { + test.test('undefined popularity should not set population in doc', function(t) { var wofRecords = { 1: { id: 1, @@ -607,7 +634,9 @@ tape('create', function(test) { var hierarchies_finder = function() { return [ - wofRecords['1'] + [ + wofRecords['1'] + ] ]; }; @@ -621,7 +650,7 @@ tape('create', function(test) { }); - test.test('defined population should set population in doc', function(t) { + test.test('defined popularity should set population in doc', function(t) { var wofRecords = { 1: { id: 1, @@ -651,7 +680,9 @@ tape('create', function(test) { var hierarchies_finder = function() { return [ - wofRecords['1'] + [ + wofRecords['1'] + ] ]; };