Skip to content

Commit

Permalink
modified peliasDocGenerators to accommodate multiple hierarchies
Browse files Browse the repository at this point in the history
  • Loading branch information
@trescube
trescube committed Apr 22, 2016
1 parent db67c44 commit 522a8ce
Show file tree
Hide file tree
Showing 2 changed files with 168 additions and 108 deletions.
185 changes: 107 additions & 78 deletions src/peliasDocGenerators.js
View file
Original file line number Diff line number Diff line change
@@ -1,95 +1,124 @@
var map_stream = require('through2-map');
var through2 = require('through2');
var _ = require('lodash');
var iso3166 = require('iso3166-1');

var Document = require('pelias-model').Document;

module.exports = {};

module.exports.create = function(hierarchy_finder) {
return map_stream.obj(function(record) {
var wofDoc = new Document( 'whosonfirst', record.place_type, record.id );

if (record.name) {
wofDoc.setName('default', record.name);
}
wofDoc.setCentroid({ lat: record.lat, lon: record.lon });
function assignField(hierarchyElement, wofDoc) {
switch (hierarchyElement.place_type) {
case 'locality':
wofDoc.addParent('locality', hierarchyElement.name, hierarchyElement.id.toString());
break;
case 'borough':
wofDoc.addParent('borough', hierarchyElement.name, hierarchyElement.id.toString());
break;
case 'localadmin':
wofDoc.addParent('localadmin', hierarchyElement.name, hierarchyElement.id.toString());
break;
case 'county':
wofDoc.addParent('county', hierarchyElement.name, hierarchyElement.id.toString());
break;
case 'macrocounty':
wofDoc.addParent('macrocounty', hierarchyElement.name, hierarchyElement.id.toString());
break;
case 'region':
// if the region has an abbreviation, set it
if (hierarchyElement.abbreviation) {
wofDoc.addParent('region', hierarchyElement.name, hierarchyElement.id.toString(), hierarchyElement.abbreviation);
} else {
wofDoc.addParent('region', hierarchyElement.name, hierarchyElement.id.toString());
}
break;
case 'macroregion':
wofDoc.addParent('macroregion', hierarchyElement.name, hierarchyElement.id.toString());
break;
case 'country':
// this is placetype=country, so lookup and set the iso3 from iso2
if (iso3166.is2(hierarchyElement.iso2)) {
var iso3 = iso3166.to3(hierarchyElement.iso2);

// only set population if available
if (record.population) {
wofDoc.setPopulation(record.population);
}
wofDoc.setAlpha3(iso3);
wofDoc.addParent('country', hierarchyElement.name, hierarchyElement.id.toString(), iso3);

// only set popularity if available
if (record.popularity) {
wofDoc.setPopularity(record.popularity);
}
} else {
wofDoc.addParent('country', hierarchyElement.name, hierarchyElement.id.toString());

// WOF bbox is defined as:
// lowerLeft.lon, lowerLeft.lat, upperRight.lon, upperRight.lat
// so convert to what ES understands
if (!_.isUndefined(record.bounding_box)) {
var parsedBoundingBox = record.bounding_box.split(',').map(parseFloat);
var marshaledBoundingBoxBox = {
upperLeft: {
lat: parsedBoundingBox[3],
lon: parsedBoundingBox[0]
},
lowerRight: {
lat: parsedBoundingBox[1],
lon: parsedBoundingBox[2]
}

};
wofDoc.setBoundingBox(marshaledBoundingBoxBox);
}
}

// iterate the hierarchy, assigning fields
hierarchy_finder(record).forEach(function(hierarchy_element) {
switch (hierarchy_element.place_type) {
case 'locality':
wofDoc.addParent('locality', hierarchy_element.name, hierarchy_element.id.toString());
break;
case 'borough':
wofDoc.addParent('borough', hierarchy_element.name, hierarchy_element.id.toString());
break;
case 'localadmin':
wofDoc.addParent('localadmin', hierarchy_element.name, hierarchy_element.id.toString());
break;
case 'county':
wofDoc.addParent('county', hierarchy_element.name, hierarchy_element.id.toString());
break;
case 'macrocounty':
wofDoc.addParent('macrocounty', hierarchy_element.name, hierarchy_element.id.toString());
break;
case 'region':
if (hierarchy_element.abbreviation) {
wofDoc.addParent('region', hierarchy_element.name, hierarchy_element.id.toString(), hierarchy_element.abbreviation);
} else {
wofDoc.addParent('region', hierarchy_element.name, hierarchy_element.id.toString());
}
break;
case 'macroregion':
wofDoc.addParent('macroregion', hierarchy_element.name, hierarchy_element.id.toString());
break;
case 'country':
// this is placetype=country, so lookup and set the iso3 from iso2
if (iso3166.is2(hierarchy_element.iso2)) {
var iso3 = iso3166.to3(hierarchy_element.iso2);

wofDoc.setAlpha3(iso3);
wofDoc.addParent('country', hierarchy_element.name, hierarchy_element.id.toString(), iso3);

} else {
wofDoc.addParent('country', hierarchy_element.name, hierarchy_element.id.toString());

}

break;
break;
}

}

// method that extracts the logic for Document creation. `hierarchy` is optional
function setupDocument(record, hierarchy) {
var wofDoc = new Document( 'whosonfirst', record.place_type, record.id );

if (record.name) {
wofDoc.setName('default', record.name);
}
wofDoc.setCentroid({ lat: record.lat, lon: record.lon });

// only set population if available
if (record.population) {
wofDoc.setPopulation(record.population);
}

// only set popularity if available
if (record.popularity) {
wofDoc.setPopularity(record.popularity);
}

// WOF bbox is defined as:
// lowerLeft.lon, lowerLeft.lat, upperRight.lon, upperRight.lat
// so convert to what ES understands
if (!_.isUndefined(record.bounding_box)) {
var parsedBoundingBox = record.bounding_box.split(',').map(parseFloat);
var marshaledBoundingBoxBox = {
upperLeft: {
lat: parsedBoundingBox[3],
lon: parsedBoundingBox[0]
},
lowerRight: {
lat: parsedBoundingBox[1],
lon: parsedBoundingBox[2]
}

};
wofDoc.setBoundingBox(marshaledBoundingBoxBox);
}

// a `hierarchy` is composed of potentially multiple WOF records, so iterate
// and assign fields
if (!_.isUndefined(hierarchy)) {
hierarchy.forEach(function(hierarchyElement) {
assignField(hierarchyElement, wofDoc);
});

return wofDoc;
}

return wofDoc;

}

module.exports.create = function(hierarchy_finder) {
return through2.obj(function(record, enc, next) {
// if there are no hierarchies, then just return the doc as-is
var hierarchies = hierarchy_finder(record);

if (hierarchies && hierarchies.length > 0) {
hierarchies.forEach(function(hierarchy) {
this.push(setupDocument(record, hierarchy));
}, this);

} else {
this.push(setupDocument(record));

}

next();

});

Expand Down
91 changes: 61 additions & 30 deletions test/peliasDocGeneratorsTest.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,27 +92,40 @@ tape('create', function(test) {
.setName('default', 'name 8')
.setCentroid({ lat: 19.191919, lon: 91.919191 })
.addParent( 'locality', 'name 8', '8')
.addParent('borough', 'name 7', '7')
.addParent( 'borough', 'name 7', '7')
.addParent( 'localadmin', 'name 6', '6')
.addParent( 'county', 'name 5', '5')
.addParent( 'macrocounty', 'name 4', '4')
.addParent( 'region', 'name 3', '3')
.addParent( 'macroregion', 'name 2', '2')
.addParent( 'country', 'name 1', '1', 'DEU')
.setAlpha3( 'DEU' )
.setBoundingBox({ upperLeft: { lat:60.847893, lon:-13.691314 }, lowerRight: { lat:49.909613 , lon:1.771169 }})
.setBoundingBox({ upperLeft: { lat:60.847893, lon:-13.691314 }, lowerRight: { lat:49.909613 , lon:1.771169 }}),
new Document( 'whosonfirst', 'locality', '8')
.setName('default', 'name 8')
.setCentroid({ lat: 19.191919, lon: 91.919191 })
.addParent( 'locality', 'name 8', '8')
.addParent( 'country', 'name 1', '1', 'DEU')
.setAlpha3( 'DEU' )
.setBoundingBox({ upperLeft: { lat:60.847893, lon:-13.691314 }, lowerRight: { lat:49.909613 , lon:1.771169 }}),
];

var hierarchies_finder = function() {
return [
wofRecords['8'],
wofRecords['7'],
wofRecords['6'],
wofRecords['5'],
wofRecords['4'],
wofRecords['3'],
wofRecords['2'],
wofRecords['1']
[
wofRecords['8'],
wofRecords['7'],
wofRecords['6'],
wofRecords['5'],
wofRecords['4'],
wofRecords['3'],
wofRecords['2'],
wofRecords['1']
],
[
wofRecords['8'],
wofRecords['1']
]
];
};

Expand Down Expand Up @@ -227,7 +240,9 @@ tape('create', function(test) {

var hierarchies_finder = function() {
return [
wofRecords['1']
[
wofRecords['1']
]
];
};

Expand Down Expand Up @@ -268,7 +283,9 @@ tape('create', function(test) {

var hierarchies_finder = function() {
return [
wofRecords['1']
[
wofRecords['1']
]
];
};

Expand Down Expand Up @@ -335,10 +352,12 @@ tape('create', function(test) {

var hierarchies_finder = function() {
return [
wofRecords['4'],
wofRecords['3'],
wofRecords['2'],
wofRecords['1']
[
wofRecords['4'],
wofRecords['3'],
wofRecords['2'],
wofRecords['1']
]
];
};

Expand Down Expand Up @@ -404,10 +423,12 @@ tape('create', function(test) {

var hierarchies_finder = function() {
return [
wofRecords['4'],
wofRecords['3'],
wofRecords['2'],
wofRecords['1']
[
wofRecords['4'],
wofRecords['3'],
wofRecords['2'],
wofRecords['1']
]
];
};

Expand Down Expand Up @@ -474,10 +495,12 @@ tape('create', function(test) {

var hierarchies_finder = function() {
return [
wofRecords['4'],
wofRecords['3'],
wofRecords['2'],
wofRecords['1']
[
wofRecords['4'],
wofRecords['3'],
wofRecords['2'],
wofRecords['1']
]
];
};

Expand Down Expand Up @@ -520,7 +543,9 @@ tape('create', function(test) {

var hierarchies_finder = function() {
return [
wofRecords['1']
[
wofRecords['1']
]
];
};

Expand Down Expand Up @@ -564,7 +589,9 @@ tape('create', function(test) {

var hierarchies_finder = function() {
return [
wofRecords['1']
[
wofRecords['1']
]
];
};

Expand All @@ -578,7 +605,7 @@ tape('create', function(test) {

});

test.test('undefined population should not set population in doc', function(t) {
test.test('undefined popularity should not set population in doc', function(t) {
var wofRecords = {
1: {
id: 1,
Expand Down Expand Up @@ -607,7 +634,9 @@ tape('create', function(test) {

var hierarchies_finder = function() {
return [
wofRecords['1']
[
wofRecords['1']
]
];
};

Expand All @@ -621,7 +650,7 @@ tape('create', function(test) {

});

test.test('defined population should set population in doc', function(t) {
test.test('defined popularity should set population in doc', function(t) {
var wofRecords = {
1: {
id: 1,
Expand Down Expand Up @@ -651,7 +680,9 @@ tape('create', function(test) {

var hierarchies_finder = function() {
return [
wofRecords['1']
[
wofRecords['1']
]
];
};

Expand Down

0 comments on commit 522a8ce

Please sign in to comment.