Skip to content

Commit

Permalink
Merge branch 'master' into add-multiple-hierarchy-support
Browse files Browse the repository at this point in the history
  • Loading branch information
@trescube
trescube committed Apr 22, 2016
2 parents 170b0b4 + e112f30 commit 9191ade
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 51 deletions.
20 changes: 0 additions & 20 deletions src/importStream.js
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,4 @@
var spy_stream = require('through2-spy');
var _ = require('lodash');

var peliasModel = require( 'pelias-model' );
var peliasLogger = require( 'pelias-logger' );

/*
This function performs the import based on the collection of WOF records
Expand All @@ -14,24 +10,8 @@ var peliasLogger = require( 'pelias-logger' );
*/
function fullImport(wof_record_stream, document_generator, destination_pipe, callback) {
var logger = peliasLogger.get( 'whosonfirst', {
transports: [
new peliasLogger.winston.transports.File( {
filename: 'missing_countries.log',
timestamp: false
})
]
});

var has_country_validation_stream = spy_stream.obj(function(wofDoc) {
if (_.isUndefined(wofDoc.getAdmin('admin0'))) {
logger.warn(wofDoc.getId());
}
});

wof_record_stream
.pipe(document_generator)
.pipe(has_country_validation_stream)
.pipe(peliasModel.createDocumentMapperStream())
.pipe(destination_pipe)
.on('finish', callback);
Expand Down
7 changes: 0 additions & 7 deletions src/peliasDocGenerators.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,27 +48,22 @@ module.exports.create = function(hierarchy_finder) {
hierarchy_finder(record).forEach(function(hierarchy_element) {
switch (hierarchy_element.place_type) {
case 'locality':
wofDoc.setAdmin( 'locality', hierarchy_element.name);
wofDoc.addParent('locality', hierarchy_element.name, hierarchy_element.id.toString());
break;
case 'borough':
wofDoc.addParent('borough', hierarchy_element.name, hierarchy_element.id.toString());
break;
case 'localadmin':
wofDoc.setAdmin( 'local_admin', hierarchy_element.name);
wofDoc.addParent('localadmin', hierarchy_element.name, hierarchy_element.id.toString());
break;
case 'county':
wofDoc.setAdmin( 'admin2', hierarchy_element.name);
wofDoc.addParent('county', hierarchy_element.name, hierarchy_element.id.toString());
break;
case 'macrocounty':
wofDoc.addParent('macrocounty', hierarchy_element.name, hierarchy_element.id.toString());
break;
case 'region':
wofDoc.setAdmin( 'admin1', hierarchy_element.name);
if (hierarchy_element.abbreviation) {
wofDoc.setAdmin( 'admin1_abbr', hierarchy_element.abbreviation );
wofDoc.addParent('region', hierarchy_element.name, hierarchy_element.id.toString(), hierarchy_element.abbreviation);
} else {
wofDoc.addParent('region', hierarchy_element.name, hierarchy_element.id.toString());
Expand All @@ -78,8 +73,6 @@ module.exports.create = function(hierarchy_finder) {
wofDoc.addParent('macroregion', hierarchy_element.name, hierarchy_element.id.toString());
break;
case 'country':
wofDoc.setAdmin( 'admin0', hierarchy_element.name);

// this is placetype=country, so lookup and set the iso3 from iso2
if (iso3166.is2(hierarchy_element.iso2)) {
var iso3 = iso3166.to3(hierarchy_element.iso2);
Expand Down
41 changes: 17 additions & 24 deletions test/peliasDocGeneratorsTest.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,14 +91,14 @@ tape('create', function(test) {
new Document( 'whosonfirst', 'locality', '8')
.setName('default', 'name 8')
.setCentroid({ lat: 19.191919, lon: 91.919191 })
.setAdmin( 'locality', 'name 8').addParent( 'locality', 'name 8', '8')
.addParent( 'locality', 'name 8', '8')
.addParent('borough', 'name 7', '7')
.setAdmin( 'local_admin', 'name 6').addParent( 'localadmin', 'name 6', '6')
.setAdmin( 'admin2', 'name 5').addParent( 'county', 'name 5', '5')
.addParent( 'localadmin', 'name 6', '6')
.addParent( 'county', 'name 5', '5')
.addParent( 'macrocounty', 'name 4', '4')
.setAdmin( 'admin1', 'name 3').addParent( 'region', 'name 3', '3')
.addParent( 'region', 'name 3', '3')
.addParent( 'macroregion', 'name 2', '2')
.setAdmin( 'admin0', 'name 1').addParent( 'country', 'name 1', '1', 'DEU')
.addParent( 'country', 'name 1', '1', 'DEU')
.setAlpha3( 'DEU' )
.setBoundingBox({ upperLeft: { lat:60.847893, lon:-13.691314 }, lowerRight: { lat:49.909613 , lon:1.771169 }})
];
Expand Down Expand Up @@ -222,7 +222,6 @@ tape('create', function(test) {
new Document( 'whosonfirst', 'country', '1')
.setName('default', 'name 1')
.setCentroid({ lat: 12.121212, lon: 21.212121 })
.setAdmin( 'admin0', 'name 1')
.addParent('country', 'name 1', '1')
];

Expand Down Expand Up @@ -264,7 +263,6 @@ tape('create', function(test) {
new Document( 'whosonfirst', 'country', '1')
.setName('default', 'name 1')
.setCentroid({ lat: 12.121212, lon: 21.212121 })
.setAdmin( 'admin0', 'name 1')
.addParent('country', 'name 1', '1')
];

Expand Down Expand Up @@ -328,14 +326,9 @@ tape('create', function(test) {
new Document( 'whosonfirst', 'locality', '4')
.setName('default', 'New York City')
.setCentroid({ lat: 15.151515, lon: 51.515151 })
.setAdmin( 'locality', 'New York City')
.addParent('locality', 'New York City', '4')
.setAdmin( 'admin2', 'Kings')
.addParent('county', 'Kings', '3')
.setAdmin( 'admin1', 'New York')
.setAdmin( 'admin1_abbr', 'NY')
.addParent('region', 'New York', '2', 'NY')
.setAdmin( 'admin0', 'United States')
.addParent('country', 'United States', '1', 'USA')
.setAlpha3( 'USA' )
];
Expand Down Expand Up @@ -402,10 +395,10 @@ tape('create', function(test) {
new Document( 'whosonfirst', 'locality', '4')
.setName('default', 'New York City')
.setCentroid({ lat: 15.151515, lon: 51.515151 })
.setAdmin( 'locality', 'New York City').addParent('locality', 'New York City', '4')
.setAdmin( 'admin2', 'Kings').addParent('county', 'Kings', '3')
.setAdmin( 'admin1', 'New York').addParent('region', 'New York', '2')
.setAdmin( 'admin0', 'United States').addParent('country', 'United States', '1', 'USA')
.addParent('locality', 'New York City', '4')
.addParent('county', 'Kings', '3')
.addParent('region', 'New York', '2')
.addParent('country', 'United States', '1', 'USA')
.setAlpha3( 'USA' )
];

Expand Down Expand Up @@ -472,10 +465,10 @@ tape('create', function(test) {
new Document( 'whosonfirst', 'locality', '4')
.setName('default', 'New York City')
.setCentroid({ lat: 15.151515, lon: 51.515151 })
.setAdmin( 'locality', 'New York City').addParent('locality', 'New York City', '4')
.setAdmin( 'admin2', 'Kings').addParent('county', 'Kings', '3')
.setAdmin( 'admin1', 'New York').addParent('region', 'New York', '2')
.setAdmin( 'admin0', 'United States').addParent('country', 'United States', '1', 'USA')
.addParent('locality', 'New York City', '4')
.addParent('county', 'Kings', '3')
.addParent('region', 'New York', '2')
.addParent('country', 'United States', '1', 'USA')
.setAlpha3( 'USA' )
];

Expand Down Expand Up @@ -521,7 +514,7 @@ tape('create', function(test) {
new Document( 'whosonfirst', 'country', '1')
.setName('default', 'United States')
.setCentroid({ lat: 12.121212, lon: 21.212121 })
.setAdmin( 'admin0', 'United States').addParent('country', 'United States', '1', 'USA')
.addParent('country', 'United States', '1', 'USA')
.setAlpha3( 'USA' )
];

Expand Down Expand Up @@ -564,7 +557,7 @@ tape('create', function(test) {
new Document( 'whosonfirst', 'country', '1')
.setName('default', 'United States')
.setCentroid({ lat: 12.121212, lon: 21.212121 })
.setAdmin( 'admin0', 'United States').addParent('country', 'United States', '1', 'USA')
.addParent('country', 'United States', '1', 'USA')
.setAlpha3( 'USA' )
.setPopulation(98765)
];
Expand Down Expand Up @@ -608,7 +601,7 @@ tape('create', function(test) {
new Document( 'whosonfirst', 'country', '1')
.setName('default', 'United States')
.setCentroid({ lat: 12.121212, lon: 21.212121 })
.setAdmin( 'admin0', 'United States').addParent('country', 'United States', '1', 'USA')
.addParent('country', 'United States', '1', 'USA')
.setAlpha3( 'USA' )
];

Expand Down Expand Up @@ -651,7 +644,7 @@ tape('create', function(test) {
new Document( 'whosonfirst', 'country', '1')
.setName('default', 'United States')
.setCentroid({ lat: 12.121212, lon: 21.212121 })
.setAdmin( 'admin0', 'United States').addParent('country', 'United States', '1', 'USA')
.addParent('country', 'United States', '1', 'USA')
.setAlpha3( 'USA' )
.setPopularity(87654)
];
Expand Down

0 comments on commit 9191ade

Please sign in to comment.