extract all hierarchies from WOF records for later processing

pelias · Apr 22, 2016 · 170b0b4 · 170b0b4
1 parent e864d42
commit 170b0b4
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 44 deletions.
diff --git a/src/components/extractFields.js b/src/components/extractFields.js
@@ -27,7 +27,7 @@ function getPopulation(properties) {
 */
 module.exports.create = function map_fields_stream() {
   return through2.obj(function(json_object, enc, callback) {
-    var base_record = {
+    var record = {
       id: json_object.id,
       name: json_object.properties['wof:name'],
       abbreviation: json_object.properties['wof:abbreviation'],
@@ -38,29 +38,16 @@ module.exports.create = function map_fields_stream() {
       bounding_box: json_object.properties['geom:bbox'],
       iso2: json_object.properties['iso:country'],
       population: getPopulation(json_object.properties),
-      popularity: json_object.properties['misc:photo_sum']
+      popularity: json_object.properties['misc:photo_sum'],
+      hierarchies: _.get(json_object, 'properties.wof:hierarchy', [])
     };
 
     // use the QS altname if US county and available
-    if (isUsCounty(base_record, json_object.properties['qs:a2_alt'])) {
-      base_record.name = json_object.properties['qs:a2_alt'];
+    if (isUsCounty(record, json_object.properties['qs:a2_alt'])) {
+      record.name = json_object.properties['qs:a2_alt'];
     }
 
-    // if there's no hierarchy then just add the base record
-    if (_.isUndefined(json_object.properties['wof:hierarchy'])) {
-      this.push(base_record);
-
-    } else {
-      // otherwise, clone the base record for each hierarchy in the list and push
-      json_object.properties['wof:hierarchy'].forEach(function(hierarchy) {
-        var clone = _.clone(base_record, true);
-        clone.hierarchy = hierarchy;
-        this.push(clone);
-      }, this);
-
-    }
-
-    return callback();
+    return callback(null, record);
 
   });
 

diff --git a/test/components/extractFieldsTest.js b/test/components/extractFieldsTest.js
@@ -69,25 +69,14 @@ tape('readStreamComponents', function(test) {
         popularity: 87654,
         abbreviation: 'XY',
         bounding_box: '-13.691314,49.909613,1.771169,60.847886',
-        hierarchy: {
-          'parent_id': 12345
-        }
-      },
-      {
-        id: 12345,
-        name: 'name 1',
-        place_type: 'place type 1',
-        parent_id: 'parent id 1',
-        lat: 12.121212,
-        lon: 21.212121,
-        iso2: 'YZ',
-        population: 98765,
-        popularity: 87654,
-        abbreviation: 'XY',
-        bounding_box: '-13.691314,49.909613,1.771169,60.847886',
-        hierarchy: {
-          'parent_id': 23456
-        }
+        hierarchies: [
+          {
+            'parent_id': 12345
+          },
+          {
+            'parent_id': 23456
+          }
+        ]
       },
       {
         id: 23456,
@@ -100,7 +89,8 @@ tape('readStreamComponents', function(test) {
         population: undefined,
         popularity: undefined,
         abbreviation: undefined,
-        bounding_box: undefined
+        bounding_box: undefined,
+        hierarchies: []
       }
     ];
 
@@ -143,6 +133,7 @@ tape('readStreamComponents', function(test) {
         popularity: undefined,
         abbreviation: 'XY',
         bounding_box: '-13.691314,49.909613,1.771169,60.847886',
+        hierarchies: []
       }
     ];
 
@@ -184,6 +175,7 @@ tape('readStreamComponents', function(test) {
         popularity: undefined,
         abbreviation: 'XY',
         bounding_box: '-13.691314,49.909613,1.771169,60.847886',
+        hierarchies: []
       }
     ];
 
@@ -225,6 +217,7 @@ tape('readStreamComponents', function(test) {
         popularity: undefined,
         abbreviation: 'XY',
         bounding_box: '-13.691314,49.909613,1.771169,60.847886',
+        hierarchies: []
       }
     ];
 
@@ -265,6 +258,7 @@ tape('readStreamComponents', function(test) {
         popularity: undefined,
         abbreviation: 'XY',
         bounding_box: '-13.691314,49.909613,1.771169,60.847886',
+        hierarchies: []
       }
     ];
 
@@ -305,6 +299,7 @@ tape('readStreamComponents', function(test) {
         popularity: undefined,
         abbreviation: 'XY',
         bounding_box: '-13.691314,49.909613,1.771169,60.847886',
+        hierarchies: []
       }
     ];
 
@@ -343,7 +338,8 @@ tape('readStreamComponents', function(test) {
         population: undefined,
         popularity: undefined,
         bounding_box: undefined,
-        abbreviation: undefined
+        abbreviation: undefined,
+        hierarchies: []
       }
     ];
 
@@ -381,7 +377,8 @@ tape('readStreamComponents', function(test) {
         population: undefined,
         popularity: undefined,
         bounding_box: undefined,
-        abbreviation: undefined
+        abbreviation: undefined,
+        hierarchies: []
       }
     ];
 
@@ -420,7 +417,8 @@ tape('readStreamComponents', function(test) {
         population: undefined,
         popularity: undefined,
         bounding_box: undefined,
-        abbreviation: undefined
+        abbreviation: undefined,
+        hierarchies: []
       }
     ];
 

diff --git a/test/readStreamTest.js b/test/readStreamTest.js
@@ -93,7 +93,8 @@ tape('readStream', function(test) {
         abbreviation: 'XY',
         bounding_box: '-13.691314,49.909613,1.771169,60.847886',
         population: 98765,
-        popularity: 87654
+        popularity: 87654,
+        hierarchies: []
       }, 'id 1234567 should have been loaded');
 
       t.deepEqual(wofRecords[12345678], {
@@ -107,7 +108,8 @@ tape('readStream', function(test) {
         abbreviation: 'XY',
         bounding_box: '-24.539906,34.815009,69.033946,81.85871',
         population: undefined,
-        popularity: undefined
+        popularity: undefined,
+        hierarchies: []
       }, 'id 12345678 should have been loaded');
 
       t.end();