From 9b7330b4eb612680e8ddf349de397d1b16a1d2d3 Mon Sep 17 00:00:00 2001 From: Bhargav Kowshik Date: Tue, 14 Mar 2017 15:01:15 +0530 Subject: [PATCH 1/4] Flag tag combinations with count less than 1 --- comparators/invalid-tag-combination.js | 36 ++++++++++++++++ comparators/tag-combinations.csv | 27 ++++++++++++ tests/fixtures/invalid-tag-combination.json | 46 +++++++++++++++++++++ 3 files changed, 109 insertions(+) create mode 100644 comparators/invalid-tag-combination.js create mode 100644 comparators/tag-combinations.csv create mode 100644 tests/fixtures/invalid-tag-combination.json diff --git a/comparators/invalid-tag-combination.js b/comparators/invalid-tag-combination.js new file mode 100644 index 0000000..2fe88d4 --- /dev/null +++ b/comparators/invalid-tag-combination.js @@ -0,0 +1,36 @@ +'use strict'; + +var csv = require('csv'); +var fs = require('fs'); +var path = require('path'); + +module.exports = invalidTagCombination; +function invalidTagCombination(newVersion, oldVersion, callback) { + // What should be the minimum value of count to be a valid tag combination. + var MIN_COUNT = 1; + + var result = {}; + if (!newVersion) return callback(null, result); + + csv.parse(fs.readFileSync(path.join(__dirname, 'tag-combinations.csv')), function (error, rows) { + var tags = Object.keys(newVersion.properties); + + for (var i = 0; i < tags.length; i++) { + for (var j = 0; j < tags.length; j++) { + if (i === j) continue; + + var tag = tags[i]; + var anotherTag = tags[j]; + + var rowIndex = rows[0].indexOf(tag); + var columnIndex = rows[0].indexOf(anotherTag); + if (rowIndex === -1 || columnIndex === -1) continue; + + var count = parseFloat(rows[rowIndex][columnIndex]); + + if (isNaN(count) || (count < MIN_COUNT)) return callback(null, {'result:invalid_tag_combination': true}); + } + } + return callback(null, result); + }); +} diff --git a/comparators/tag-combinations.csv b/comparators/tag-combinations.csv new file mode 100644 index 0000000..1f2b086 --- /dev/null +++ b/comparators/tag-combinations.csv @@ -0,0 +1,27 @@ +,aerialway,aeroway,amenity,barrier,boundary,building,craft,emergency,geological,highway,historic,landuse,leisure,man_made,military,natural,office,places,power,public_transport,railway,route,shop,sport,tourism,waterway +aerialway,,,0.05,0.01,,2.49,,,,0.33,0.02,,0.07,0.09,,0.01,,,0.12,0.74,0.19,0.04,0.02,0.2,0.1, +aeroway,,,0.03,0.14,0.01,6.69,,0.03,,0.21,0.01,0.27,0.02,0.29,0.24,0.02,,,,,,,,0.06,0.01, +amenity,,,,0.41,,13.51,0.01,0.29,,0.13,0.16,0.26,0.12,0.28,,0.07,0.08,,,0.07,,,0.47,0.23,0.36,0.01 +barrier,,0.01,0.61,,0.05,0.14,,0.03,,0.06,0.26,2.27,1.49,0.14,0.04,0.1,0.01,,0.39,,0.02,,0.02,0.11,0.07,0.06 +boundary,,,0.01,0.24,,0.01,,,,1.52,0.33,1,2.28,0.04,0.01,1.46,,,0.01,,0.03,0.01,,,0.02,2.67 +building,,0.01,0.66,,,,0.01,,,,0.06,0.04,0.04,0.12,0.01,,0.04,,0.06,,0.01,,0.25,0.02,0.08, +craft,,,1.85,0.25,,27.75,,,,0.04,0.08,1.7,0.03,1.43,,0.01,0.7,,0.03,,,,9.56,0.01,0.83,0.01 +emergency,,0.01,3.79,0.28,0.01,1.11,,,,5.32,0.07,0.04,0.03,0.15,,0.11,0.01,,,0.01,0.02,0.01,0.01,0.01,0.3, +geological,,,0.02,0.05,0.69,0.3,,,,0.14,2.1,0.37,0.51,0.2,,60.05,,,,,,0.02,,0.03,4.16,0.12 +highway,,,0.01,,0.02,,,0.04,,,0.01,0.01,0.01,0.03,,,,,0.01,0.63,0.07,0.04,,0.01,0.01, +historic,,0.01,2.43,2.65,0.73,17.33,0.01,0.09,0.02,0.99,,0.62,0.2,1.81,2.02,0.46,0.03,,0.01,0.01,0.68,0.1,0.05,0.06,4.34,0.13 +landuse,,0.01,0.15,0.89,0.09,0.5,0.01,,,0.05,0.02,,0.41,0.14,0.02,0.54,0.01,,0.04,,,,0.05,0.17,0.03,0.01 +leisure,,,0.35,2.9,0.96,2.15,,0.01,,0.16,0.04,2.01,,0.03,,0.39,,,,,,,0.02,28.4,0.25,0.01 +man_made,,0.05,1.44,0.48,0.03,12.41,0.05,0.06,,1.57,0.61,1.19,0.05,,0.01,0.77,0.02,,0.25,0.01,0.06,0.01,0.15,0.02,0.35,0.43 +military,,1.61,0.63,5,0.18,28.04,,0.02,,0.88,27.28,6.28,0.25,0.54,,0.19,0.44,,0.01,,,,0.01,0.33,1.07,0.03 +natural,,,0.03,0.03,0.09,0.02,,,0.01,0.01,0.01,0.39,0.06,0.06,,,,,0.01,,,,,0.05,0.04,0.07 +office,,,3.13,0.2,0.01,31.85,0.16,0.03,,0.02,0.07,0.62,0.05,0.14,0.08,0.01,,,0.01,0.01,,,1.17,0.04,0.12, +places,,,,,,,,,,,,,,,,,,,,,,,,,, +power,,,,0.21,,1,,,,0.1,,0.05,,0.04,,0.01,,,,,,,,,, +public_transport,0.05,,0.54,0.01,,0.49,,,,51.51,0.01,0.01,,0.02,,,,,,,8.81,0.04,0.01,,0.01, +railway,,,0.01,0.03,0.01,0.38,,,,2.17,0.13,0.02,,0.04,,,,,,3.33,,0.08,,,0.02, +route,0.01,,0.08,,0.02,0.02,,0.01,,8.09,0.13,0.08,0.03,0.03,,,,,0.01,0.1,0.55,,,0.07,0.08,0.1 +shop,,,1.94,0.07,,21.33,0.25,,,0.02,0.01,0.37,0.02,0.12,,0.01,0.13,,,0.01,,,,0.05,0.11, +sport,0.01,0.02,2.18,0.71,,4.11,,,,0.83,0.04,2.75,90.66,0.04,0.01,1.09,0.01,,,,0.01,0.03,0.11,,0.15,0.01 +tourism,0.01,,2.77,0.38,0.02,13.04,0.04,0.17,0.02,0.4,2.17,0.43,0.67,0.52,0.04,0.71,0.02,,0.01,0.01,0.05,0.03,0.2,0.12,,0.11 +waterway,,,0.01,0.03,0.31,0.05,,,,0.03,0.01,0.01,,0.07,,0.14,,,,,,,,,0.01, diff --git a/tests/fixtures/invalid-tag-combination.json b/tests/fixtures/invalid-tag-combination.json new file mode 100644 index 0000000..7d76410 --- /dev/null +++ b/tests/fixtures/invalid-tag-combination.json @@ -0,0 +1,46 @@ +{ + "compareFunction": "invalid-tag-combination", + "fixtures": [ + { + "description": "Test feature with invalid tag combination with count less than 1", + "expectedResult": { + "result:invalid_tag_combination": true + }, + "newVersion": { + "type": "Feature", + "properties": { + "tourism": "yes", + "amenity": "yes" + }, + "geometry": null + }, + "oldVersion": null + }, + { + "description": "Test feature with invalid tag combination with count greater than 1", + "expectedResult": {}, + "newVersion": { + "type": "Feature", + "properties": { + "sport": "yes", + "leisure": "yes" + }, + "geometry": null + }, + "oldVersion": null + }, + { + "description": "Test feature with no primary tags", + "expectedResult": {}, + "newVersion": { + "type": "Feature", + "properties": { + "name": "Hello, world!", + "version": "1" + }, + "geometry": null + }, + "oldVersion": null + } + ] +} From f18738ed6e69d507efe984f1131aa1b2b71ff3d9 Mon Sep 17 00:00:00 2001 From: Bhargav Kowshik Date: Tue, 14 Mar 2017 15:04:10 +0530 Subject: [PATCH 2/4] Add invalid_tag_combination comparator to index.js --- index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index fe27924..d7c8509 100755 --- a/index.js +++ b/index.js @@ -20,5 +20,6 @@ module.exports = { 'added_place': require('./comparators/added_place'), 'modified_place_wikidata': require('./comparators/modified-place-wikidata'), 'osm_landmarks': require('./comparators/osm-landmarks'), - 'modifiedMonument': require('./comparators/modifiedMonument') + 'modifiedMonument': require('./comparators/modifiedMonument'), + 'invalid_tag_combination': require('./comparators/invalid-tag-combination') }; From 71121c31a3dea288fb207c42b0551f0817640714 Mon Sep 17 00:00:00 2001 From: Bhargav Kowshik Date: Tue, 14 Mar 2017 15:11:29 +0530 Subject: [PATCH 3/4] Add description about comparator on comparator README --- comparators/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/comparators/README.md b/comparators/README.md index 67fadf4..5e0300b 100644 --- a/comparators/README.md +++ b/comparators/README.md @@ -78,3 +78,6 @@ Reports when `wikidata` tag is added, deleted or modified for a place type featu ### osm-landmarks Reports when any feature in the [osm-landmarks list](https://github.com/osmlab/osm-landmarks) is edited. + +### invalid-tag-combination +Reports when a feature has two uncommon primary tag combinations. Ex: `0.06 %` of features with `building` also have `historic` tag. From 1f8980fb8fdca7452fcbe1276ba8f8ccadff0827 Mon Sep 17 00:00:00 2001 From: Bhargav Kowshik Date: Tue, 14 Mar 2017 15:16:21 +0530 Subject: [PATCH 4/4] Add csv module to package.json --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 8f0375b..9152ac4 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "author": "Mapbox", "license": "ISC", "dependencies": { + "csv": "^1.1.1", "d3-queue": "^3.0.3", "geopoint": "^1.0.1", "levenshtein": "^1.0.5",