From 2e71de7eac1b70ddc217300eff95ac0f30d21a98 Mon Sep 17 00:00:00 2001 From: Christopher Blanchard Date: Sun, 23 Aug 2020 15:32:50 +0000 Subject: [PATCH 1/2] Add test to capture issue #577 SPD have appended extra characters to a small number of postcodes - causing those postcodes to be unqueryable --- test/scottish_postcode.unit.js | 35 ++++++++++++++++++++-------------- test/seed/SmallUser.csv | 2 ++ 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/test/scottish_postcode.unit.js b/test/scottish_postcode.unit.js index 3ca0b777..04e4b7ac 100644 --- a/test/scottish_postcode.unit.js +++ b/test/scottish_postcode.unit.js @@ -19,7 +19,7 @@ describe("Scottish Postcode Model", () => { const testPostcodeLarge = "ML11 0GH"; const testPostcodeSmall = "G82 1JW"; - before(function(done) { + before(function (done) { this.timeout(0); series([clearPostcodeDb, seedPostcodeDb], done); }); @@ -27,24 +27,24 @@ describe("Scottish Postcode Model", () => { after(clearPostcodeDb); describe("#setupTable", () => { - before(function(done) { + before(function (done) { this.timeout(0); - ScottishPostcode._destroyRelation(error => { + ScottishPostcode._destroyRelation((error) => { if (error) return done(error); ScottishPostcode._setupTable(seedFilePath, done); }); }); - after(function(done) { + after(function (done) { this.timeout(0); - ScottishPostcode._destroyRelation(error => { + ScottishPostcode._destroyRelation((error) => { if (error) return done(error); ScottishPostcode._setupTable(seedFilePath, done); }); }); describe("#_createRelation", () => { - it(`creates a relation that matches ${ScottishPostcode.relation} schema`, done => { + it(`creates a relation that matches ${ScottishPostcode.relation} schema`, (done) => { const query = ` SELECT column_name, data_type, character_maximum_length, collation_name @@ -54,7 +54,7 @@ describe("Scottish Postcode Model", () => { ScottishPostcode._query(query, (error, result) => { if (error) return done(error); const impliedSchema = {}; - result.rows.forEach(columnInfo => { + result.rows.forEach((columnInfo) => { let columnName, dataType; [columnName, dataType] = inferSchemaData(columnInfo); impliedSchema[columnName] = dataType; @@ -66,7 +66,7 @@ describe("Scottish Postcode Model", () => { }); describe("#seedData", () => { - it("loads correct data from data directory", done => { + it("loads correct data from data directory", (done) => { const query = `SELECT count(*) FROM ${ScottishPostcode.relation}`; ScottishPostcode._query(query, (error, result) => { if (error) return done(error); @@ -74,6 +74,13 @@ describe("Scottish Postcode Model", () => { done(); }); }); + it("loads postcodes suffixed with additional character", (done) => { + ScottishPostcode.find("PA31 8UA", (error, result) => { + if (error) return done(error); + assert.isNotNull(result); + done(); + }); + }); }); }); @@ -100,7 +107,7 @@ describe("Scottish Postcode Model", () => { }); describe("#find", () => { - it("should return postcode with the right attributes for large user", done => { + it("should return postcode with the right attributes for large user", (done) => { ScottishPostcode.find(testPostcodeLarge, (error, result) => { if (error) return done(error); assert.deepEqual(result, { @@ -114,7 +121,7 @@ describe("Scottish Postcode Model", () => { }); }); - it("should return postcode with the right attributes for small users", done => { + it("should return postcode with the right attributes for small users", (done) => { ScottishPostcode.find(testPostcodeSmall, (error, result) => { if (error) return done(error); assert.deepEqual(result, { @@ -128,7 +135,7 @@ describe("Scottish Postcode Model", () => { }); }); - it("should return null for null/undefined postcode search", done => { + it("should return null for null/undefined postcode search", (done) => { ScottishPostcode.find(null, (error, result) => { if (error) return done(error); assert.isNull(result); @@ -136,7 +143,7 @@ describe("Scottish Postcode Model", () => { }); }); - it("returns null if invalid postcode", done => { + it("returns null if invalid postcode", (done) => { ScottishPostcode.find("1", (error, result) => { if (error) return done(error); assert.isNull(result); @@ -144,7 +151,7 @@ describe("Scottish Postcode Model", () => { }); }); - it("should be insensitive to space", done => { + it("should be insensitive to space", (done) => { ScottishPostcode.find( testPostcodeLarge.replace(/\s/, ""), (error, result) => { @@ -155,7 +162,7 @@ describe("Scottish Postcode Model", () => { ); }); - it("should return null if postcode does not exist", done => { + it("should return null if postcode does not exist", (done) => { ScottishPostcode.find("ID11QD", (error, result) => { if (error) return done(error); assert.isNull(result); diff --git a/test/seed/SmallUser.csv b/test/seed/SmallUser.csv index 865370d3..57f1ab1c 100644 --- a/test/seed/SmallUser.csv +++ b/test/seed/SmallUser.csv @@ -98,3 +98,5 @@ "PA20 9BA","PA20","PA20 9",1/8/1973 00:00:00,,"208889","664416",55.83501986,-5.05330374,"N","S12000035","S14000005","S17000011","S16000083","S13002523","S08000022","S08000008","08","S37000004","S00094213","S00004763","6332AC07C","S01007350","S01000733","S02001382","S02000133",40,49,45,63,45,56,"1458","S30000036","UKM6","UKM63","S19001816","164001","506","S20001449","164","S35000726","S09000003","019","32","32",,"S12000035",,,,"S22000058","4","5","Y",1, "EH41 4JZ","EH41","EH41 4",11/7/2011 00:00:00,,"350023","664926",55.87489737,-2.80032623,"N","S12000010","S14000020","S17000015","S16000102","S13002912","S08000024","S08000010","05","S37000010","S00102365","S00012406","6228AT12B","S01008248","S01001551","S02001547","S02000288",,,,,,,"4461","S30000005","UKM7","UKM73",,,,,,"S35000870","S09000002","000","28","28",,"S12000010",,,"S11000003","S22000059","5","6","Y",1, "KY13 8GG","KY13","KY13 8",30/6/2016 00:00:00,,"311666","702764",56.20924110,-3.42566487,"N","S12000048","S14000050","S17000013","S16000138","S13003132","S08000030","S08000013","03","S37000033","S00126336","S00034434","6453AK07A","S01011838","S01004979","S02002220","S02000950",,,,,,,"6721","S30000043","UKM7","UKM77","S19001635","308001","339","S20001308","308","S35000490","S09000005","000","53","53",,"S12000048",,,"S11000005","S22000074","3","3","Y",1, +"PA31 8UAB","PA31","PA31 8",7/7/2004 00:00:00,,"179709","708103",56.214350030,-5.554182280,"Y","S12000035","S14000005","S17000011","S16000083","S13002518","S08000022","S08000008","08","S37000004","S00094054","S00004613","6332AR16B","S01007313","S01000803","S02001375","S02000145",0,0,,,,,"3967","S30000036","UKM6","UKM63",,,,,,"S35000177","S09000003","157","32","32",,"S12000035",,,,"S22000086","6","8","N",, +"PA31 8UAA","PA31","PA31 8",7/7/2004 00:00:00,,"179562","707515",56.209009330,-5.556059030,"Y","S12000035","S14000005","S17000011","S16000083","S13002518","S08000022","S08000008","08","S37000004","S00094054","S00004613","6332AR16B","S01007313","S01000803","S02001375","S02000145",9,18,,,,,"3967","S30000036","UKM6","UKM63",,,,,,"S35000177","S09000003","000","32","32",,"S12000035",,,,"S22000086","6","8","N",, From 954469cd2c7f461182e931ba3c2550edbaedadf4 Mon Sep 17 00:00:00 2001 From: Christopher Blanchard Date: Sun, 23 Aug 2020 15:47:35 +0000 Subject: [PATCH 2/2] fix(SPD): Correct and ingest invalid SPD postcodes Some postcodes on SPD have an extra character appended. Since these are picked up as invalid postcodes, they can no longer be queried This change will ingest the first instance of these invalid postcodes (i.e. those ending with A) and drop all others --- app/models/scottish_postcode.js | 39 ++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/app/models/scottish_postcode.js b/app/models/scottish_postcode.js index e6da67fc..f1663d56 100644 --- a/app/models/scottish_postcode.js +++ b/app/models/scottish_postcode.js @@ -45,7 +45,7 @@ const foreignColumns = [ const toJoinString = () => { return relationships .map( - r => + (r) => `LEFT OUTER JOIN ${r.table} ON ${TABLE_NAME}.${r.key}=${r.table}.${r.foreignKey}` ) .join(" "); @@ -65,17 +65,35 @@ const findQuery = ` `; const SPD_COL_MAPPINGS = Object.freeze([ - { column: "postcode", method: row => row.extract("Postcode") }, + { column: "postcode", method: (row) => row.extract("Postcode") }, { column: "pc_compact", - method: row => row.extract("Postcode").replace(/\s/g, ""), + method: (row) => row.extract("Postcode").replace(/\s/g, ""), }, { column: "scottish_constituency_id", - method: row => row.extract("ScottishParliamentaryConstituency2014Code"), + method: (row) => row.extract("ScottishParliamentaryConstituency2014Code"), }, ]); +const EXCEPTION_REGEX = /A$/; + +/** + * Validates and potentially mutates a CSV row for ingest + * + * Unfortunately SPD appends extra characters to some postcodes. This method returns null when these cases are met, unless the postcode ends in `A`. + * + * Postcodes suffixed with `A` are made valid and returned to the stream for ingest + */ +const clean = (row) => { + const postcode = row[0]; + if (Postcode.isValid(postcode)) return row; + // Reject if invalid postcide has a non-A suffix + if (postcode.match(EXCEPTION_REGEX) === null) return null; + row[0] = postcode.replace(EXCEPTION_REGEX, ""); + return row; +}; + class ScottishPostcode extends Base { constructor() { super(TABLE_NAME, schema, indexes); @@ -117,13 +135,14 @@ class ScottishPostcode extends Base { this._csvSeed( { filepath, - transform: row => { - row.extract = code => extractor(row, code); + transform: (row) => { + clean(row); + row.extract = (code) => extractor(row, code); if (row.extract("Postcode") === "Postcode") return null; // Skip if header if (row.extract("DateOfDeletion").length !== 0) return null; // Skip row if terminated - return SPD_COL_MAPPINGS.map(elem => elem.method(row)); + return SPD_COL_MAPPINGS.map((elem) => elem.method(row)); }, - columns: SPD_COL_MAPPINGS.map(elem => elem.column).join(","), + columns: SPD_COL_MAPPINGS.map((elem) => elem.column).join(","), }, callback ); @@ -136,13 +155,13 @@ class ScottishPostcode extends Base { [ this._createRelation.bind(this), this.clear.bind(this), - cb => + (cb) => this.seedPostcodes.call( this, { filepath: largeUserFile, extractor: largeUserExtractor }, cb ), - cb => + (cb) => this.seedPostcodes.call( this, { filepath: smallUserFile, extractor: smallUserExtractor },