Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SPD fix #578

Merged
merged 2 commits into from
Aug 23, 2020
Merged

SPD fix #578

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2e71de7
Add test to capture issue #577
cblanc Aug 23, 2020
954469c
fix(SPD): Correct and ingest invalid SPD postcodes
cblanc Aug 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 29 additions & 10 deletions app/models/scottish_postcode.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ const foreignColumns = [
const toJoinString = () => {
return relationships
.map(
r =>
(r) =>
`LEFT OUTER JOIN ${r.table} ON ${TABLE_NAME}.${r.key}=${r.table}.${r.foreignKey}`
)
.join(" ");
Expand All @@ -65,17 +65,35 @@ const findQuery = `
`;

const SPD_COL_MAPPINGS = Object.freeze([
{ column: "postcode", method: row => row.extract("Postcode") },
{ column: "postcode", method: (row) => row.extract("Postcode") },
{
column: "pc_compact",
method: row => row.extract("Postcode").replace(/\s/g, ""),
method: (row) => row.extract("Postcode").replace(/\s/g, ""),
},
{
column: "scottish_constituency_id",
method: row => row.extract("ScottishParliamentaryConstituency2014Code"),
method: (row) => row.extract("ScottishParliamentaryConstituency2014Code"),
},
]);

const EXCEPTION_REGEX = /A$/;

/**
* Validates and potentially mutates a CSV row for ingest
*
* Unfortunately SPD appends extra characters to some postcodes. This method returns null when these cases are met, unless the postcode ends in `A`.
*
* Postcodes suffixed with `A` are made valid and returned to the stream for ingest
*/
const clean = (row) => {
const postcode = row[0];
if (Postcode.isValid(postcode)) return row;
// Reject if invalid postcide has a non-A suffix
if (postcode.match(EXCEPTION_REGEX) === null) return null;
row[0] = postcode.replace(EXCEPTION_REGEX, "");
return row;
};

class ScottishPostcode extends Base {
constructor() {
super(TABLE_NAME, schema, indexes);
Expand Down Expand Up @@ -117,13 +135,14 @@ class ScottishPostcode extends Base {
this._csvSeed(
{
filepath,
transform: row => {
row.extract = code => extractor(row, code);
transform: (row) => {
clean(row);
row.extract = (code) => extractor(row, code);
if (row.extract("Postcode") === "Postcode") return null; // Skip if header
if (row.extract("DateOfDeletion").length !== 0) return null; // Skip row if terminated
return SPD_COL_MAPPINGS.map(elem => elem.method(row));
return SPD_COL_MAPPINGS.map((elem) => elem.method(row));
},
columns: SPD_COL_MAPPINGS.map(elem => elem.column).join(","),
columns: SPD_COL_MAPPINGS.map((elem) => elem.column).join(","),
},
callback
);
Expand All @@ -136,13 +155,13 @@ class ScottishPostcode extends Base {
[
this._createRelation.bind(this),
this.clear.bind(this),
cb =>
(cb) =>
this.seedPostcodes.call(
this,
{ filepath: largeUserFile, extractor: largeUserExtractor },
cb
),
cb =>
(cb) =>
this.seedPostcodes.call(
this,
{ filepath: smallUserFile, extractor: smallUserExtractor },
Expand Down
35 changes: 21 additions & 14 deletions test/scottish_postcode.unit.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,32 +19,32 @@ describe("Scottish Postcode Model", () => {
const testPostcodeLarge = "ML11 0GH";
const testPostcodeSmall = "G82 1JW";

before(function(done) {
before(function (done) {
this.timeout(0);
series([clearPostcodeDb, seedPostcodeDb], done);
});

after(clearPostcodeDb);

describe("#setupTable", () => {
before(function(done) {
before(function (done) {
this.timeout(0);
ScottishPostcode._destroyRelation(error => {
ScottishPostcode._destroyRelation((error) => {
if (error) return done(error);
ScottishPostcode._setupTable(seedFilePath, done);
});
});

after(function(done) {
after(function (done) {
this.timeout(0);
ScottishPostcode._destroyRelation(error => {
ScottishPostcode._destroyRelation((error) => {
if (error) return done(error);
ScottishPostcode._setupTable(seedFilePath, done);
});
});

describe("#_createRelation", () => {
it(`creates a relation that matches ${ScottishPostcode.relation} schema`, done => {
it(`creates a relation that matches ${ScottishPostcode.relation} schema`, (done) => {
const query = `
SELECT
column_name, data_type, character_maximum_length, collation_name
Expand All @@ -54,7 +54,7 @@ describe("Scottish Postcode Model", () => {
ScottishPostcode._query(query, (error, result) => {
if (error) return done(error);
const impliedSchema = {};
result.rows.forEach(columnInfo => {
result.rows.forEach((columnInfo) => {
let columnName, dataType;
[columnName, dataType] = inferSchemaData(columnInfo);
impliedSchema[columnName] = dataType;
Expand All @@ -66,14 +66,21 @@ describe("Scottish Postcode Model", () => {
});

describe("#seedData", () => {
it("loads correct data from data directory", done => {
it("loads correct data from data directory", (done) => {
const query = `SELECT count(*) FROM ${ScottishPostcode.relation}`;
ScottishPostcode._query(query, (error, result) => {
if (error) return done(error);
assert.isTrue(result.rows[0].count > 0);
done();
});
});
it("loads postcodes suffixed with additional character", (done) => {
ScottishPostcode.find("PA31 8UA", (error, result) => {
if (error) return done(error);
assert.isNotNull(result);
done();
});
});
});
});

Expand All @@ -100,7 +107,7 @@ describe("Scottish Postcode Model", () => {
});

describe("#find", () => {
it("should return postcode with the right attributes for large user", done => {
it("should return postcode with the right attributes for large user", (done) => {
ScottishPostcode.find(testPostcodeLarge, (error, result) => {
if (error) return done(error);
assert.deepEqual(result, {
Expand All @@ -114,7 +121,7 @@ describe("Scottish Postcode Model", () => {
});
});

it("should return postcode with the right attributes for small users", done => {
it("should return postcode with the right attributes for small users", (done) => {
ScottishPostcode.find(testPostcodeSmall, (error, result) => {
if (error) return done(error);
assert.deepEqual(result, {
Expand All @@ -128,23 +135,23 @@ describe("Scottish Postcode Model", () => {
});
});

it("should return null for null/undefined postcode search", done => {
it("should return null for null/undefined postcode search", (done) => {
ScottishPostcode.find(null, (error, result) => {
if (error) return done(error);
assert.isNull(result);
done();
});
});

it("returns null if invalid postcode", done => {
it("returns null if invalid postcode", (done) => {
ScottishPostcode.find("1", (error, result) => {
if (error) return done(error);
assert.isNull(result);
done();
});
});

it("should be insensitive to space", done => {
it("should be insensitive to space", (done) => {
ScottishPostcode.find(
testPostcodeLarge.replace(/\s/, ""),
(error, result) => {
Expand All @@ -155,7 +162,7 @@ describe("Scottish Postcode Model", () => {
);
});

it("should return null if postcode does not exist", done => {
it("should return null if postcode does not exist", (done) => {
ScottishPostcode.find("ID11QD", (error, result) => {
if (error) return done(error);
assert.isNull(result);
Expand Down
2 changes: 2 additions & 0 deletions test/seed/SmallUser.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,3 +98,5 @@
"PA20 9BA","PA20","PA20 9",1/8/1973 00:00:00,,"208889","664416",55.83501986,-5.05330374,"N","S12000035","S14000005","S17000011","S16000083","S13002523","S08000022","S08000008","08","S37000004","S00094213","S00004763","6332AC07C","S01007350","S01000733","S02001382","S02000133",40,49,45,63,45,56,"1458","S30000036","UKM6","UKM63","S19001816","164001","506","S20001449","164","S35000726","S09000003","019","32","32",,"S12000035",,,,"S22000058","4","5","Y",1,
"EH41 4JZ","EH41","EH41 4",11/7/2011 00:00:00,,"350023","664926",55.87489737,-2.80032623,"N","S12000010","S14000020","S17000015","S16000102","S13002912","S08000024","S08000010","05","S37000010","S00102365","S00012406","6228AT12B","S01008248","S01001551","S02001547","S02000288",,,,,,,"4461","S30000005","UKM7","UKM73",,,,,,"S35000870","S09000002","000","28","28",,"S12000010",,,"S11000003","S22000059","5","6","Y",1,
"KY13 8GG","KY13","KY13 8",30/6/2016 00:00:00,,"311666","702764",56.20924110,-3.42566487,"N","S12000048","S14000050","S17000013","S16000138","S13003132","S08000030","S08000013","03","S37000033","S00126336","S00034434","6453AK07A","S01011838","S01004979","S02002220","S02000950",,,,,,,"6721","S30000043","UKM7","UKM77","S19001635","308001","339","S20001308","308","S35000490","S09000005","000","53","53",,"S12000048",,,"S11000005","S22000074","3","3","Y",1,
"PA31 8UAB","PA31","PA31 8",7/7/2004 00:00:00,,"179709","708103",56.214350030,-5.554182280,"Y","S12000035","S14000005","S17000011","S16000083","S13002518","S08000022","S08000008","08","S37000004","S00094054","S00004613","6332AR16B","S01007313","S01000803","S02001375","S02000145",0,0,,,,,"3967","S30000036","UKM6","UKM63",,,,,,"S35000177","S09000003","157","32","32",,"S12000035",,,,"S22000086","6","8","N",,
"PA31 8UAA","PA31","PA31 8",7/7/2004 00:00:00,,"179562","707515",56.209009330,-5.556059030,"Y","S12000035","S14000005","S17000011","S16000083","S13002518","S08000022","S08000008","08","S37000004","S00094054","S00004613","6332AR16B","S01007313","S01000803","S02001375","S02000145",9,18,,,,,"3967","S30000036","UKM6","UKM63",,,,,,"S35000177","S09000003","000","32","32",,"S12000035",,,,"S22000086","6","8","N",,