From 980a4e300ced687e83557d79893875771978700e Mon Sep 17 00:00:00 2001 From: Alexander Fenster Date: Thu, 15 Mar 2018 15:38:45 -0700 Subject: [PATCH 1/5] test: fix system and samples tests --- .circleci/config.yml | 19 +++++++++++++------ samples/deid.js | 8 +++++++- samples/resources/dates.csv | 4 ++-- .../resources/date-shift-context.correct.csv | 4 ++-- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a0ae3194..bd3f8130 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -55,19 +55,25 @@ workflows: - lint - docs filters: - branches: - only: master tags: - only: '/^v[\d.]+$/' + only: /.*/ +# filters: +# branches: +# only: master +# tags: +# only: '/^v[\d.]+$/' - sample_tests: requires: - lint - docs filters: - branches: - only: master tags: - only: '/^v[\d.]+$/' + only: /.*/ +# filters: +# branches: +# only: master +# tags: +# only: '/^v[\d.]+$/' - publish_npm: requires: - system_tests @@ -209,6 +215,7 @@ jobs: name: Run system tests. command: npm run system-test environment: + GCLOUD_PROJECT: long-door-651 GOOGLE_APPLICATION_CREDENTIALS: .circleci/key.json - run: name: Remove unencrypted key. diff --git a/samples/deid.js b/samples/deid.js index 10797e1f..d1bded04 100644 --- a/samples/deid.js +++ b/samples/deid.js @@ -157,6 +157,9 @@ function deidentifyWithDateShift( .toString() .split('\n'); const csvHeaders = csvLines[0].split(','); + if (csvLines[csvLines.length - 1] === '') { + csvLines.splice(-1); + } const csvRows = csvLines.slice(1); // Construct the table object @@ -224,7 +227,10 @@ function deidentifyWithDateShift( ); csvLines[rowIndex + 1] = rowValues.join(','); }); - fs.writeFileSync(outputCsvFile, csvLines.join('\n')); + fs.writeFileSync( + outputCsvFile, + csvLines.map(line => line + '\n').join('') + ); // Print status console.log(`Successfully saved date-shift output to ${outputCsvFile}`); diff --git a/samples/resources/dates.csv b/samples/resources/dates.csv index 056fccb3..6a80d40a 100644 --- a/samples/resources/dates.csv +++ b/samples/resources/dates.csv @@ -1,5 +1,5 @@ name,birth_date,register_date,credit_card -Ann,01/01/1970,07/21/1996,4532908762519852 +Ann,01/01/1980,07/21/1996,4532908762519852 James,03/06/1988,04/09/2001,4301261899725540 Dan,08/14/1945,11/15/2011,4620761856015295 -Laura,11/03/1992,01/04/2017,4564981067258901 \ No newline at end of file +Laura,11/03/1992,01/04/2017,4564981067258901 diff --git a/samples/system-test/resources/date-shift-context.correct.csv b/samples/system-test/resources/date-shift-context.correct.csv index f8bf3234..2329cb63 100644 --- a/samples/system-test/resources/date-shift-context.correct.csv +++ b/samples/system-test/resources/date-shift-context.correct.csv @@ -1,5 +1,5 @@ name,birth_date,register_date,credit_card -Ann,1/31/1970,8/20/1996,4532908762519852 +Ann,1/31/1980,8/20/1996,4532908762519852 James,4/5/1988,5/9/2001,4301261899725540 Dan,9/13/1945,12/15/2011,4620761856015295 -Laura,12/3/1992,2/3/2017,4564981067258901 \ No newline at end of file +Laura,12/3/1992,2/3/2017,4564981067258901 From 92a01d5946d28a078bd7e00d18477a33ba58591b Mon Sep 17 00:00:00 2001 From: Alexander Fenster Date: Thu, 15 Mar 2018 15:51:19 -0700 Subject: [PATCH 2/5] chore: restore circleci config --- .circleci/config.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bd3f8130..287ea02a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -55,25 +55,19 @@ workflows: - lint - docs filters: + branches: + only: master tags: - only: /.*/ -# filters: -# branches: -# only: master -# tags: -# only: '/^v[\d.]+$/' + only: '/^v[\d.]+$/' - sample_tests: requires: - lint - docs filters: + branches: + only: master tags: - only: /.*/ -# filters: -# branches: -# only: master -# tags: -# only: '/^v[\d.]+$/' + only: '/^v[\d.]+$/' - publish_npm: requires: - system_tests From aff075e00f7c44829a54ca3c9b294cf0ded22104 Mon Sep 17 00:00:00 2001 From: Alexander Fenster Date: Thu, 15 Mar 2018 16:17:04 -0700 Subject: [PATCH 3/5] better handling of new line character --- samples/deid.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/samples/deid.js b/samples/deid.js index d1bded04..19c40625 100644 --- a/samples/deid.js +++ b/samples/deid.js @@ -155,11 +155,9 @@ function deidentifyWithDateShift( const csvLines = fs .readFileSync(inputCsvFile) .toString() - .split('\n'); + .split('\n') + .filter(line => line.match(/,/)); const csvHeaders = csvLines[0].split(','); - if (csvLines[csvLines.length - 1] === '') { - csvLines.splice(-1); - } const csvRows = csvLines.slice(1); // Construct the table object @@ -227,9 +225,10 @@ function deidentifyWithDateShift( ); csvLines[rowIndex + 1] = rowValues.join(','); }); + csvLines.push(''); fs.writeFileSync( outputCsvFile, - csvLines.map(line => line + '\n').join('') + csvLines.join('\n') ); // Print status From 991936c5649826628b9ffa8c5b950fd23e15ea01 Mon Sep 17 00:00:00 2001 From: Alexander Fenster Date: Thu, 15 Mar 2018 16:18:03 -0700 Subject: [PATCH 4/5] prettier --- samples/deid.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/samples/deid.js b/samples/deid.js index 19c40625..82919f82 100644 --- a/samples/deid.js +++ b/samples/deid.js @@ -226,10 +226,7 @@ function deidentifyWithDateShift( csvLines[rowIndex + 1] = rowValues.join(','); }); csvLines.push(''); - fs.writeFileSync( - outputCsvFile, - csvLines.join('\n') - ); + fs.writeFileSync(outputCsvFile, csvLines.join('\n')); // Print status console.log(`Successfully saved date-shift output to ${outputCsvFile}`); From 4f7768dd5bb1a7cd2f496ea0c64c1d42a33a6049 Mon Sep 17 00:00:00 2001 From: Alexander Fenster Date: Thu, 15 Mar 2018 16:28:07 -0700 Subject: [PATCH 5/5] use String.includes instead of String.match --- samples/deid.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/deid.js b/samples/deid.js index 82919f82..34084855 100644 --- a/samples/deid.js +++ b/samples/deid.js @@ -156,7 +156,7 @@ function deidentifyWithDateShift( .readFileSync(inputCsvFile) .toString() .split('\n') - .filter(line => line.match(/,/)); + .filter(line => line.includes(',')); const csvHeaders = csvLines[0].split(','); const csvRows = csvLines.slice(1);