From 648e3e4661af7f2bb07296005b78148141b904a6 Mon Sep 17 00:00:00 2001 From: Centil Allan Garces Buendia Date: Mon, 10 Jan 2022 19:25:49 -0600 Subject: [PATCH 01/16] [FEAT] create the scraper in the getEuropeVariants.js file, modify the api_gov.spec.js and getGovData.js files --- tests/v3/covid-19/api_gov.spec.js | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/v3/covid-19/api_gov.spec.js b/tests/v3/covid-19/api_gov.spec.js index 90794570..af0d3804 100644 --- a/tests/v3/covid-19/api_gov.spec.js +++ b/tests/v3/covid-19/api_gov.spec.js @@ -71,6 +71,34 @@ describe.skip('TESTING /v3/covid-19/gov/canada', () => { }); }); +// here +describe.skip('TESTING /v3/covid-19/gov/europe variants', () => { + it('/v3/covid-19/gov/europe variants correct fields set', (done) => { + chai.request(app) + .get('/v3/covid-19/gov/canada') + .end((err, res) => { + testBasicProperties(err, res, 200, 'array'); + res.body.length.should.be.above(1); + res.body.forEach((element) => { + element.should.have.property('updated'); + element.should.have.property('country'); + element.should.have.property('yearWeek'); + element.should.have.property('source'); + element.should.have.property('newCases'); + element.should.have.property('numberSequenced'); + element.should.have.property('percentSequenced'); + element.should.have.property('validDenominator'); + element.should.have.property('variant'); + element.should.have.property('numberDetectionsVariant'); + element.should.have.property('numberSequencedKnownVariant'); + element.should.have.property('percentVariant'); + }); + done(); + }); + }); +}); +// here + describe.skip('TESTING /v3/covid-19/gov/italy', () => { it('/v3/covid-19/gov/italy correct amount of provinces', (done) => { chai.request(app) From 69e948a6132d21ded0c69e55b8d6342e1744412e Mon Sep 17 00:00:00 2001 From: Centil Allan Garces Buendia Date: Mon, 10 Jan 2022 20:14:11 -0600 Subject: [PATCH 02/16] [FEAT] create the scraper in the getEuropeVariants.js file, modify the api_gov.spec.js and getGovData.js files --- .../covid-19/govScrapers/getEuropeVariants.js | 35 +++++++++++++++++++ scrapers/covid-19/govScrapers/getGovData.js | 4 ++- 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 scrapers/covid-19/govScrapers/getEuropeVariants.js diff --git a/scrapers/covid-19/govScrapers/getEuropeVariants.js b/scrapers/covid-19/govScrapers/getEuropeVariants.js new file mode 100644 index 00000000..cc3f6631 --- /dev/null +++ b/scrapers/covid-19/govScrapers/getEuropeVariants.js @@ -0,0 +1,35 @@ +const axios = require('axios'); +const logger = require('../../../utils/logger'); +const csvUtils = require('../../../utils/csvUtils'); + +const PATH = 'https://opendata.ecdc.europa.eu/covid19/virusvariant/csv/data.csv' + + +/** + * Requests and parses csv data that is used to populate the data table on the European Centre for Disease Prevention and Control (ECDPC) site + */ + const europeData = async () => { + try { + const europeRes = (await axios.get(PATH)).data; + const parsedEuropeData = await csvUtils.parseCsvData(europeRes); + return parsedEuropeData.map(country => ({ + updated: Date.now(), + country: country.country, + yearWeek: country.year_week, + source: country.source, + newCases: parseInt(country.new_cases) || null, + numberSequenced: parseInt(country.number_sequenced) || null, + percentSequenced: parseFloat(country.percent_sequenced) || null, + validDenominator: country.valid_denominator, + variant: country.variant, + numberDetectionsVariant: parseInt(country.number_detections_variant) || null, + numberSequencedKnownVariant: parseInt(country.number_sequenced_known_variant) || null, + percentVariant: parseFloat(country.percent_variant) || null + })); + } catch (err) { + logger.err('Error: Requesting ECDPC Data failed!', err); + return null; + } +}; + +module.exports = europeData; diff --git a/scrapers/covid-19/govScrapers/getGovData.js b/scrapers/covid-19/govScrapers/getGovData.js index 306b4839..b31db678 100644 --- a/scrapers/covid-19/govScrapers/getGovData.js +++ b/scrapers/covid-19/govScrapers/getGovData.js @@ -9,6 +9,7 @@ const ukData = require('./getUK'); const israelData = require('./getIsrael'); const vietnamData = require('./getVietnam'); const indonesiaData = require('./getIndonesia'); +const europeData = require('./getEuropeVariants'); const nameUtils = require('../../../utils/nameUtils'); const logger = require('../../../utils/logger'); @@ -43,7 +44,8 @@ const govData = async (keys, redis) => { { country: 'UK', fn: ukData }, { country: 'Indonesia', fn: indonesiaData }, { country: 'Israel', fn: israelData }, - { country: 'Vietnam', fn: vietnamData } + { country: 'Vietnam', fn: vietnamData }, + { country: 'Europe', fn: europeData } ].map(_resolveData)); logger.info(`Updated gov data: ${(await redis.hkeys(keys.gov_countries)).length} government sources`); From 51095ca8b3d6dffe2ff0619321509d87d32bdcec Mon Sep 17 00:00:00 2001 From: Centil Allan Garces Buendia Date: Tue, 11 Jan 2022 23:59:25 -0600 Subject: [PATCH 03/16] [MODIFY] Rename the getEuropeVariants to --> getVariants --- routes/v3/covid-19/apiVariants.js | 41 +++++++++++++++++++ .../getEuropeVariants.js => getVariants.js} | 4 +- 2 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 routes/v3/covid-19/apiVariants.js rename scrapers/covid-19/{govScrapers/getEuropeVariants.js => getVariants.js} (92%) diff --git a/routes/v3/covid-19/apiVariants.js b/routes/v3/covid-19/apiVariants.js new file mode 100644 index 00000000..938e953a --- /dev/null +++ b/routes/v3/covid-19/apiVariants.js @@ -0,0 +1,41 @@ +// *from here* This model comes from apiGov.js: + +// eslint-disable-next-line new-cap +const router = require('express').Router(); + +const nameUtils = require('../../../utils/nameUtils'); +const { wordToBoolean } = require('../../../utils/stringUtils'); +const { redis, keys } = require('../../instances'); + +router.get('/v3/covid-19/variants/europe/:query', async (req, res) => { + const { allowNull } = req.query; + const { country: countryName, yearWeek } = req.params; + if (countryName) { + const standardizedCountryName = nameUtils.getCountryData(countryName.trim()).country || countryName.trim(); + const data = JSON.parse(await redis.hget(keys.gov_countries, standardizedCountryName)); + if (data) { + res.send(!wordToBoolean(allowNull) ? nameUtils.transformNull(data) : data); + } else { + res.status(404).send({ message: `Country '${standardizedCountryName}' not found or no data found for country` }); + } + } else { + res.send((await redis.hkeys(keys.gov_countries)).sort()); + } +}); + +module.exports = router; + +// *to here* + +// Take this code below as an example: +// router.get('/v3/covid-19/countries/:query', async (req, res) => { +// const { yesterday, twoDaysAgo, strict, allowNull } = req.query; +// const { query } = req.params; +// let countries = JSON.parse(await redis.get(wordToBoolean(yesterday) ? keys.yesterday_countries : wordToBoolean(twoDaysAgo) ? keys.twoDaysAgo_countries : keys.countries)) +// .filter(country => country.country.toLowerCase() !== 'world').map(fixApostrophe); +// countries = splitQuery(query) +// .map(country => nameUtils.getWorldometersData(countries, country, strict !== 'false')) +// .filter(value => value).map(country => !wordToBoolean(allowNull) ? nameUtils.transformNull(country) : country); +// if (countries.length > 0) res.send(countries.length === 1 ? countries[0] : countries); +// else res.status(404).send({ message: 'Country not found or doesn\'t have any cases' }); +// }); diff --git a/scrapers/covid-19/govScrapers/getEuropeVariants.js b/scrapers/covid-19/getVariants.js similarity index 92% rename from scrapers/covid-19/govScrapers/getEuropeVariants.js rename to scrapers/covid-19/getVariants.js index cc3f6631..b897d593 100644 --- a/scrapers/covid-19/govScrapers/getEuropeVariants.js +++ b/scrapers/covid-19/getVariants.js @@ -1,6 +1,6 @@ const axios = require('axios'); -const logger = require('../../../utils/logger'); -const csvUtils = require('../../../utils/csvUtils'); +const logger = require('../../utils/logger'); +const csvUtils = require('../../utils/csvUtils'); const PATH = 'https://opendata.ecdc.europa.eu/covid19/virusvariant/csv/data.csv' From c88a70a4c8a9cd3ee6c8c9df7b2f11f17078a7e4 Mon Sep 17 00:00:00 2001 From: Centil Allan Garces Buendia Date: Wed, 12 Jan 2022 00:00:52 -0600 Subject: [PATCH 04/16] [MODIFY] Add the line 22 variants to the json --- config/config.keys.json | 1 + 1 file changed, 1 insertion(+) diff --git a/config/config.keys.json b/config/config.keys.json index 424e26db..030804e1 100644 --- a/config/config.keys.json +++ b/config/config.keys.json @@ -19,6 +19,7 @@ "vaccine_coverage": "covidapi:vaccine_coverage", "vaccine_state_coverage": "covidapi:vaccine_state_coverage", "therapeutics": "covidapi:therapeutics", + "variants":"covidapi:variants", "influenza_ILINET": "influenza:ILINET", "influenza_USPHL": "influenza:USPHL", "influenza_USCL": "influenza:USCL" From 12611a67c6ff31c3ce18ee504934b73d50f81af2 Mon Sep 17 00:00:00 2001 From: Centil Allan Garces Buendia Date: Wed, 12 Jan 2022 00:02:21 -0600 Subject: [PATCH 05/16] [MODIFY] change the values of keys.variants in the line 15 and 22 --- routes/v3/covid-19/apiVariants.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/routes/v3/covid-19/apiVariants.js b/routes/v3/covid-19/apiVariants.js index 938e953a..a6a6c8a3 100644 --- a/routes/v3/covid-19/apiVariants.js +++ b/routes/v3/covid-19/apiVariants.js @@ -7,19 +7,19 @@ const nameUtils = require('../../../utils/nameUtils'); const { wordToBoolean } = require('../../../utils/stringUtils'); const { redis, keys } = require('../../instances'); -router.get('/v3/covid-19/variants/europe/:query', async (req, res) => { +router.get('/v3/covid-19/variants/countries/:query', async (req, res) => { const { allowNull } = req.query; - const { country: countryName, yearWeek } = req.params; + const { country: countryName, yearWeek, variant } = req.params; if (countryName) { const standardizedCountryName = nameUtils.getCountryData(countryName.trim()).country || countryName.trim(); - const data = JSON.parse(await redis.hget(keys.gov_countries, standardizedCountryName)); + const data = JSON.parse(await redis.hget(keys.variants, standardizedCountryName)); if (data) { res.send(!wordToBoolean(allowNull) ? nameUtils.transformNull(data) : data); } else { res.status(404).send({ message: `Country '${standardizedCountryName}' not found or no data found for country` }); } } else { - res.send((await redis.hkeys(keys.gov_countries)).sort()); + res.send((await redis.hkeys(keys.variants)).sort()); } }); From 89d6cb2b3e8b5907cd96007f35bd85c14dce6546 Mon Sep 17 00:00:00 2001 From: Centil Allan Garces Buendia Date: Wed, 12 Jan 2022 00:32:53 -0600 Subject: [PATCH 06/16] [ADD/MODIFY] add the instance of the variant.js file and modify the getGovData.js since the variant scraper doesn't belong in there --- routes/instances.js | 5 +++++ scrapers/covid-19/govScrapers/getGovData.js | 4 +--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/routes/instances.js b/routes/instances.js index 6cd8c2bd..802a5f34 100644 --- a/routes/instances.js +++ b/routes/instances.js @@ -12,6 +12,7 @@ const appleData = require('../scrapers/covid-19/appleMobilityData'); const govData = require('../scrapers/covid-19/govScrapers/getGovData'); const { getVaccineData, getVaccineCoverageData, getVaccineStateCoverageData } = require('../scrapers/covid-19/getVaccine'); const getTherapeuticsData = require('../scrapers/covid-19/getTherapeutics'); +const getVariants = require('../scrpers/covid-19/getVariants'); const getCDCDInfluenzaData = require('../scrapers/influenza/getCDC'); // KEYS @@ -72,6 +73,10 @@ module.exports = { await getTherapeuticsData(keys, redis); logger.info('Finished Therapeutics scraping!'); }, + executeScraperVariants: async () => { + await getVariants(keys, redis); + logger.info('Finished Variants scraping!'); + }, excecuteScraperInfluenza: async () => { await getCDCDInfluenzaData(keys, redis); logger.info('Finished CDC Influenza scraping!'); diff --git a/scrapers/covid-19/govScrapers/getGovData.js b/scrapers/covid-19/govScrapers/getGovData.js index b31db678..306b4839 100644 --- a/scrapers/covid-19/govScrapers/getGovData.js +++ b/scrapers/covid-19/govScrapers/getGovData.js @@ -9,7 +9,6 @@ const ukData = require('./getUK'); const israelData = require('./getIsrael'); const vietnamData = require('./getVietnam'); const indonesiaData = require('./getIndonesia'); -const europeData = require('./getEuropeVariants'); const nameUtils = require('../../../utils/nameUtils'); const logger = require('../../../utils/logger'); @@ -44,8 +43,7 @@ const govData = async (keys, redis) => { { country: 'UK', fn: ukData }, { country: 'Indonesia', fn: indonesiaData }, { country: 'Israel', fn: israelData }, - { country: 'Vietnam', fn: vietnamData }, - { country: 'Europe', fn: europeData } + { country: 'Vietnam', fn: vietnamData } ].map(_resolveData)); logger.info(`Updated gov data: ${(await redis.hkeys(keys.gov_countries)).length} government sources`); From 2dc3beb363adb46b0e00f91b86d27a812e825dc2 Mon Sep 17 00:00:00 2001 From: Centil Allan Garces Buendia Date: Thu, 13 Jan 2022 00:11:17 -0600 Subject: [PATCH 07/16] [MODIFY] change a several files in order to connect the variant scraper with the endpoint v3/covid-19/variants/countries/{country} --- config/index.js | 2 + package-lock.json | 173 +++++++++++------------------- routes/instances.js | 4 +- routes/v3/covid-19/apiVariants.js | 2 +- scrapers/covid-19/getVariants.js | 85 ++++++++++----- server.js | 1 + serverScraper.js | 5 +- tests/v3/covid-19/api_gov.spec.js | 52 ++++----- 8 files changed, 154 insertions(+), 170 deletions(-) diff --git a/config/index.js b/config/index.js index 03c3251a..19e72cf7 100644 --- a/config/index.js +++ b/config/index.js @@ -38,6 +38,8 @@ config.therapeuticsInterval = process.env.THERAPEUTICS_INTERVAL || 864e5; config.ebolaInterval = process.env.EBOLA_INTERVAL || 864e5; // eslint-disable-next-line camelcase config.cdcInterval = process.env.CDC_INTERVAL || 864e5; +// eslint-disable-next-line camelcase +config.variantInterval = process.env.VARIANT_INTERVAL || 864e5; // SENTRY KEY (ONLY FOR PRODUCTION) // eslint-disable-next-line camelcase diff --git a/package-lock.json b/package-lock.json index 9ef374ed..eaff6561 100644 --- a/package-lock.json +++ b/package-lock.json @@ -333,41 +333,12 @@ } }, "node_modules/ansi-align": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.0.tgz", - "integrity": "sha512-ZpClVKqXN3RGBmKibdfWzqCY4lnjEuoNzU5T0oEFpfd/z5qJHVarukridD4juLO2FXMiwUQxr9WqQtaYa8XRYw==", - "dev": true, - "dependencies": { - "string-width": "^3.0.0" - } - }, - "node_modules/ansi-align/node_modules/emoji-regex": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", - "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", - "dev": true - }, - "node_modules/ansi-align/node_modules/is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/ansi-align/node_modules/string-width": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", - "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.1.tgz", + "integrity": "sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==", "dev": true, "dependencies": { - "emoji-regex": "^7.0.1", - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^5.1.0" - }, - "engines": { - "node": ">=6" + "string-width": "^4.1.0" } }, "node_modules/ansi-colors": { @@ -401,9 +372,9 @@ } }, "node_modules/ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, "engines": { "node": ">=8" @@ -994,9 +965,9 @@ "dev": true }, "node_modules/concurrently": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-5.2.0.tgz", - "integrity": "sha512-XxcDbQ4/43d6CxR7+iV8IZXhur4KbmEJk1CetVMUqCy34z9l0DkszbY+/9wvmSnToTej0SYomc2WSRH+L0zVJw==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-5.3.0.tgz", + "integrity": "sha512-8MhqOB6PWlBfA2vJ8a0bSFKATOdWlHiQlk11IfmQBPaHVP8oP2gsh2MObE6UR3hqDHqvaIvLTyceNW6obVuFHQ==", "dev": true, "dependencies": { "chalk": "^2.4.2", @@ -1968,9 +1939,9 @@ "dev": true }, "node_modules/follow-redirects": { - "version": "1.14.4", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.4.tgz", - "integrity": "sha512-zwGkiSXC1MUJG/qmeIFH2HBJx9u0V46QGUe3YR1fXG8bXQxq7fLj0RjLZQ5nubr9qNJUZrH+xUcwXEoXNpfS+g==", + "version": "1.14.7", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.7.tgz", + "integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==", "funding": [ { "type": "individual", @@ -2181,9 +2152,9 @@ } }, "node_modules/glob-parent": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz", - "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==", + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", "dev": true, "dependencies": { "is-glob": "^4.0.1" @@ -3410,9 +3381,9 @@ } }, "node_modules/normalize-url": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz", - "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==", + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.1.tgz", + "integrity": "sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA==", "dev": true, "engines": { "node": ">=8" @@ -3692,9 +3663,9 @@ } }, "node_modules/path-parse": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, "node_modules/path-proxy": { @@ -4560,19 +4531,22 @@ } }, "node_modules/swagger-ui-dist": { - "version": "3.25.0", - "resolved": "https://registry.npmjs.org/swagger-ui-dist/-/swagger-ui-dist-3.25.0.tgz", - "integrity": "sha512-vwvJPPbdooTvDwLGzjIXinOXizDJJ6U1hxnJL3y6U3aL1d2MSXDmKg2139XaLBhsVZdnQJV2bOkX4reB+RXamg==" + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/swagger-ui-dist/-/swagger-ui-dist-4.1.3.tgz", + "integrity": "sha512-WvfPSfAAMlE/sKS6YkW47nX/hA7StmhYnAHc6wWCXNL0oclwLj6UXv0hQCkLnDgvebi0MEV40SJJpVjKUgH1IQ==" }, "node_modules/swagger-ui-express": { - "version": "4.1.4", - "resolved": "https://registry.npmjs.org/swagger-ui-express/-/swagger-ui-express-4.1.4.tgz", - "integrity": "sha512-Ea96ecpC+Iq9GUqkeD/LFR32xSs8gYqmTW1gXCuKg81c26WV6ZC2FsBSPVExQP6WkyUuz5HEiR0sEv/HCC343g==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/swagger-ui-express/-/swagger-ui-express-4.3.0.tgz", + "integrity": "sha512-jN46SEEe9EoXa3ZgZoKgnSF6z0w3tnM1yqhO4Y+Q4iZVc8JOQB960EZpIAz6rNROrDApVDwcMHR0mhlnc/5Omw==", "dependencies": { - "swagger-ui-dist": "^3.18.1" + "swagger-ui-dist": ">=4.1.3" }, "engines": { "node": ">= v0.10.32" + }, + "peerDependencies": { + "express": ">=4.0.0" } }, "node_modules/table": { @@ -5500,37 +5474,12 @@ } }, "ansi-align": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.0.tgz", - "integrity": "sha512-ZpClVKqXN3RGBmKibdfWzqCY4lnjEuoNzU5T0oEFpfd/z5qJHVarukridD4juLO2FXMiwUQxr9WqQtaYa8XRYw==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.1.tgz", + "integrity": "sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==", "dev": true, "requires": { - "string-width": "^3.0.0" - }, - "dependencies": { - "emoji-regex": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", - "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", - "dev": true - }, - "is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", - "dev": true - }, - "string-width": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", - "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", - "dev": true, - "requires": { - "emoji-regex": "^7.0.1", - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^5.1.0" - } - } + "string-width": "^4.1.0" } }, "ansi-colors": { @@ -5557,9 +5506,9 @@ } }, "ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true }, "ansi-styles": { @@ -6049,9 +5998,9 @@ "dev": true }, "concurrently": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-5.2.0.tgz", - "integrity": "sha512-XxcDbQ4/43d6CxR7+iV8IZXhur4KbmEJk1CetVMUqCy34z9l0DkszbY+/9wvmSnToTej0SYomc2WSRH+L0zVJw==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-5.3.0.tgz", + "integrity": "sha512-8MhqOB6PWlBfA2vJ8a0bSFKATOdWlHiQlk11IfmQBPaHVP8oP2gsh2MObE6UR3hqDHqvaIvLTyceNW6obVuFHQ==", "dev": true, "requires": { "chalk": "^2.4.2", @@ -6846,9 +6795,9 @@ "dev": true }, "follow-redirects": { - "version": "1.14.4", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.4.tgz", - "integrity": "sha512-zwGkiSXC1MUJG/qmeIFH2HBJx9u0V46QGUe3YR1fXG8bXQxq7fLj0RjLZQ5nubr9qNJUZrH+xUcwXEoXNpfS+g==" + "version": "1.14.7", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.7.tgz", + "integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==" }, "form-data": { "version": "2.5.1", @@ -7019,9 +6968,9 @@ } }, "glob-parent": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz", - "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==", + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", "dev": true, "requires": { "is-glob": "^4.0.1" @@ -8022,9 +7971,9 @@ "dev": true }, "normalize-url": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz", - "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==", + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.1.tgz", + "integrity": "sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA==", "dev": true }, "nth-check": { @@ -8246,9 +8195,9 @@ "dev": true }, "path-parse": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, "path-proxy": { @@ -8970,16 +8919,16 @@ } }, "swagger-ui-dist": { - "version": "3.25.0", - "resolved": "https://registry.npmjs.org/swagger-ui-dist/-/swagger-ui-dist-3.25.0.tgz", - "integrity": "sha512-vwvJPPbdooTvDwLGzjIXinOXizDJJ6U1hxnJL3y6U3aL1d2MSXDmKg2139XaLBhsVZdnQJV2bOkX4reB+RXamg==" + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/swagger-ui-dist/-/swagger-ui-dist-4.1.3.tgz", + "integrity": "sha512-WvfPSfAAMlE/sKS6YkW47nX/hA7StmhYnAHc6wWCXNL0oclwLj6UXv0hQCkLnDgvebi0MEV40SJJpVjKUgH1IQ==" }, "swagger-ui-express": { - "version": "4.1.4", - "resolved": "https://registry.npmjs.org/swagger-ui-express/-/swagger-ui-express-4.1.4.tgz", - "integrity": "sha512-Ea96ecpC+Iq9GUqkeD/LFR32xSs8gYqmTW1gXCuKg81c26WV6ZC2FsBSPVExQP6WkyUuz5HEiR0sEv/HCC343g==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/swagger-ui-express/-/swagger-ui-express-4.3.0.tgz", + "integrity": "sha512-jN46SEEe9EoXa3ZgZoKgnSF6z0w3tnM1yqhO4Y+Q4iZVc8JOQB960EZpIAz6rNROrDApVDwcMHR0mhlnc/5Omw==", "requires": { - "swagger-ui-dist": "^3.18.1" + "swagger-ui-dist": ">=4.1.3" } }, "table": { diff --git a/routes/instances.js b/routes/instances.js index 802a5f34..4f041b35 100644 --- a/routes/instances.js +++ b/routes/instances.js @@ -12,7 +12,7 @@ const appleData = require('../scrapers/covid-19/appleMobilityData'); const govData = require('../scrapers/covid-19/govScrapers/getGovData'); const { getVaccineData, getVaccineCoverageData, getVaccineStateCoverageData } = require('../scrapers/covid-19/getVaccine'); const getTherapeuticsData = require('../scrapers/covid-19/getTherapeutics'); -const getVariants = require('../scrpers/covid-19/getVariants'); +const variantsData = require('../scrapers/covid-19/getVariants'); const getCDCDInfluenzaData = require('../scrapers/influenza/getCDC'); // KEYS @@ -74,7 +74,7 @@ module.exports = { logger.info('Finished Therapeutics scraping!'); }, executeScraperVariants: async () => { - await getVariants(keys, redis); + await variantsData(keys, redis); logger.info('Finished Variants scraping!'); }, excecuteScraperInfluenza: async () => { diff --git a/routes/v3/covid-19/apiVariants.js b/routes/v3/covid-19/apiVariants.js index a6a6c8a3..966c5092 100644 --- a/routes/v3/covid-19/apiVariants.js +++ b/routes/v3/covid-19/apiVariants.js @@ -7,7 +7,7 @@ const nameUtils = require('../../../utils/nameUtils'); const { wordToBoolean } = require('../../../utils/stringUtils'); const { redis, keys } = require('../../instances'); -router.get('/v3/covid-19/variants/countries/:query', async (req, res) => { +router.get('/v3/covid-19/variants/countries/:country?', async (req, res) => { const { allowNull } = req.query; const { country: countryName, yearWeek, variant } = req.params; if (countryName) { diff --git a/scrapers/covid-19/getVariants.js b/scrapers/covid-19/getVariants.js index b897d593..56d34371 100644 --- a/scrapers/covid-19/getVariants.js +++ b/scrapers/covid-19/getVariants.js @@ -1,35 +1,64 @@ -const axios = require('axios'); -const logger = require('../../utils/logger'); -const csvUtils = require('../../utils/csvUtils'); - -const PATH = 'https://opendata.ecdc.europa.eu/covid19/virusvariant/csv/data.csv' +const axios = require("axios"); +const logger = require("../../utils/logger"); +const csvUtils = require("../../utils/csvUtils"); +const PATH = + "https://opendata.ecdc.europa.eu/covid19/virusvariant/csv/data.csv"; /** * Requests and parses csv data that is used to populate the data table on the European Centre for Disease Prevention and Control (ECDPC) site */ - const europeData = async () => { - try { - const europeRes = (await axios.get(PATH)).data; - const parsedEuropeData = await csvUtils.parseCsvData(europeRes); - return parsedEuropeData.map(country => ({ - updated: Date.now(), - country: country.country, - yearWeek: country.year_week, - source: country.source, - newCases: parseInt(country.new_cases) || null, - numberSequenced: parseInt(country.number_sequenced) || null, - percentSequenced: parseFloat(country.percent_sequenced) || null, - validDenominator: country.valid_denominator, - variant: country.variant, - numberDetectionsVariant: parseInt(country.number_detections_variant) || null, - numberSequencedKnownVariant: parseInt(country.number_sequenced_known_variant) || null, - percentVariant: parseFloat(country.percent_variant) || null - })); - } catch (err) { - logger.err('Error: Requesting ECDPC Data failed!', err); - return null; - } +const europeanCountriesData = async () => { + try { + const europeRes = (await axios.get(PATH)).data; + const parsedEuropeanCountriesData = await csvUtils.parseCsvData( + europeRes + ); + return parsedEuropeanCountriesData.map((country) => ({ + updated: Date.now(), + country: country.country, + yearWeek: country.year_week, + source: country.source, + newCases: parseInt(country.new_cases) || null, + numberSequenced: parseInt(country.number_sequenced) || null, + percentSequenced: parseFloat(country.percent_sequenced) || null, + validDenominator: country.valid_denominator, + variant: country.variant, + numberDetectionsVariant: + parseInt(country.number_detections_variant) || null, + numberSequencedKnownVariant: + parseInt(country.number_sequenced_known_variant) || null, + percentVariant: parseFloat(country.percent_variant) || null, + })); + } catch (err) { + logger.err("Error: Requesting ECDPC Data failed!", err); + return null; + } +}; + +const variantsData = async (keys, redis) => { + try { + const countryData = await europeanCountriesData(); + + if (countryData) { + const standardizedCountryName = nameUtils.getCountryData( + countryData.country.trim() + ).country; + await redis.hset( + keys.variants, + standardizedCountryName, + JSON.stringify(countryData) + ); + } else { + logger.info(`${countryData.country} scraper has failed.`); + } + + logger.info( + `Updated ECDC data: ${(await redis.hkeys(keys.variants)).length}` + ); + } catch (err) { + logger.err("Error: Requesting ECDC data failed!", err); + } }; -module.exports = europeData; +module.exports = variantsData; diff --git a/server.js b/server.js index 4214c527..e156c534 100644 --- a/server.js +++ b/server.js @@ -94,6 +94,7 @@ app.use(require('./routes/v3/covid-19/apiApple')); app.use(require('./routes/v3/covid-19/apiGov')); app.use(require('./routes/v3/covid-19/apiVaccine')); app.use(require('./routes/v3/covid-19/apiTherapeutics')); +app.use(require('./routes/v3/covid-19/apiVariants')); app.use(require('./routes/v3/influenza/apiInfluenza')); app.listen(port, () => logger.info(`Your app is listening on port ${port}`)); diff --git a/serverScraper.js b/serverScraper.js index 3103b89d..5f799bbe 100644 --- a/serverScraper.js +++ b/serverScraper.js @@ -1,5 +1,5 @@ const { scraper: { executeScraper, executeScraperNYTData, excecuteScraperAppleData, excecuteScraperGov, excecuteScraperVaccine, excecuteScraperVaccineCoverage, excecuteScraperVaccineStateCoverage, - executeScraperTherapeutics, excecuteScraperInfluenza }, + executeScraperTherapeutics, executeScraperVariants, excecuteScraperInfluenza }, config } = require('./routes/instances'); executeScraper(); @@ -11,6 +11,7 @@ excecuteScraperVaccine(); excecuteScraperVaccineCoverage(); excecuteScraperVaccineStateCoverage(); executeScraperTherapeutics(); +executeScraperVariants(); // Update Worldometer and Johns Hopkins data every 10 minutes setInterval(executeScraper, config.worldometersInterval); @@ -30,3 +31,5 @@ setInterval(excecuteScraperVaccineStateCoverage, config.vaccineCoverageInterval) setInterval(executeScraperTherapeutics, config.therapeuticsInterval); // Update CDC Influenza data every 24 hours setInterval(excecuteScraperInfluenza, config.cdcInterval); +// Update ECDC data every 24 hours +setInterval(executeScraperVariants, config.variantInterval); diff --git a/tests/v3/covid-19/api_gov.spec.js b/tests/v3/covid-19/api_gov.spec.js index af0d3804..f4a2f2d9 100644 --- a/tests/v3/covid-19/api_gov.spec.js +++ b/tests/v3/covid-19/api_gov.spec.js @@ -72,32 +72,32 @@ describe.skip('TESTING /v3/covid-19/gov/canada', () => { }); // here -describe.skip('TESTING /v3/covid-19/gov/europe variants', () => { - it('/v3/covid-19/gov/europe variants correct fields set', (done) => { - chai.request(app) - .get('/v3/covid-19/gov/canada') - .end((err, res) => { - testBasicProperties(err, res, 200, 'array'); - res.body.length.should.be.above(1); - res.body.forEach((element) => { - element.should.have.property('updated'); - element.should.have.property('country'); - element.should.have.property('yearWeek'); - element.should.have.property('source'); - element.should.have.property('newCases'); - element.should.have.property('numberSequenced'); - element.should.have.property('percentSequenced'); - element.should.have.property('validDenominator'); - element.should.have.property('variant'); - element.should.have.property('numberDetectionsVariant'); - element.should.have.property('numberSequencedKnownVariant'); - element.should.have.property('percentVariant'); - }); - done(); - }); - }); -}); -// here +// describe.skip('TESTING /v3/covid-19/gov/europe variants', () => { +// it('/v3/covid-19/gov/europe variants correct fields set', (done) => { +// chai.request(app) +// .get('/v3/covid-19/gov/canada') +// .end((err, res) => { +// testBasicProperties(err, res, 200, 'array'); +// res.body.length.should.be.above(1); +// res.body.forEach((element) => { +// element.should.have.property('updated'); +// element.should.have.property('country'); +// element.should.have.property('yearWeek'); +// element.should.have.property('source'); +// element.should.have.property('newCases'); +// element.should.have.property('numberSequenced'); +// element.should.have.property('percentSequenced'); +// element.should.have.property('validDenominator'); +// element.should.have.property('variant'); +// element.should.have.property('numberDetectionsVariant'); +// element.should.have.property('numberSequencedKnownVariant'); +// element.should.have.property('percentVariant'); +// }); +// done(); +// }); +// }); +// }); +// // here describe.skip('TESTING /v3/covid-19/gov/italy', () => { it('/v3/covid-19/gov/italy correct amount of provinces', (done) => { From ac42848884826c8e289f38b60552b609ccba1063 Mon Sep 17 00:00:00 2001 From: Centil Allan Garces Buendia Date: Fri, 14 Jan 2022 15:47:43 -0600 Subject: [PATCH 08/16] [MODIFY] now the scraper works and return the data in the correct format --- scrapers/covid-19/getVariants.js | 38 +++++++++++++++++++------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/scrapers/covid-19/getVariants.js b/scrapers/covid-19/getVariants.js index 56d34371..08a729e2 100644 --- a/scrapers/covid-19/getVariants.js +++ b/scrapers/covid-19/getVariants.js @@ -31,33 +31,41 @@ const europeanCountriesData = async () => { percentVariant: parseFloat(country.percent_variant) || null, })); } catch (err) { - logger.err("Error: Requesting ECDPC Data failed!", err); + logger.err("Error: Requesting ECDC Data failed!", err); return null; } }; const variantsData = async (keys, redis) => { try { - const countryData = await europeanCountriesData(); + const countriesData = await europeanCountriesData(); - if (countryData) { - const standardizedCountryName = nameUtils.getCountryData( - countryData.country.trim() - ).country; + const dataByCountry = countriesData + .map((obj) => obj.country) + .reduce((obj, country) => { + const groupByCountry = countriesData.filter( + (item) => item.country === country + ); + obj[country] = groupByCountry; + return obj; + }, {}); + + console.log(dataByCountry); + + const uniquesCountries = countriesData + .map((country) => country.country) + .filter((value, index, self) => self.indexOf(value) === index); + console.log(uniquesCountries); + + for (var i in uniquesCountries) { await redis.hset( keys.variants, - standardizedCountryName, - JSON.stringify(countryData) + uniquesCountries[i], + JSON.stringify(dataByCountry[uniquesCountries[i]]) ); - } else { - logger.info(`${countryData.country} scraper has failed.`); } - - logger.info( - `Updated ECDC data: ${(await redis.hkeys(keys.variants)).length}` - ); } catch (err) { - logger.err("Error: Requesting ECDC data failed!", err); + logger.err("Error: Formating ECDC data failed!", err); } }; From 614077cdba41a4f2de733e515d881e68b4b53120 Mon Sep 17 00:00:00 2001 From: Centil Allan Garces Buendia Date: Mon, 17 Jan 2022 09:42:28 -0600 Subject: [PATCH 09/16] [MODIFY] remove the parameters yearWeek and variant from router.get in apiVariants.js file since they're never use --- routes/v3/covid-19/apiVariants.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/v3/covid-19/apiVariants.js b/routes/v3/covid-19/apiVariants.js index 966c5092..2d580926 100644 --- a/routes/v3/covid-19/apiVariants.js +++ b/routes/v3/covid-19/apiVariants.js @@ -9,7 +9,7 @@ const { redis, keys } = require('../../instances'); router.get('/v3/covid-19/variants/countries/:country?', async (req, res) => { const { allowNull } = req.query; - const { country: countryName, yearWeek, variant } = req.params; + const { country: countryName } = req.params; if (countryName) { const standardizedCountryName = nameUtils.getCountryData(countryName.trim()).country || countryName.trim(); const data = JSON.parse(await redis.hget(keys.variants, standardizedCountryName)); From 5d133ed249dfb5ee33f7f6bf836fefe54af4a166 Mon Sep 17 00:00:00 2001 From: Centil Allan Garces Buendia Date: Mon, 17 Jan 2022 15:21:51 -0600 Subject: [PATCH 10/16] [MODIFY] add test for the getVariants.js scraper and remove console.log and commented code from the others files. --- associateTests.js | 4 +- routes/v3/covid-19/apiVariants.js | 15 ----- scrapers/covid-19/getVariants.js | 2 - tests/mochaSetup.spec.js | 68 +++++++++++-------- tests/v3/covid-19/api_gov.spec.js | 28 -------- tests/v3/covid-19/api_variants.spec.js | 91 ++++++++++++++++++++++++++ 6 files changed, 134 insertions(+), 74 deletions(-) create mode 100644 tests/v3/covid-19/api_variants.spec.js diff --git a/associateTests.js b/associateTests.js index c806d9da..ac1f29d6 100644 --- a/associateTests.js +++ b/associateTests.js @@ -14,7 +14,8 @@ const v3CovidTests = [ 'tests/v3/covid-19/api_nyt.spec.js', 'tests/v3/covid-19/api_therapeutics.spec.js', 'tests/v3/covid-19/api_vaccine.spec.js', - 'tests/v3/covid-19/api_worldometers.spec.js' + 'tests/v3/covid-19/api_worldometers.spec.js', + 'tests/v3/covid-19/api_variants.spec.js' ]; // Influenza tests are broken @@ -41,6 +42,7 @@ const fileNameToTestMap = { 'apiWorldometers.js': ['tests/v2/api_worldometers.spec.js', 'tests/v3/covid-19/api_worldometers.spec.js'], 'apiTherapeutics.js': ['tests/v3/covid-19/api_therapeutics.spec.js'], 'apiVaccine.js': ['tests/v3/covid-19/api_vaccine.spec.js'], + 'apiVariants.js': ['tests/v3/covid-19/api_variants.spec.js'], 'instances.js': allTests, 'appleMobilityData.js': ['tests/v2/api_apple.spec.js', 'tests/v3/covid-19/api_apple.spec.js'], 'getVaccine.js': ['tests/v3/covid-19/api_vaccine.spec.js'], diff --git a/routes/v3/covid-19/apiVariants.js b/routes/v3/covid-19/apiVariants.js index 2d580926..31d2b3af 100644 --- a/routes/v3/covid-19/apiVariants.js +++ b/routes/v3/covid-19/apiVariants.js @@ -24,18 +24,3 @@ router.get('/v3/covid-19/variants/countries/:country?', async (req, res) => { }); module.exports = router; - -// *to here* - -// Take this code below as an example: -// router.get('/v3/covid-19/countries/:query', async (req, res) => { -// const { yesterday, twoDaysAgo, strict, allowNull } = req.query; -// const { query } = req.params; -// let countries = JSON.parse(await redis.get(wordToBoolean(yesterday) ? keys.yesterday_countries : wordToBoolean(twoDaysAgo) ? keys.twoDaysAgo_countries : keys.countries)) -// .filter(country => country.country.toLowerCase() !== 'world').map(fixApostrophe); -// countries = splitQuery(query) -// .map(country => nameUtils.getWorldometersData(countries, country, strict !== 'false')) -// .filter(value => value).map(country => !wordToBoolean(allowNull) ? nameUtils.transformNull(country) : country); -// if (countries.length > 0) res.send(countries.length === 1 ? countries[0] : countries); -// else res.status(404).send({ message: 'Country not found or doesn\'t have any cases' }); -// }); diff --git a/scrapers/covid-19/getVariants.js b/scrapers/covid-19/getVariants.js index 08a729e2..0f39a844 100644 --- a/scrapers/covid-19/getVariants.js +++ b/scrapers/covid-19/getVariants.js @@ -50,8 +50,6 @@ const variantsData = async (keys, redis) => { return obj; }, {}); - console.log(dataByCountry); - const uniquesCountries = countriesData .map((country) => country.country) .filter((value, index, self) => self.indexOf(value) === index); diff --git a/tests/mochaSetup.spec.js b/tests/mochaSetup.spec.js index ec76ccfc..8dbf2814 100644 --- a/tests/mochaSetup.spec.js +++ b/tests/mochaSetup.spec.js @@ -1,35 +1,47 @@ -const { scraper: { executeScraper, executeScraperNYTData, excecuteScraperAppleData, excecuteScraperGov, excecuteScraperInfluenza, excecuteScraperVaccineCoverage, excecuteScraperVaccineStateCoverage }, - redis } = require('../routes/instances'); -const logger = require('../utils/logger'); +const { + scraper: { + executeScraper, + executeScraperNYTData, + excecuteScraperAppleData, + excecuteScraperGov, + excecuteScraperInfluenza, + excecuteScraperVaccineCoverage, + excecuteScraperVaccineStateCoverage, + executeScraperVariants, + }, + redis, +} = require("../routes/instances"); +const logger = require("../utils/logger"); -const [arg] = process.argv[5].split('/').slice(-1); -const argValue = arg.substring(arg.indexOf('_') + 1, arg.indexOf('.')); +const [arg] = process.argv[5].split("/").slice(-1); +const argValue = arg.substring(arg.indexOf("_") + 1, arg.indexOf(".")); const mapArgToScraper = { - worldometers: executeScraper, - jhucsse: executeScraper, - historical: executeScraper, - nyt: executeScraperNYTData, - apple: excecuteScraperAppleData, - gov: excecuteScraperGov, - influenza: excecuteScraperInfluenza, - vaccine: excecuteScraperVaccineCoverage, - vaccinestate: excecuteScraperVaccineStateCoverage + worldometers: executeScraper, + jhucsse: executeScraper, + historical: executeScraper, + nyt: executeScraperNYTData, + apple: excecuteScraperAppleData, + gov: excecuteScraperGov, + influenza: excecuteScraperInfluenza, + vaccine: excecuteScraperVaccineCoverage, + vaccinestate: excecuteScraperVaccineStateCoverage, + variants: executeScraperVariants }; // eslint-disable-next-line before(async () => { - await redis.flushall(); - logger.info('Finished flushing all data from redis.'); - if (argValue in mapArgToScraper) { - await mapArgToScraper[argValue](); - } else { - await executeScraper(); - await executeScraperNYTData(); - await excecuteScraperAppleData(); - await excecuteScraperGov(); - await excecuteScraperInfluenza(); - await excecuteScraperVaccineCoverage(); - await excecuteScraperVaccineStateCoverage(); - logger.info('Scraping all data finished.'); - } + await redis.flushall(); + logger.info("Finished flushing all data from redis."); + if (argValue in mapArgToScraper) { + await mapArgToScraper[argValue](); + } else { + await executeScraper(); + await executeScraperNYTData(); + await excecuteScraperAppleData(); + await excecuteScraperGov(); + await excecuteScraperInfluenza(); + await excecuteScraperVaccineCoverage(); + await excecuteScraperVaccineStateCoverage(); + logger.info("Scraping all data finished."); + } }); diff --git a/tests/v3/covid-19/api_gov.spec.js b/tests/v3/covid-19/api_gov.spec.js index f4a2f2d9..90794570 100644 --- a/tests/v3/covid-19/api_gov.spec.js +++ b/tests/v3/covid-19/api_gov.spec.js @@ -71,34 +71,6 @@ describe.skip('TESTING /v3/covid-19/gov/canada', () => { }); }); -// here -// describe.skip('TESTING /v3/covid-19/gov/europe variants', () => { -// it('/v3/covid-19/gov/europe variants correct fields set', (done) => { -// chai.request(app) -// .get('/v3/covid-19/gov/canada') -// .end((err, res) => { -// testBasicProperties(err, res, 200, 'array'); -// res.body.length.should.be.above(1); -// res.body.forEach((element) => { -// element.should.have.property('updated'); -// element.should.have.property('country'); -// element.should.have.property('yearWeek'); -// element.should.have.property('source'); -// element.should.have.property('newCases'); -// element.should.have.property('numberSequenced'); -// element.should.have.property('percentSequenced'); -// element.should.have.property('validDenominator'); -// element.should.have.property('variant'); -// element.should.have.property('numberDetectionsVariant'); -// element.should.have.property('numberSequencedKnownVariant'); -// element.should.have.property('percentVariant'); -// }); -// done(); -// }); -// }); -// }); -// // here - describe.skip('TESTING /v3/covid-19/gov/italy', () => { it('/v3/covid-19/gov/italy correct amount of provinces', (done) => { chai.request(app) diff --git a/tests/v3/covid-19/api_variants.spec.js b/tests/v3/covid-19/api_variants.spec.js new file mode 100644 index 00000000..29b53ed9 --- /dev/null +++ b/tests/v3/covid-19/api_variants.spec.js @@ -0,0 +1,91 @@ +/* eslint-disable no-undef */ +const chai = require('chai'); +const chaiHttp = require('chai-http'); +const app = require('../../../server'); +const { testBasicProperties } = require('../../testingFunctions'); +const { should } = require('chai'); + +chai.use(chaiHttp); + +const countries = [ + "Austria", + "Belgium", + "Bulgaria", + "Croatia", + "Cyprus", + "Czechia", + "Denmark", + "Estonia", + "Finland", + "France", + "Germany", + "Greece", + "Hungary", + "Iceland", + "Ireland", + "Italy", + "Latvia", + "Liechtenstein", + "Lithuania", + "Luxembourg", + "Malta", + "Netherlands", + "Norway", + "Poland", + "Portugal", + "Romania", + "Slovakia", + "Slovenia", + "Spain", + "Sweden" + ]; + +describe('TESTING /v3/covid-19/variants/countries general', () => { + it('/v3/covid-19/variants/countries/ correct countries', (done) => { + chai.request(app) + .get('/v3/covid-19/variants/countries') + .end((err, res) => { + testBasicProperties(err, res, 200, 'array'); + res.body.length.should.be.equal(countries.length); + res.body.forEach((country) => countries.should.include(country)); + done(); + }); + }); + + + it('TESTING /v3/covid-19/variants/countries invalid country', (done) => { + chai.request(app) + .get('/v3/covid-19/variants/countries/notACountry') + .end((err, res) => { + testBasicProperties(err, res, 404, 'object'); + res.body.should.have.property('message'); + done(); + }); + }); +}); + +describe('TESTING /v3/covid-19/variants/countries/country', () => { + it('/v3/covid-19/variants/Country variants correct fields set', (done) => { + chai.request(app) + .get('/v3/covid-19/variants/countries/Austria') + .end((err, res) => { + testBasicProperties(err, res, 200, 'array'); + res.body.length.should.be.above(1); + res.body.forEach((element) => { + element.should.have.property('updated'); + element.should.have.property('country'); + element.should.have.property('yearWeek'); + element.should.have.property('source'); + element.should.have.property('newCases'); + element.should.have.property('numberSequenced'); + element.should.have.property('percentSequenced'); + element.should.have.property('validDenominator'); + element.should.have.property('variant'); + element.should.have.property('numberDetectionsVariant'); + element.should.have.property('numberSequencedKnownVariant'); + element.should.have.property('percentVariant'); + }); + done(); + }); + }); +}); From 2fb8e425d80160677511a0e71f7f5586d35e62f8 Mon Sep 17 00:00:00 2001 From: Centil Allan Garces Buendia Date: Mon, 17 Jan 2022 18:21:50 -0600 Subject: [PATCH 11/16] [MODIFY] add swagger documentation --- public/apidocs/swagger_v3.json | 65 ++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/public/apidocs/swagger_v3.json b/public/apidocs/swagger_v3.json index b1ce38c2..702108e3 100644 --- a/public/apidocs/swagger_v3.json +++ b/public/apidocs/swagger_v3.json @@ -44,6 +44,10 @@ "name": "COVID-19: Therapeutics", "description": "(COVID-19 therapeutic trial data from raps.org, updated every 24 hours)" }, + { + "name": "COVID-19: Variants", + "description": "(COVID-19 data from The European Surveillance System -TESSy, provided by [Austria, Belgium, Bulgaria, Croatia, Cyprus, Czechia, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Iceland, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Norway, Poland, Portugal, Romania, Slovakia, Slovenia, Spain and Sweden] https://www.ecdc.europa.eu and released by ECDC updated every week)" + }, { "name": "Influenza: CDC", "description": "(Influenza data reported by the United States CDC, updated every 24 hours)" @@ -1507,6 +1511,61 @@ } } }, + "/v3/covid-19/variants/countries/": { + "get": { + "tags": [ + "COVID-19: Variants" + ], + "summary": "Get a list of supported countries for ECDC specific data", + "description": "Returns a list of supported country names", + "responses": { + "200": { + "description": "Status Ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/variantsECDC" + } + } + } + } + } + } + }, + "/v3/covid-19/variants/countries/{country}": { + "get": { + "tags": [ + "COVID-19: Variants" + ], + "parameters": [ + { + "name": "country", + "in": "path", + "required": true, + "description": "A valid country name from the /v3/covid-19/variants/countries/ endpoint", + "type": "string" + }, + { + "name": "allowNull", + "in": "query", + "enum": [ + "true", + "false", + "1", + "0" + ], + "description": "By default, if a value is missing, it is returned as 0. This allows nulls to be returned", + "type": "string" + } + ], + "summary": "Get COVID-19 ECDC reported data for a specific country", + "responses": { + "200": { + "description": "Status Ok" + } + } + } + }, "/v3/influenza/cdc/ILINet": { "get": { "tags": [ @@ -2593,6 +2652,12 @@ } } }, + "variantsECDC": { + "type": "array", + "items": { + "type": "string" + } + }, "influenzaUSPHL": { "properties": { "updated": { From 77816d71ced4b403faa046e13ea69b49b19b12b7 Mon Sep 17 00:00:00 2001 From: Centil Allan Garces Buendia Date: Tue, 18 Jan 2022 12:08:44 -0600 Subject: [PATCH 12/16] [MODIFY] fix swagger documentation --- public/apidocs/swagger_v3.json | 54 +++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/public/apidocs/swagger_v3.json b/public/apidocs/swagger_v3.json index 702108e3..4e61f75c 100644 --- a/public/apidocs/swagger_v3.json +++ b/public/apidocs/swagger_v3.json @@ -1561,7 +1561,14 @@ "summary": "Get COVID-19 ECDC reported data for a specific country", "responses": { "200": { - "description": "Status Ok" + "description": "Status Ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/variantsCountriesECDC" + } + } + } } } } @@ -2658,6 +2665,51 @@ "type": "string" } }, + "variantsCountriesECDC": { + "type": "array", + "items": { + "type": "object", + "properties": { + "updated": { + "type": "number", + "format": "date" + }, + "country": { + "type": "string" + }, + "yearWeek": { + "type": "string" + }, + "source": { + "type": "string" + }, + "newCases": { + "type": "number" + }, + "numberSequenced": { + "type": "number" + }, + "percentSequenced": { + "type": "number" + }, + "validDenominator": { + "type": "string" + }, + "variant": { + "type": "string" + }, + "numberDetectionsVariant": { + "type": "number" + }, + "numberSequencedKnownVariant": { + "type": "number" + }, + "percentVariant": { + "type": "number" + } + } + } + }, "influenzaUSPHL": { "properties": { "updated": { From c9c2f11de231c701472af8c120c7ceedbc00acec Mon Sep 17 00:00:00 2001 From: Centil Allan Garces Buendia Date: Wed, 19 Jan 2022 11:40:44 -0600 Subject: [PATCH 13/16] [MODIFY] fix eslint style problems --- routes/v3/covid-19/apiVariants.js | 2 +- scrapers/covid-19/getVariants.js | 106 ++++++++++++------------- tests/mochaSetup.spec.js | 76 +++++++++--------- tests/v3/covid-19/api_variants.spec.js | 63 ++++++++------- 4 files changed, 123 insertions(+), 124 deletions(-) diff --git a/routes/v3/covid-19/apiVariants.js b/routes/v3/covid-19/apiVariants.js index 31d2b3af..46799796 100644 --- a/routes/v3/covid-19/apiVariants.js +++ b/routes/v3/covid-19/apiVariants.js @@ -9,7 +9,7 @@ const { redis, keys } = require('../../instances'); router.get('/v3/covid-19/variants/countries/:country?', async (req, res) => { const { allowNull } = req.query; - const { country: countryName } = req.params; + const { country: countryName } = req.params; if (countryName) { const standardizedCountryName = nameUtils.getCountryData(countryName.trim()).country || countryName.trim(); const data = JSON.parse(await redis.hget(keys.variants, standardizedCountryName)); diff --git a/scrapers/covid-19/getVariants.js b/scrapers/covid-19/getVariants.js index 0f39a844..0dfd0e8c 100644 --- a/scrapers/covid-19/getVariants.js +++ b/scrapers/covid-19/getVariants.js @@ -1,70 +1,70 @@ -const axios = require("axios"); -const logger = require("../../utils/logger"); -const csvUtils = require("../../utils/csvUtils"); +const axios = require('axios'); +const logger = require('../../utils/logger'); +const csvUtils = require('../../utils/csvUtils'); -const PATH = - "https://opendata.ecdc.europa.eu/covid19/virusvariant/csv/data.csv"; +const PATH + = 'https://opendata.ecdc.europa.eu/covid19/virusvariant/csv/data.csv'; /** * Requests and parses csv data that is used to populate the data table on the European Centre for Disease Prevention and Control (ECDPC) site */ const europeanCountriesData = async () => { - try { - const europeRes = (await axios.get(PATH)).data; - const parsedEuropeanCountriesData = await csvUtils.parseCsvData( - europeRes - ); - return parsedEuropeanCountriesData.map((country) => ({ - updated: Date.now(), - country: country.country, - yearWeek: country.year_week, - source: country.source, - newCases: parseInt(country.new_cases) || null, - numberSequenced: parseInt(country.number_sequenced) || null, - percentSequenced: parseFloat(country.percent_sequenced) || null, - validDenominator: country.valid_denominator, - variant: country.variant, - numberDetectionsVariant: + try { + const europeRes = (await axios.get(PATH)).data; + const parsedEuropeanCountriesData = await csvUtils.parseCsvData( + europeRes + ); + return parsedEuropeanCountriesData.map((country) => ({ + updated: Date.now(), + country: country.country, + yearWeek: country.year_week, + source: country.source, + newCases: parseInt(country.new_cases) || null, + numberSequenced: parseInt(country.number_sequenced) || null, + percentSequenced: parseFloat(country.percent_sequenced) || null, + validDenominator: country.valid_denominator, + variant: country.variant, + numberDetectionsVariant: parseInt(country.number_detections_variant) || null, - numberSequencedKnownVariant: + numberSequencedKnownVariant: parseInt(country.number_sequenced_known_variant) || null, - percentVariant: parseFloat(country.percent_variant) || null, - })); - } catch (err) { - logger.err("Error: Requesting ECDC Data failed!", err); - return null; - } + percentVariant: parseFloat(country.percent_variant) || null + })); + } catch (err) { + logger.err('Error: Requesting ECDC Data failed!', err); + return null; + } }; const variantsData = async (keys, redis) => { - try { - const countriesData = await europeanCountriesData(); + try { + const countriesData = await europeanCountriesData(); - const dataByCountry = countriesData - .map((obj) => obj.country) - .reduce((obj, country) => { - const groupByCountry = countriesData.filter( - (item) => item.country === country - ); - obj[country] = groupByCountry; - return obj; - }, {}); + const dataByCountry = countriesData + .map((obj) => obj.country) + .reduce((obj, country) => { + const groupByCountry = countriesData.filter( + (item) => item.country === country + ); + obj[country] = groupByCountry; + return obj; + }, {}); - const uniquesCountries = countriesData - .map((country) => country.country) - .filter((value, index, self) => self.indexOf(value) === index); - console.log(uniquesCountries); + const uniquesCountries = countriesData + .map((country) => country.country) + .filter((value, index, self) => self.indexOf(value) === index); + console.log(uniquesCountries); - for (var i in uniquesCountries) { - await redis.hset( - keys.variants, - uniquesCountries[i], - JSON.stringify(dataByCountry[uniquesCountries[i]]) - ); - } - } catch (err) { - logger.err("Error: Formating ECDC data failed!", err); - } + for (var i in uniquesCountries) { + await redis.hset( + keys.variants, + uniquesCountries[i], + JSON.stringify(dataByCountry[uniquesCountries[i]]) + ); + } + } catch (err) { + logger.err('Error: Formating ECDC data failed!', err); + } }; module.exports = variantsData; diff --git a/tests/mochaSetup.spec.js b/tests/mochaSetup.spec.js index 8dbf2814..b8c86ca4 100644 --- a/tests/mochaSetup.spec.js +++ b/tests/mochaSetup.spec.js @@ -1,47 +1,47 @@ const { - scraper: { - executeScraper, - executeScraperNYTData, - excecuteScraperAppleData, - excecuteScraperGov, - excecuteScraperInfluenza, - excecuteScraperVaccineCoverage, - excecuteScraperVaccineStateCoverage, - executeScraperVariants, - }, - redis, -} = require("../routes/instances"); -const logger = require("../utils/logger"); + scraper: { + executeScraper, + executeScraperNYTData, + excecuteScraperAppleData, + excecuteScraperGov, + excecuteScraperInfluenza, + excecuteScraperVaccineCoverage, + excecuteScraperVaccineStateCoverage, + executeScraperVariants + }, + redis +} = require('../routes/instances'); +const logger = require('../utils/logger'); -const [arg] = process.argv[5].split("/").slice(-1); -const argValue = arg.substring(arg.indexOf("_") + 1, arg.indexOf(".")); +const [arg] = process.argv[5].split('/').slice(-1); +const argValue = arg.substring(arg.indexOf('_') + 1, arg.indexOf('.')); const mapArgToScraper = { - worldometers: executeScraper, - jhucsse: executeScraper, - historical: executeScraper, - nyt: executeScraperNYTData, - apple: excecuteScraperAppleData, - gov: excecuteScraperGov, - influenza: excecuteScraperInfluenza, - vaccine: excecuteScraperVaccineCoverage, - vaccinestate: excecuteScraperVaccineStateCoverage, + worldometers: executeScraper, + jhucsse: executeScraper, + historical: executeScraper, + nyt: executeScraperNYTData, + apple: excecuteScraperAppleData, + gov: excecuteScraperGov, + influenza: excecuteScraperInfluenza, + vaccine: excecuteScraperVaccineCoverage, + vaccinestate: excecuteScraperVaccineStateCoverage, variants: executeScraperVariants }; // eslint-disable-next-line before(async () => { - await redis.flushall(); - logger.info("Finished flushing all data from redis."); - if (argValue in mapArgToScraper) { - await mapArgToScraper[argValue](); - } else { - await executeScraper(); - await executeScraperNYTData(); - await excecuteScraperAppleData(); - await excecuteScraperGov(); - await excecuteScraperInfluenza(); - await excecuteScraperVaccineCoverage(); - await excecuteScraperVaccineStateCoverage(); - logger.info("Scraping all data finished."); - } + await redis.flushall(); + logger.info('Finished flushing all data from redis.'); + if (argValue in mapArgToScraper) { + await mapArgToScraper[argValue](); + } else { + await executeScraper(); + await executeScraperNYTData(); + await excecuteScraperAppleData(); + await excecuteScraperGov(); + await excecuteScraperInfluenza(); + await excecuteScraperVaccineCoverage(); + await excecuteScraperVaccineStateCoverage(); + logger.info('Scraping all data finished.'); + } }); diff --git a/tests/v3/covid-19/api_variants.spec.js b/tests/v3/covid-19/api_variants.spec.js index 29b53ed9..4d8fc461 100644 --- a/tests/v3/covid-19/api_variants.spec.js +++ b/tests/v3/covid-19/api_variants.spec.js @@ -3,42 +3,41 @@ const chai = require('chai'); const chaiHttp = require('chai-http'); const app = require('../../../server'); const { testBasicProperties } = require('../../testingFunctions'); -const { should } = require('chai'); chai.use(chaiHttp); const countries = [ - "Austria", - "Belgium", - "Bulgaria", - "Croatia", - "Cyprus", - "Czechia", - "Denmark", - "Estonia", - "Finland", - "France", - "Germany", - "Greece", - "Hungary", - "Iceland", - "Ireland", - "Italy", - "Latvia", - "Liechtenstein", - "Lithuania", - "Luxembourg", - "Malta", - "Netherlands", - "Norway", - "Poland", - "Portugal", - "Romania", - "Slovakia", - "Slovenia", - "Spain", - "Sweden" - ]; + 'Austria', + 'Belgium', + 'Bulgaria', + 'Croatia', + 'Cyprus', + 'Czechia', + 'Denmark', + 'Estonia', + 'Finland', + 'France', + 'Germany', + 'Greece', + 'Hungary', + 'Iceland', + 'Ireland', + 'Italy', + 'Latvia', + 'Liechtenstein', + 'Lithuania', + 'Luxembourg', + 'Malta', + 'Netherlands', + 'Norway', + 'Poland', + 'Portugal', + 'Romania', + 'Slovakia', + 'Slovenia', + 'Spain', + 'Sweden' +]; describe('TESTING /v3/covid-19/variants/countries general', () => { it('/v3/covid-19/variants/countries/ correct countries', (done) => { From 46bfc6aee35543ed3fff304c8c57627bd6add64c Mon Sep 17 00:00:00 2001 From: Centil Allan Garces Buendia Date: Tue, 25 Jan 2022 12:42:00 -0600 Subject: [PATCH 14/16] [MODIFY] I commented the lines 2-5 in api_influenza.spec.js file to fix the linting problem, as there are variables assigned that are never used. --- tests/v3/influenza/api_influenza.spec.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/v3/influenza/api_influenza.spec.js b/tests/v3/influenza/api_influenza.spec.js index 8393ccaa..b0acf718 100644 --- a/tests/v3/influenza/api_influenza.spec.js +++ b/tests/v3/influenza/api_influenza.spec.js @@ -1,8 +1,8 @@ /* eslint-disable no-undef */ -const chai = require('chai'); -const chaiHttp = require('chai-http'); -const app = require('../../../server'); -const { testBasicProperties } = require('../../testingFunctions'); +// const chai = require('chai'); +// const chaiHttp = require('chai-http'); +// const app = require('../../../server'); +// const { testBasicProperties } = require('../../testingFunctions'); // chai.use(chaiHttp); From e2a66aa4699bfb116b6a1d6ff2a5af6ddcdefdca Mon Sep 17 00:00:00 2001 From: Centil Allan Garces Buendia Date: Mon, 14 Feb 2022 14:57:24 -0600 Subject: [PATCH 15/16] [MODIFY] I change the --timeout value from 200000 to 300000 for the 'test' and the 'test-single' in the package.json file in order to get time enough to complete all tests --- package.json | 4 ++-- routes/v3/covid-19/apiVariants.js | 2 -- scrapers/covid-19/getVariants.js | 5 ++--- tests/mochaSetup.spec.js | 2 ++ tests/v3/covid-19/api_variants.spec.js | 6 +++--- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 6bc0c8bd..c2c87866 100644 --- a/package.json +++ b/package.json @@ -21,8 +21,8 @@ "lint": "eslint '**/*.js'", "lint:fix": "eslint ./**/*.js --fix", "lint:win32": "eslint ./**/*.js", - "test": "mocha ./tests --exit --recursive --timeout 200000", - "test-single": "mocha $1 --exit --timeout 200000", + "test": "mocha ./tests --exit --recursive --timeout 300000", + "test-single": "mocha $1 --exit --timeout 300000", "docker-start": "docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d --build", "docker-down": "docker-compose -f docker-compose.yml -f docker-compose.prod.yml down --rmi all", "docker-start-dev": "docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build", diff --git a/routes/v3/covid-19/apiVariants.js b/routes/v3/covid-19/apiVariants.js index 46799796..ee1e1cd4 100644 --- a/routes/v3/covid-19/apiVariants.js +++ b/routes/v3/covid-19/apiVariants.js @@ -1,5 +1,3 @@ -// *from here* This model comes from apiGov.js: - // eslint-disable-next-line new-cap const router = require('express').Router(); diff --git a/scrapers/covid-19/getVariants.js b/scrapers/covid-19/getVariants.js index 0dfd0e8c..68eb4ba5 100644 --- a/scrapers/covid-19/getVariants.js +++ b/scrapers/covid-19/getVariants.js @@ -53,9 +53,8 @@ const variantsData = async (keys, redis) => { const uniquesCountries = countriesData .map((country) => country.country) .filter((value, index, self) => self.indexOf(value) === index); - console.log(uniquesCountries); - for (var i in uniquesCountries) { + for (let i in uniquesCountries) { await redis.hset( keys.variants, uniquesCountries[i], @@ -63,7 +62,7 @@ const variantsData = async (keys, redis) => { ); } } catch (err) { - logger.err('Error: Formating ECDC data failed!', err); + logger.err('Error: Formating Variants data failed!', err); } }; diff --git a/tests/mochaSetup.spec.js b/tests/mochaSetup.spec.js index b8c86ca4..f6f50e92 100644 --- a/tests/mochaSetup.spec.js +++ b/tests/mochaSetup.spec.js @@ -15,6 +15,7 @@ const logger = require('../utils/logger'); const [arg] = process.argv[5].split('/').slice(-1); const argValue = arg.substring(arg.indexOf('_') + 1, arg.indexOf('.')); +console.log(argValue); const mapArgToScraper = { worldometers: executeScraper, jhucsse: executeScraper, @@ -42,6 +43,7 @@ before(async () => { await excecuteScraperInfluenza(); await excecuteScraperVaccineCoverage(); await excecuteScraperVaccineStateCoverage(); + await executeScraperVariants(); logger.info('Scraping all data finished.'); } }); diff --git a/tests/v3/covid-19/api_variants.spec.js b/tests/v3/covid-19/api_variants.spec.js index 4d8fc461..f6898423 100644 --- a/tests/v3/covid-19/api_variants.spec.js +++ b/tests/v3/covid-19/api_variants.spec.js @@ -6,7 +6,7 @@ const { testBasicProperties } = require('../../testingFunctions'); chai.use(chaiHttp); -const countries = [ +const variantCountries = [ 'Austria', 'Belgium', 'Bulgaria', @@ -45,8 +45,8 @@ describe('TESTING /v3/covid-19/variants/countries general', () => { .get('/v3/covid-19/variants/countries') .end((err, res) => { testBasicProperties(err, res, 200, 'array'); - res.body.length.should.be.equal(countries.length); - res.body.forEach((country) => countries.should.include(country)); + res.body.length.should.be.equal(variantCountries.length); + res.body.forEach((country) => variantCountries.should.include(country)); done(); }); }); From 7442e60078dad7927cb906e196e88f48274d86f7 Mon Sep 17 00:00:00 2001 From: Centil Allan Garces Buendia Date: Mon, 14 Feb 2022 16:47:02 -0600 Subject: [PATCH 16/16] [MODIFY] Put back the var in line 57 in order to get a right eslint performance --- scrapers/covid-19/getVariants.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapers/covid-19/getVariants.js b/scrapers/covid-19/getVariants.js index 68eb4ba5..86d8cf60 100644 --- a/scrapers/covid-19/getVariants.js +++ b/scrapers/covid-19/getVariants.js @@ -54,7 +54,7 @@ const variantsData = async (keys, redis) => { .map((country) => country.country) .filter((value, index, self) => self.indexOf(value) === index); - for (let i in uniquesCountries) { + for (var i in uniquesCountries) { await redis.hset( keys.variants, uniquesCountries[i],