From 17eebfc2353809d3d00c77e8ed8badcd55b95e90 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 31 Mar 2022 17:26:52 +0000 Subject: [PATCH 1/3] fix(deps): update dependency got to v12.0.3 (#120) Co-authored-by: Renovate Bot --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index e12a6af7..80b55775 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "bcryptjs": "^2.4.3", "cron": "^1.8.2", "dotenv": "^16.0.0", - "got": "12.0.2", + "got": "12.0.3", "graphql": "^16.3.0", "graphql-request": "^4.0.0", "helmet": "^5.0.2", diff --git a/yarn.lock b/yarn.lock index c1196a08..3e602609 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4091,10 +4091,10 @@ globby@^11.0.4: merge2 "^1.4.1" slash "^3.0.0" -got@12.0.2: - version "12.0.2" - resolved "https://registry.yarnpkg.com/got/-/got-12.0.2.tgz#8ce4c3baa50bb18a0858d2539caa0fac19e109bf" - integrity sha512-Zi4yHiqCgaorUbknr/RHFBsC3XqjSodaw0F3qxlqAqyj+OGYZl37/uy01R0qz++KANKQYdY5FHJ0okXZpEzwWQ== +got@12.0.3: + version "12.0.3" + resolved "https://registry.yarnpkg.com/got/-/got-12.0.3.tgz#c7314daab26d42039e624adbf98f6d442e5de749" + integrity sha512-hmdcXi/S0gcAtDg4P8j/rM7+j3o1Aq6bXhjxkDhRY2ipe7PHpvx/14DgTY2czHOLaGeU8VRvRecidwfu9qdFug== dependencies: "@sindresorhus/is" "^4.6.0" "@szmarczak/http-timer" "^5.0.1" From bb708eb09d30e1dd181dba1a3122b8329f8ea09f Mon Sep 17 00:00:00 2001 From: Social Groovy Bot <45039513+SocialGroovyBot@users.noreply.github.com> Date: Thu, 31 Mar 2022 17:27:30 +0000 Subject: [PATCH 2/3] chore(release): version 1.25.1 ## [1.25.1](https://github.com/SocialGouv/mon-psy-sante/compare/v1.25.0...v1.25.1) (2022-03-31) ### Bug Fixes * **deps:** update dependency got to v12.0.3 ([#120](https://github.com/SocialGouv/mon-psy-sante/issues/120)) ([17eebfc](https://github.com/SocialGouv/mon-psy-sante/commit/17eebfc2353809d3d00c77e8ed8badcd55b95e90)) --- CHANGELOG.md | 7 +++++++ package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0879371d..241e3db3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.25.1](https://github.com/SocialGouv/mon-psy-sante/compare/v1.25.0...v1.25.1) (2022-03-31) + + +### Bug Fixes + +* **deps:** update dependency got to v12.0.3 ([#120](https://github.com/SocialGouv/mon-psy-sante/issues/120)) ([17eebfc](https://github.com/SocialGouv/mon-psy-sante/commit/17eebfc2353809d3d00c77e8ed8badcd55b95e90)) + # [1.25.0](https://github.com/SocialGouv/mon-psy-sante/compare/v1.24.0...v1.25.0) (2022-03-31) diff --git a/package.json b/package.json index 80b55775..610802be 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "monpsy", - "version": "1.25.0", + "version": "1.25.1", "commitlint": { "extends": [ "@commitlint/config-conventional" From ea4e0dcb59fe128af92bf0cc0325731a1137bfa3 Mon Sep 17 00:00:00 2001 From: Caroline <4971715+carolineBda@users.noreply.github.com> Date: Mon, 4 Apr 2022 17:20:15 +0200 Subject: [PATCH 3/3] feat(scripts): add script to extract email + update logs on import scripts (#133) --- package.json | 3 +- scripts/extract-emails-acceptes.ts | 37 ++++++++++++++++ ...-departements.js => stats-departements.ts} | 42 ++++++++++++------- ...portState.spec.ts => import-state.spec.ts} | 0 src/cron/demarchesSimplifiees.ts | 18 ++++++-- .../demarchesSimplifiees/buildRequest.ts | 4 +- src/services/demarchesSimplifiees/import.ts | 6 +-- 7 files changed, 88 insertions(+), 22 deletions(-) create mode 100644 scripts/extract-emails-acceptes.ts rename scripts/{stats-departements.js => stats-departements.ts} (64%) rename src/__tests__/cron/{importState.spec.ts => import-state.spec.ts} (100%) diff --git a/package.json b/package.json index 610802be..f7d9e57a 100644 --- a/package.json +++ b/package.json @@ -85,6 +85,7 @@ "cy": "cypress open", "cy:run": "cypress run", "cron:launch": "ts-node --transpile-only src/cron/launch.ts", - "script:extract-departments": "ts-node --transpile-only scripts/stats-departements.js" + "script:extract-departments": "ts-node --transpile-only scripts/stats-departements.ts", + "script:extract-emails-acceptes": "ts-node --transpile-only scripts/extract-emails-acceptes.ts" } } diff --git a/scripts/extract-emails-acceptes.ts b/scripts/extract-emails-acceptes.ts new file mode 100644 index 00000000..4f839dbb --- /dev/null +++ b/scripts/extract-emails-acceptes.ts @@ -0,0 +1,37 @@ +/* eslint-disable no-process-exit */ + +import fs from "fs"; +import _ from "lodash"; + +import { countAll, getAll } from "../src/services/psychologists"; + +fs.createWriteStream("emails-monpsy-acceptes.csv"); + +let output = '"Email";"FirstName";"LastName"\n'; +(async () => { + const count = await countAll(); + console.log(`Fetching ${count} psy`); + const psychologists = await getAll({ + pagesize: count.toString(), + pageindex: "0", + }).catch((e) => { + console.log("Error while fetching psy: ", e); + process.exit(-1); + }); + const accepte = psychologists.map((item) => { + return `"${item.email}";"${item.firstName}";"${item.lastName}"`; + }); + + output += _.uniq(accepte).join("\n"); + if (psychologists.length !== _.uniq(accepte).length) { + console.log("!!!! some emails are not uniqs"); + } + + fs.writeFileSync("emails-monpsy-acceptes.csv", output); + + console.log("Nombre total de dossier en accepté", accepte.length); + + setTimeout(() => { + process.exit(); + }, 200); +})(); diff --git a/scripts/stats-departements.js b/scripts/stats-departements.ts similarity index 64% rename from scripts/stats-departements.js rename to scripts/stats-departements.ts index 170d0210..7b34aec3 100644 --- a/scripts/stats-departements.js +++ b/scripts/stats-departements.ts @@ -1,12 +1,16 @@ -const fs = require("fs"); -const _ = require("lodash"); -const { getAll, countAll } = require("../src/services/psychologists"); +import fs from "fs"; +import _ from "lodash"; -const file = fs.createWriteStream("departments-stats.csv"); +import { requestPsychologistsState } from "../src/services/demarchesSimplifiees/buildRequest"; +import { getAllPsychologistList } from "../src/services/demarchesSimplifiees/import"; import { DEPARTMENTS } from "../src/types/enums/department"; +const file = fs.createWriteStream("departments-stats.csv"); + +const extractDepNumber = (dep) => dep.split(" - ")[0]; + const deps = DEPARTMENTS.map((dep) => { - return { count: 0, name: dep, department: dep.split(" - ")[0] }; + return { count: 0, name: dep, department: extractDepNumber(dep) }; }); function log(text, count) { @@ -22,14 +26,22 @@ function displayList(list) { log("Dep", "total"); (async () => { - const count = await countAll(); - console.log(`Fetching ${count} psy`); - const psychologists = await getAll({ pagesize: count, pageindex: 0 }).catch( - (e) => { - console.log("Error while fetching psy: ", e); - process.exit(-1); - } + const requestPsychologistsStateWithDep = _.bind( + requestPsychologistsState, + null, + _, + `groupeInstructeur { + label + }` ); + const result = await getAllPsychologistList( + requestPsychologistsStateWithDep + ).catch((e) => { + console.log(e); + process.exit(-1); + }); + + const psychologists = result.psychologists; console.log(psychologists[0]); const sansSuite = _.filter(psychologists, { state: "sans_suite" }); const refuse = _.filter(psychologists, { state: "refuse" }); @@ -42,8 +54,10 @@ log("Dep", "total"); ); valids.forEach((psy) => { - const dep = _.find(deps, { department: psy.department }); - if (!dep) console.log(">>>>>>>>>", psy.department); + const dep = _.find(deps, { + department: extractDepNumber(psy.groupeInstructeur.label), + }); + if (!dep) console.log(">>>>>>>>>", psy.groupeInstructeur.label); dep.count++; }); diff --git a/src/__tests__/cron/importState.spec.ts b/src/__tests__/cron/import-state.spec.ts similarity index 100% rename from src/__tests__/cron/importState.spec.ts rename to src/__tests__/cron/import-state.spec.ts diff --git a/src/cron/demarchesSimplifiees.ts b/src/cron/demarchesSimplifiees.ts index 75835626..fcfb4494 100644 --- a/src/cron/demarchesSimplifiees.ts +++ b/src/cron/demarchesSimplifiees.ts @@ -7,14 +7,21 @@ import { getPsychologistState, } from "../services/demarchesSimplifiees/import"; import { + countAll, filterIdsNotInDb, saveMany, updateState, } from "../services/psychologists"; +async function logPsyNumber(): Promise { + const count = await countAll(); + console.log("Psy in DB: ", count); +} + export const importData = async (): Promise => { try { console.log("Starting importData..."); + await logPsyNumber(); const latestCursor = await models.DSCursor.findOne({ raw: true, where: { id: 1 }, @@ -22,18 +29,21 @@ export const importData = async (): Promise => { //@ts-ignore const dsAPIData = await getPsychologistList(latestCursor.cursor); - + console.log(dsAPIData); if (dsAPIData.psychologists.length > 0) { await saveMany(dsAPIData.psychologists); + console.log(`${dsAPIData.psychologists.length} saved`); await models.DSCursor.update( { cursor: dsAPIData.lastCursor, }, { where: { id: 1 } } ); + console.log(`New cursor ${dsAPIData.lastCursor} saved`); } - console.log(`Import done, ${dsAPIData.psychologists.length} saved`); + console.log(`Import done`); + await logPsyNumber(); } catch (err) { Sentry.captureException(err); console.error("ERROR: Could not import DS API data to PG", err); @@ -42,7 +52,8 @@ export const importData = async (): Promise => { export const importState = async (): Promise => { try { - console.log("Starting importState..."); + console.log("Starting importState."); + await logPsyNumber(); const dsAPIData = await getPsychologistState(); @@ -56,6 +67,7 @@ export const importState = async (): Promise => { console.log(`Added ${missingPsy.length} missing psys`); } console.log(`importState done.`); + await logPsyNumber(); } catch (err) { Sentry.captureException(err); console.error("ERROR importState: ", err); diff --git a/src/services/demarchesSimplifiees/buildRequest.ts b/src/services/demarchesSimplifiees/buildRequest.ts index afac7e4e..a317aac3 100644 --- a/src/services/demarchesSimplifiees/buildRequest.ts +++ b/src/services/demarchesSimplifiees/buildRequest.ts @@ -20,7 +20,8 @@ const getWhereConditionAfterCursor = (cursor: string): string => { }; export const requestPsychologistsState = async ( - afterCursor: string | undefined + afterCursor: string | undefined, + extraInfos?: string | undefined ): Promise => { const paginationCondition = getWhereConditionAfterCursor(afterCursor); const query = gql` @@ -36,6 +37,7 @@ export const requestPsychologistsState = async ( archived state number + ${extraInfos ?? ""} } } } diff --git a/src/services/demarchesSimplifiees/import.ts b/src/services/demarchesSimplifiees/import.ts index 4587efdf..e878c4a9 100644 --- a/src/services/demarchesSimplifiees/import.ts +++ b/src/services/demarchesSimplifiees/import.ts @@ -10,7 +10,7 @@ import { import parsePsychologists from "./parse-psychologists"; const limit = pLimit(5); -const getAllPsychologistList = async ( +export const getAllPsychologistList = async ( graphqlFunction: (string) => Promise, cursor: string | undefined = undefined, accumulator: DSPsychologist[] = [] @@ -43,7 +43,7 @@ export const getPsychologistList = async ( psychologists: Psychologist[]; lastCursor: string; }> => { - const time = `Fetching all psychologists from DS (query id #${Math.random().toString()} with cursor ${cursor})`; + const time = `Fetch all psychologists from DS (query id #${Math.random().toString()} with cursor ${cursor})`; console.time(time); const list = await getAllPsychologistList( @@ -79,7 +79,7 @@ export const getPsychologistFromListIds = async ( export const getPsychologistState = async (): Promise< Partial[] > => { - const time = `Fetching all psychologists state from DS (query id #${Math.random().toString()})`; + const time = `Fetch all psychologists state from DS (query id #${Math.random().toString()})`; console.time(time); const list = await getAllPsychologistList(requestPsychologistsState);