[FEATURE] Connecte un datamart externe pour servir les résultats de certification pour ParcourSup (PIX-15800).

.circleci/config.yml

@@ -398,12 +398,19 @@ jobs:
           environment:
             NODE_ENV: test
             TEST_DATABASE_URL: postgres://circleci@localhost:5432/circleci_integration
+      - run:
+          name: Prepare datamart
+          command: npm run datamart:prepare
+          environment:
+            NODE_ENV: test
+            TEST_DATAMART_DATABASE_URL: postgres://circleci@localhost:5432/circleci_integration_datamart
       - run:
           name: Test
           command: npm run test:api:integration
           environment:
             NODE_ENV: test
             TEST_DATABASE_URL: postgres://circleci@localhost:5432/circleci_integration
+            TEST_DATAMART_DATABASE_URL: postgres://circleci@localhost:5432/circleci_integration_datamart
             TEST_REDIS_URL: redis://localhost:6379
             TEST_IMPORT_STORAGE_ENDPOINT: http://localhost:9090
             TEST_IMPORT_STORAGE_BUCKET_NAME: pix-import-test
@@ -428,6 +435,12 @@ jobs:
           environment:
             NODE_ENV: test
             TEST_DATABASE_URL: postgres://circleci@localhost:5432/circleci_acceptance
+      - run:
+          name: Prepare datamart
+          command: npm run datamart:prepare
+          environment:
+            NODE_ENV: test
+            TEST_DATAMART_DATABASE_URL: postgres://circleci@localhost:5432/circleci_integration_datamart
       - run:
           name: Test
           command: |
@@ -437,6 +450,7 @@ jobs:
           environment:
             NODE_ENV: test
             TEST_DATABASE_URL: postgres://circleci@localhost:5432/circleci_acceptance
+            TEST_DATAMART_DATABASE_URL: postgres://circleci@localhost:5432/circleci_integration_datamart
             TEST_REDIS_URL: redis://localhost:6379
             TEST_IMPORT_STORAGE_ENDPOINT: http://localhost:9090
             TEST_IMPORT_STORAGE_BUCKET_NAME: pix-import-test

api/datamart/datamart-builder/datamart-buffer.js

@@ -0,0 +1,15 @@
+const datamartBuffer = {
+  objectsToInsert: [],
+
+  pushInsertable({ tableName, values }) {
+    this.objectsToInsert.push({ tableName, values });
+
+    return values;
+  },
+
+  purge() {
+    this.objectsToInsert = [];
+  },
+};
+
+export { datamartBuffer };

api/datamart/datamart-builder/datamart-builder.js

@@ -0,0 +1,58 @@
+import { datamartBuffer } from './datamart-buffer.js';
+import { factory } from './factory/index.js';
+
+/**
+ * @class DatamartBuilder
+ * @property {Factory} factory
+ */
+class DatamartBuilder {
+  constructor({ knex }) {
+    this.knex = knex;
+    this.datamartBuffer = datamartBuffer;
+    this.factory = factory;
+  }
+
+  static async create({ knex }) {
+    const datamartBuilder = new DatamartBuilder({ knex });
+
+    try {
+      await datamartBuilder._init();
+    } catch (_) {
+      // Error thrown only with unit tests
+    }
+
+    return datamartBuilder;
+  }
+
+  async commit() {
+    try {
+      const trx = await this.knex.transaction();
+      for (const objectToInsert of this.datamartBuffer.objectsToInsert) {
+        await trx(objectToInsert.tableName).insert(objectToInsert.values);
+      }
+      await trx.commit();
+    } catch (err) {
+      // eslint-disable-next-line no-console
+      console.error(`Erreur dans datamartBuilder.commit() : ${err}`);
+      throw err;
+    } finally {
+      this.datamartBuffer.purge();
+    }
+  }
+
+  async clean() {
+    let rawQuery = '';
+
+    ['data_export_parcoursup_certif_result'].forEach((tableName) => {
+      rawQuery += `DELETE FROM ${tableName};`;
+    });
+
+    try {
+      await this.knex.raw(rawQuery);
+    } catch {
+      // ignore error
+    }
+  }
+}
+
+export { DatamartBuilder };

api/datamart/datamart-builder/factory/build-certification-result.js

@@ -0,0 +1,18 @@
+import { datamartBuffer } from '../datamart-buffer.js';
+
+const buildCertificationResult = function ({ nationalStudentId } = {}) {
+  const values = {
+    national_student_id: nationalStudentId,
+  };
+
+  datamartBuffer.pushInsertable({
+    tableName: 'data_export_parcoursup_certif_result',
+    values,
+  });
+
+  return {
+    nationalStudentId,
+  };
+};
+
+export { buildCertificationResult };

api/datamart/datamart-builder/factory/index.js

@@ -0,0 +1,16 @@
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { importNamedExportsFromDirectory } from '../../../src/shared/infrastructure/utils/import-named-exports-from-directory.js';
+
+const path = dirname(fileURLToPath(import.meta.url));
+const unwantedFiles = ['index.js'];
+
+const datamartBuilders = await importNamedExportsFromDirectory({
+  path: join(path, './'),
+  ignoredFileNames: unwantedFiles,
+});
+
+export const factory = {
+  ...datamartBuilders,
+};

api/datamart/knexfile.js

@@ -0,0 +1,22 @@
+import * as url from 'node:url';
+const __dirname = url.fileURLToPath(new URL('.', import.meta.url));
+import * as dotenv from 'dotenv';
+
+import { buildPostgresEnvironment } from '../db/utils/build-postgres-environment.js';
+dotenv.config({ path: `${__dirname}/../.env` });
+
+const baseConfiguration = {
+  migrationsDirectory: './migrations/',
+  seedsDirectory: './seeds/',
+  databaseUrl: process.env.DATAMART_DATABASE_URL,
+};
+
+const environments = {
+  development: buildPostgresEnvironment(baseConfiguration),
+
+  test: buildPostgresEnvironment({ ...baseConfiguration, databaseUrl: process.env.TEST_DATAMART_DATABASE_URL }),
+
+  production: buildPostgresEnvironment(baseConfiguration),
+};
+
+export default environments;

api/datamart/migrations/20241224080125-create-certification-results-table.js

@@ -0,0 +1,14 @@
+const TABLE_NAME = 'data_export_parcoursup_certif_result';
+
+const up = async function (knex) {
+  await knex.schema.createTable(TABLE_NAME, function (table) {
+    table.string('national_student_id');
+    table.index('national_student_id');
+  });
+};
+
+const down = async function (knex) {
+  await knex.schema.dropTable(TABLE_NAME);
+};
+
+export { down, up };

api/datamart/seeds/csv/data_export_parcoursup_certif_result.csv

@@ -0,0 +1,2 @@
+national_student_id
+123456789OK

api/datamart/seeds/populate-certification-results.js

@@ -0,0 +1,25 @@
+import { readdirSync } from 'node:fs';
+import path from 'node:path';
+
+import { parseCsvWithHeader } from '../../scripts/helpers/csvHelpers.js';
+import { logger } from '../../src/shared/infrastructure/utils/logger.js';
+
+// data are populated from exported data in csv files
+// csv files shall be named with the table name they represent
+const csvFolder = path.join(import.meta.dirname, 'csv');
+
+export async function seed(knex) {
+  const csvFilesToImport = readdirSync(csvFolder);
+  for (const file of csvFilesToImport) {
+    await insertDataFromFile(path.join(csvFolder, file), knex);
+  }
+}
+
+async function insertDataFromFile(file, knex) {
+  const tableName = path.basename(file, '.csv');
+  logger.info(`Inserting data from ${file} to ${tableName}`);
+
+  const data = await parseCsvWithHeader(file);
+  await knex(tableName).truncate();
+  await knex.batchInsert(tableName, data);
+}

api/db/knex-database-connection.js

@@ -33,12 +33,16 @@ Links :
  */
 types.setTypeParser(types.builtins.INT8, (value) => parseInt(value));
 
+import * as datamartKnexConfigs from '../datamart/knexfile.js';
 import * as knexConfigs from './knexfile.js';
 
 const { logging, environment } = config;
 const knexConfig = knexConfigs.default[environment];
 const configuredKnex = Knex(knexConfig);
 
+const datamartKnexConfig = datamartKnexConfigs.default[environment];
+const configuredDatamartKnex = Knex(datamartKnexConfig);
+
 /* QueryBuilder Extension */
 try {
   Knex.QueryBuilder.extend('whereInArray', function (column, values) {
@@ -79,6 +83,7 @@ configuredKnex.on('query-response', function (response, data) {
 });
 
 async function disconnect() {
+  await configuredDatamartKnex?.destroy();
   return configuredKnex.destroy();
 }
 
@@ -123,4 +128,10 @@ async function prepareDatabaseConnection() {
   logger.info('Connection to database established.');
 }
 
-export { disconnect, emptyAllTables, configuredKnex as knex, prepareDatabaseConnection };
+export {
+  configuredDatamartKnex as datamartKnex,
+  disconnect,
+  emptyAllTables,
+  configuredKnex as knex,
+  prepareDatabaseConnection,
+};

api/db/knexfile.js

@@ -1,51 +1,29 @@
 import * as url from 'node:url';
 const __dirname = url.fileURLToPath(new URL('.', import.meta.url));
 import * as dotenv from 'dotenv';
+
+import { buildPostgresEnvironment } from './utils/build-postgres-environment.js';
 dotenv.config({ path: `${__dirname}/../.env` });
 
-function localPostgresEnv(databaseUrl, knexAsyncStacktraceEnabled) {
-  return {
-    client: 'postgresql',
-    connection: databaseUrl,
-    pool: {
-      min: 1,
-      max: 4,
-    },
-    migrations: {
-      tableName: 'knex_migrations',
-      directory: './migrations',
-      loadExtensions: ['.js'],
-    },
-    seeds: {
-      directory: './seeds',
-      loadExtensions: ['.js'],
-    },
-    asyncStackTraces: knexAsyncStacktraceEnabled !== 'false',
-  };
-}
-const environments = {
-  development: localPostgresEnv(process.env.DATABASE_URL, process.env.KNEX_ASYNC_STACKTRACE_ENABLED),
+const baseConfiguration = {
+  migrationsDirectory: './migrations/',
+  seedsDirectory: './seeds/',
+  databaseUrl: process.env.DATABASE_URL,
+};
+
+export default {
+  development: buildPostgresEnvironment(baseConfiguration),
 
-  test: localPostgresEnv(process.env.TEST_DATABASE_URL, process.env.KNEX_ASYNC_STACKTRACE_ENABLED),
+  test: buildPostgresEnvironment({
+    ...baseConfiguration,
+    databaseUrl: process.env.TEST_DATABASE_URL,
+  }),
 
-  production: {
-    client: 'postgresql',
-    connection: process.env.DATABASE_URL,
+  production: buildPostgresEnvironment({
+    ...baseConfiguration,
     pool: {
-      min: parseInt(process.env.DATABASE_CONNECTION_POOL_MIN_SIZE, 10) || 1,
-      max: parseInt(process.env.DATABASE_CONNECTION_POOL_MAX_SIZE, 10) || 4,
-    },
-    migrations: {
-      tableName: 'knex_migrations',
-      directory: './migrations',
-      loadExtensions: ['.js'],
+      min: parseInt(process.env.DATABASE_CONNECTION_POOL_MIN_SIZE, 10),
+      max: parseInt(process.env.DATABASE_CONNECTION_POOL_MAX_SIZE, 10),
     },
-    seeds: {
-      directory: './seeds',
-      loadExtensions: ['.js'],
-    },
-    asyncStackTraces: process.env.KNEX_ASYNC_STACKTRACE_ENABLED !== 'false',
-  },
+  }),
 };
-
-export default environments;

api/db/utils/build-postgres-environment.js

@@ -0,0 +1,20 @@
+export function buildPostgresEnvironment({ databaseUrl, pool, migrationsDirectory, seedsDirectory }) {
+  return {
+    client: 'postgresql',
+    connection: databaseUrl,
+    pool: {
+      min: pool?.min || 1,
+      max: pool?.max || 4,
+    },
+    migrations: {
+      tableName: 'knex_migrations',
+      directory: migrationsDirectory,
+      loadExtensions: ['.js'],
+    },
+    seeds: {
+      directory: seedsDirectory,
+      loadExtensions: ['.js'],
+    },
+    asyncStackTraces: process.env.KNEX_ASYNC_STACKTRACE_ENABLED !== 'false',
+  };
+}

api/package.json

@@ -127,6 +127,12 @@
   "scripts": {
     "clean": "rm -rf node_modules",
     "cache:refresh": "node scripts/refresh-cache",
+    "datamart:create": "node scripts/datamart/create-datamart",
+    "datamart:delete": "node scripts/datamart/drop-datamart",
+    "datamart:new-migration": "npx knex --knexfile datamart/knexfile.js migrate:make --stub $PWD/db/template.js $migrationname",
+    "datamart:migrate": "knex --knexfile datamart/knexfile.js migrate:latest",
+    "datamart:prepare": "run-s datamart:delete datamart:create datamart:migrate",
+    "datamart:seed": "knex --knexfile datamart/knexfile.js seed:run",
     "db:new-migration": "npx knex --knexfile db/knexfile.js migrate:make --stub $PWD/db/template.js $migrationname",
     "db:create": "node scripts/database/create-database",
     "db:delete": "node scripts/database/drop-database",
@@ -161,7 +167,7 @@
     "start:job:watch": "nodemon worker.js",
     "start:job:fast:watch": "nodemon worker.js fast",
     "test": "npm run test:db:reset && npm run test:api",
-    "test:db:reset": "NODE_ENV=test npm run db:prepare",
+    "test:db:reset": "NODE_ENV=test run-p db:prepare datamart:prepare",
     "test:api": "for testType in 'unit' 'integration' 'acceptance'; do npm run test:api:$testType || status=1 ; done ; exit $status",
     "test:api:path": "NODE_ENV=test mocha --exit --recursive --reporter=${MOCHA_REPORTER:-dot}",
     "test:api:scripts": "npm run test:api:path -- tests/integration/scripts",

api/sample.env

@@ -155,6 +155,25 @@ PGBOSS_CONNECTION_POOL_MAX_SIZE=
 # sample: START_JOB_IN_WEB_PROCESS=true
 START_JOB_IN_WEB_PROCESS=
 
+# URL of the PostgreSQL database used for storing datamart
+#
+# If not present, the application will not be able to responds to requests involving
+# data from datamart (i.e. cold data)
+#
+# presence: optional
+# type: Url
+# default: none
+DATAMART_DATABASE_URL=postgresql://postgres@localhost/datamart
+
+# URL of the PostgreSQL database used for API local testing.
+#
+# If not present, the tests will fail.
+#
+# presence: required
+# type: Url
+# default: none
+TEST_DATAMART_DATABASE_URL=postgresql://postgres@localhost/datamart_test
+
 # ========
 # EMAILING
 # ========