tech: add datamart connection

datamart is an external database serving cold data

.circleci/config.yml

@@ -398,12 +398,19 @@ jobs:
           environment:
             NODE_ENV: test
             TEST_DATABASE_URL: postgres://circleci@localhost:5432/circleci_integration
+      - run:
+          name: Prepare datamart
+          command: npm run datamart:prepare
+          environment:
+            NODE_ENV: test
+            TEST_DATAMART_DATABASE_URL: postgres://circleci@localhost:5432/circleci_integration_datamart
       - run:
           name: Test
           command: npm run test:api:integration
           environment:
             NODE_ENV: test
             TEST_DATABASE_URL: postgres://circleci@localhost:5432/circleci_integration
+            TEST_DATAMART_DATABASE_URL: postgres://circleci@localhost:5432/circleci_integration_datamart
             TEST_REDIS_URL: redis://localhost:6379
             TEST_IMPORT_STORAGE_ENDPOINT: http://localhost:9090
             TEST_IMPORT_STORAGE_BUCKET_NAME: pix-import-test
@@ -428,6 +435,12 @@ jobs:
           environment:
             NODE_ENV: test
             TEST_DATABASE_URL: postgres://circleci@localhost:5432/circleci_acceptance
+      - run:
+          name: Prepare datamart
+          command: npm run datamart:prepare
+          environment:
+            NODE_ENV: test
+            TEST_DATAMART_DATABASE_URL: postgres://circleci@localhost:5432/circleci_integration_datamart
       - run:
           name: Test
           command: |
@@ -437,6 +450,7 @@ jobs:
           environment:
             NODE_ENV: test
             TEST_DATABASE_URL: postgres://circleci@localhost:5432/circleci_acceptance
+            TEST_DATAMART_DATABASE_URL: postgres://circleci@localhost:5432/circleci_integration_datamart
             TEST_REDIS_URL: redis://localhost:6379
             TEST_IMPORT_STORAGE_ENDPOINT: http://localhost:9090
             TEST_IMPORT_STORAGE_BUCKET_NAME: pix-import-test

api/datamart/datamart-builder/datamart-buffer.js

@@ -0,0 +1,15 @@
+const datamartBuffer = {
+  objectsToInsert: [],
+
+  pushInsertable({ tableName, values }) {
+    this.objectsToInsert.push({ tableName, values });
+
+    return values;
+  },
+
+  purge() {
+    this.objectsToInsert = [];
+  },
+};
+
+export { datamartBuffer };

api/datamart/datamart-builder/datamart-builder.js

@@ -0,0 +1,58 @@
+import { datamartBuffer } from './datamart-buffer.js';
+import { factory } from './factory/index.js';
+
+/**
+ * @class DatamartBuilder
+ * @property {Factory} factory
+ */
+class DatamartBuilder {
+  constructor({ knex }) {
+    this.knex = knex;
+    this.datamartBuffer = datamartBuffer;
+    this.factory = factory;
+  }
+
+  static async create({ knex }) {
+    const datamartBuilder = new DatamartBuilder({ knex });
+
+    try {
+      await datamartBuilder._init();
+    } catch (_) {
+      // Error thrown only with unit tests
+    }
+
+    return datamartBuilder;
+  }
+
+  async commit() {
+    try {
+      const trx = await this.knex.transaction();
+      for (const objectToInsert of this.datamartBuffer.objectsToInsert) {
+        await trx(objectToInsert.tableName).insert(objectToInsert.values);
+      }
+      await trx.commit();
+    } catch (err) {
+      // eslint-disable-next-line no-console
+      console.error(`Erreur dans datamartBuilder.commit() : ${err}`);
+      throw err;
+    } finally {
+      this.datamartBuffer.purge();
+    }
+  }
+
+  async clean() {
+    let rawQuery = '';
+
+    ['data_export_parcoursup_certif_result'].forEach((tableName) => {
+      rawQuery += `DELETE FROM ${tableName};`;
+    });
+
+    try {
+      await this.knex.raw(rawQuery);
+    } catch {
+      // ignore error
+    }
+  }
+}
+
+export { DatamartBuilder };

api/datamart/datamart-builder/factory/build-certification-result.js

@@ -0,0 +1,18 @@
+import { datamartBuffer } from '../datamart-buffer.js';
+
+const buildCertificationResult = function ({ nationalStudentId } = {}) {
+  const values = {
+    national_student_id: nationalStudentId,
+  };
+
+  datamartBuffer.pushInsertable({
+    tableName: 'data_export_parcoursup_certif_result',
+    values,
+  });
+
+  return {
+    nationalStudentId,
+  };
+};
+
+export { buildCertificationResult };

api/datamart/datamart-builder/factory/index.js

@@ -0,0 +1,16 @@
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { importNamedExportsFromDirectory } from '../../../src/shared/infrastructure/utils/import-named-exports-from-directory.js';
+
+const path = dirname(fileURLToPath(import.meta.url));
+const unwantedFiles = ['index.js'];
+
+const datamartBuilders = await importNamedExportsFromDirectory({
+  path: join(path, './'),
+  ignoredFileNames: unwantedFiles,
+});
+
+export const factory = {
+  ...datamartBuilders,
+};

api/datamart/knexfile.js

@@ -0,0 +1,51 @@
+import * as url from 'node:url';
+const __dirname = url.fileURLToPath(new URL('.', import.meta.url));
+import * as dotenv from 'dotenv';
+dotenv.config({ path: `${__dirname}/../.env` });
+
+function localPostgresEnv(databaseUrl, knexAsyncStacktraceEnabled) {
+  return {
+    client: 'postgresql',
+    connection: databaseUrl,
+    pool: {
+      min: 1,
+      max: 4,
+    },
+    migrations: {
+      tableName: 'knex_migrations',
+      directory: './migrations',
+      loadExtensions: ['.js'],
+    },
+    seeds: {
+      directory: './seeds',
+      loadExtensions: ['.js'],
+      asyncStackTraces: knexAsyncStacktraceEnabled !== 'false',
+    },
+  };
+}
+const environments = {
+  development: localPostgresEnv(process.env.DATAMART_DATABASE_URL, process.env.KNEX_ASYNC_STACKTRACE_ENABLED),
+
+  test: localPostgresEnv(process.env.TEST_DATAMART_DATABASE_URL, process.env.KNEX_ASYNC_STACKTRACE_ENABLED),
+
+  production: {
+    client: 'postgresql',
+    connection: process.env.DATAMART_DATABASE_URL,
+    pool: {
+      min: 1,
+      max: 4,
+    },
+    migrations: {
+      tableName: 'knex_migrations',
+      directory: './migrations',
+      loadExtensions: ['.js'],
+    },
+    seeds: {
+      directory: './seeds',
+      loadExtensions: ['.js'],
+    },
+    asyncStackTraces: process.env.KNEX_ASYNC_STACKTRACE_ENABLED !== 'false',
+  },
+};
+
+export default environments;

api/datamart/migrations/20241224080125-create-certification-results-table.js

@@ -0,0 +1,14 @@
+const TABLE_NAME = 'data_export_parcoursup_certif_result';
+
+const up = async function (knex) {
+  await knex.schema.createTable(TABLE_NAME, function (table) {
+    table.string('national_student_id');
+    table.index('national_student_id');
+  });
+};
+
+const down = async function (knex) {
+  await knex.schema.dropTable(TABLE_NAME);
+};
+
+export { down, up };

api/datamart/seeds/csv/data_export_parcoursup_certif_result.csv

@@ -0,0 +1,2 @@
+national_student_id
+123456789OK

api/datamart/seeds/populate-certification-results.js

@@ -0,0 +1,25 @@
+import { readdirSync } from 'node:fs';
+import path from 'node:path';
+
+import { parseCsvWithHeader } from '../../scripts/helpers/csvHelpers.js';
+import { logger } from '../../src/shared/infrastructure/utils/logger.js';
+
+// data are populated from exported data in csv files
+// csv files shall be named with the table name they represent
+const csvFolder = path.join(import.meta.dirname, 'csv');
+
+export async function seed(knex) {
+  const csvFilesToImport = readdirSync(csvFolder);
+  for (const file of csvFilesToImport) {
+    await insertDataFromFile(path.join(csvFolder, file), knex);
+  }
+}
+
+async function insertDataFromFile(file, knex) {
+  const tableName = path.basename(file, '.csv');
+  logger.info(`Inserting data from ${file} to ${tableName}`);
+
+  const data = await parseCsvWithHeader(file);
+  await knex(tableName).truncate();
+  await knex.batchInsert(tableName, data);
+}

api/db/knex-database-connection.js

@@ -33,12 +33,16 @@ Links :
  */
 types.setTypeParser(types.builtins.INT8, (value) => parseInt(value));
 
+import * as datamartKnexConfigs from '../datamart/knexfile.js';
 import * as knexConfigs from './knexfile.js';
 
 const { logging, environment } = config;
 const knexConfig = knexConfigs.default[environment];
 const configuredKnex = Knex(knexConfig);
 
+const datamartKnexConfig = datamartKnexConfigs.default[environment];
+const configuredDatamartKnex = Knex(datamartKnexConfig);
+
 /* QueryBuilder Extension */
 try {
   Knex.QueryBuilder.extend('whereInArray', function (column, values) {
@@ -79,6 +83,7 @@ configuredKnex.on('query-response', function (response, data) {
 });
 
 async function disconnect() {
+  await configuredDatamartKnex?.destroy();
   return configuredKnex.destroy();
 }
 
@@ -123,4 +128,10 @@ async function prepareDatabaseConnection() {
   logger.info('Connection to database established.');
 }
 
-export { disconnect, emptyAllTables, configuredKnex as knex, prepareDatabaseConnection };
+export {
+  configuredDatamartKnex as datamartKnex,
+  disconnect,
+  emptyAllTables,
+  configuredKnex as knex,
+  prepareDatabaseConnection,
+};

api/package.json

@@ -127,6 +127,12 @@
   "scripts": {
     "clean": "rm -rf node_modules",
     "cache:refresh": "node scripts/refresh-cache",
+    "datamart:create": "node scripts/datamart/create-datamart",
+    "datamart:delete": "node scripts/datamart/drop-datamart",
+    "datamart:new-migration": "npx knex --knexfile datamart/knexfile.js migrate:make --stub $PWD/db/template.js $migrationname",
+    "datamart:migrate": "knex --knexfile datamart/knexfile.js migrate:latest",
+    "datamart:prepare": "run-s datamart:delete datamart:create datamart:migrate",
+    "datamart:seed": "knex --knexfile datamart/knexfile.js seed:run",
     "db:new-migration": "npx knex --knexfile db/knexfile.js migrate:make --stub $PWD/db/template.js $migrationname",
     "db:create": "node scripts/database/create-database",
     "db:delete": "node scripts/database/drop-database",
@@ -161,7 +167,7 @@
     "start:job:watch": "nodemon worker.js",
     "start:job:fast:watch": "nodemon worker.js fast",
     "test": "npm run test:db:reset && npm run test:api",
-    "test:db:reset": "NODE_ENV=test npm run db:prepare",
+    "test:db:reset": "NODE_ENV=test run-p db:prepare datamart:prepare",
     "test:api": "for testType in 'unit' 'integration' 'acceptance'; do npm run test:api:$testType || status=1 ; done ; exit $status",
     "test:api:path": "NODE_ENV=test mocha --exit --recursive --reporter=${MOCHA_REPORTER:-dot}",
     "test:api:scripts": "npm run test:api:path -- tests/integration/scripts",

api/sample.env

@@ -155,6 +155,25 @@ PGBOSS_CONNECTION_POOL_MAX_SIZE=
 # sample: START_JOB_IN_WEB_PROCESS=true
 START_JOB_IN_WEB_PROCESS=
 
+# URL of the PostgreSQL database used for storing datamart
+#
+# If not present, the application will not be able to responds to requests involving
+# data from datamart (i.e. cold data)
+#
+# presence: optional
+# type: Url
+# default: none
+DATAMART_DATABASE_URL=postgresql://postgres@localhost/datamart
+
+# URL of the PostgreSQL database used for API local testing.
+#
+# If not present, the tests will fail.
+#
+# presence: required
+# type: Url
+# default: none
+TEST_DATAMART_DATABASE_URL=postgresql://postgres@localhost/datamart_test
+
 # ========
 # EMAILING
 # ========

api/scripts/datamart/create-datamart.js

@@ -0,0 +1,30 @@
+import 'dotenv/config';
+
+import { PGSQL_DUPLICATE_DATABASE_ERROR } from '../../db/pgsql-errors.js';
+import { logger } from '../../src/shared/infrastructure/utils/logger.js';
+import { PgClient } from '../PgClient.js';
+
+const dbUrl =
+  process.env.NODE_ENV === 'test' ? process.env.TEST_DATAMART_DATABASE_URL : process.env.DATAMART_DATABASE_URL;
+
+const url = new URL(dbUrl);
+
+const DB_TO_CREATE_NAME = url.pathname.slice(1);
+
+url.pathname = '/postgres';
+
+PgClient.getClient(url.href).then(async (client) => {
+  try {
+    await client.query_and_log(`CREATE DATABASE ${DB_TO_CREATE_NAME};`);
+    logger.info('Database created');
+    await client.end();
+  } catch (error) {
+    if (error.code === PGSQL_DUPLICATE_DATABASE_ERROR) {
+      logger.info(`Database ${DB_TO_CREATE_NAME} already created`);
+    } else {
+      logger.error(`Database creation failed: ${error.detail}`);
+    }
+  } finally {
+    await client.end();
+  }
+});