feat: use @socialgouv/codes-naf fix #5 (#23)

* feat: use @socialgouv/codes-naf fix #5 * fix(index): ensure docs are created before unaliasing
SocialGouv · Apr 13, 2021 · 2c4a498 · 2c4a498
1 parent f8ac223
commit 2c4a498
Show file tree

Hide file tree

Showing 10 changed files with 40 additions and 878 deletions.
diff --git a/.k8s/__tests__/__snapshots__/indexing-dev.ts.snap b/.k8s/__tests__/__snapshots__/indexing-dev.ts.snap
@@ -261,7 +261,7 @@ metadata:
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: update-index
+  name: update-index-ccff889
   annotations:
     app.gitlab.com/app: socialgouv-recherche-entreprises
     app.gitlab.com/env: recherche-entreprises-209-gitlab3-dev2

diff --git a/.k8s/__tests__/__snapshots__/indexing-preprod.ts.snap b/.k8s/__tests__/__snapshots__/indexing-preprod.ts.snap
@@ -261,7 +261,7 @@ metadata:
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: update-index
+  name: update-index-ccff889
   annotations:
     app.gitlab.com/app: socialgouv-recherche-entreprises
     app.gitlab.com/env: recherche-entreprises-209-gitlab3-dev2

diff --git a/.k8s/__tests__/__snapshots__/indexing-prod.ts.snap b/.k8s/__tests__/__snapshots__/indexing-prod.ts.snap
@@ -259,7 +259,7 @@ metadata:
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: update-index
+  name: update-index-ccff889
   annotations:
     app.gitlab.com/app: socialgouv-recherche-entreprises
     app.gitlab.com/env: recherche-entreprises-209-gitlab3-dev2

diff --git a/.k8s/components/jobs/indexing.ts b/.k8s/components/jobs/indexing.ts
@@ -185,7 +185,7 @@ const configMap = new ConfigMap({
 // create the final job
 const job = new Job({
   metadata: {
-    name: "update-index",
+    name: `update-index-${process.env.CI_COMMIT_SHORT_SHA}`,
   },
   spec: {
     backoffLimit: 3,

diff --git a/index/package.json b/index/package.json
@@ -3,17 +3,19 @@
   "version": "1.0.0",
   "description": "",
   "main": "index.js",
+  "private": true,
   "scripts": {
     "start": "node --max_old_space_size=8192 -r ts-node/register src/index.ts",
     "build": "NODE_OPTIONS='--max-old-space-size=8192' ncc build src/index.ts -o dist",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "keywords": [],
   "author": "",
-  "license": "ISC",
+  "license": "Apache-2.0",
   "dependencies": {
     "@babel/runtime": "^7.12.13",
     "@elastic/elasticsearch": "^7.12.0",
+    "@socialgouv/codes-naf": "^1.0.5",
     "@socialgouv/kali-data": "^2.61.0",
     "fast-csv": "^4.3.6",
     "p-all": "^3.0.0",

diff --git a/index/src/elastic.ts b/index/src/elastic.ts
@@ -179,3 +179,8 @@ export const add = async (enterprises: Enterprise[], indexName: string) => {
     { concurrency: 5 }
   );
 };
+
+export const getDocsCount = async (indexName: string): Promise<number> => {
+  const stats = await esClient.indices.stats({index: indexName});
+  return stats.body._all.primaries.docs.count;
+}
diff --git a/index/src/index.ts b/index/src/index.ts
@@ -1,7 +1,13 @@
 import * as fs from "fs";
 import * as path from "path";
 import * as csv from "fast-csv";
-import { add, createIndex, updateAlias, deleteOldIndices } from "./elastic";
+import {
+  add,
+  createIndex,
+  updateAlias,
+  deleteOldIndices,
+  getDocsCount,
+} from "./elastic";
 import { Enterprise } from "./enterprise";
 
 const ASSEMBLY_FILE = process.env.ASSEMBLY_FILE || "../output/assembly.csv";
@@ -39,6 +45,15 @@ if (require.main === module) {
   // use elastic alias feature to prevent downtimes
   createIndex().then(async (indexName) =>
     insertEntreprises(indexName)
+      .then(async () => {
+        // ensure we have some data
+        const docsCount = await getDocsCount(indexName);
+        if (!docsCount) {
+          throw new Error(`No document created in index ${indexName}, skip aliasing`);
+        } else {
+          console.log(`Created ${docsCount} documents in index ${indexName}`);
+        }
+      })
       .then(() => updateAlias(indexName))
       .then(() => deleteOldIndices(indexName))
   );

diff --git a/index/src/naf.ts b/index/src/naf.ts
diff --git a/index/tsconfig.json b/index/tsconfig.json
@@ -6,7 +6,7 @@
       "lib": ["es2020", "dom"],
       "module": "commonjs",
       "target": "es2020",
-
+      "moduleResolution": "node",
       "strict": true,
       "esModuleInterop": true,
       "skipLibCheck": true,

diff --git a/index/yarn.lock b/index/yarn.lock
@@ -66,6 +66,11 @@
     lodash.isundefined "^3.0.1"
     lodash.uniq "^4.5.0"
 
+"@socialgouv/codes-naf@^1.0.5":
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/@socialgouv/codes-naf/-/codes-naf-1.0.5.tgz#fa64c00b6299d0d41e1e88a4422ca72cb688e08c"
+  integrity sha512-unWkbh6qcG/9B+Oa2LsES6+xFYRK+KMyfa+NNszaEE7JYxSAibRnICTEkRUjriufrRXREV0fiB3MR0ytk7uAMA==
+
 "@socialgouv/kali-data@^2.61.0":
   version "2.61.0"
   resolved "https://registry.yarnpkg.com/@socialgouv/kali-data/-/kali-data-2.61.0.tgz#8e83bca951c99a0cf000c5e8298d309b378d6bcc"