Copy contents and update folder structure using GitHub Actions

DataMapper/v1/hbs/backoffice/hbs/hbs/reflect_waiting_time.handlebars

@@ -0,0 +1,4 @@
+{
+  "durationInSeconds": "{{durationInSeconds}}",
+  "positionInUnassignedChats": "{{positionInUnassignedChats}}"
+}

DataMapper/v1/training/hbs/js/convert/jsonToYamlStories.js

@@ -19,7 +19,9 @@ router.post('/', multer().array('file'), async (req, res) => {
                         case !!step.intent:
                             formattedStep.intent = step.intent;
                             if (step.entities && step.entities.length > 0) {
-                                formattedStep.entities = step.entities;
+                                formattedStep.entities = step.entities.map(entity => ({
+                                    [entity]: ""
+                                }));
                             }
                             break;
                         case !!step.action:
@@ -49,7 +51,9 @@ router.post('/', multer().array('file'), async (req, res) => {
                         case !!step.intent:
                             formattedStep.intent = step.intent;
                             if (step.entities && step.entities.length > 0) {
-                                formattedStep.entities = step.entities;
+                                formattedStep.entities = step.entities.map(entity => ({
+                                    [entity]: ""
+                                }));
                             }
                             break;
                         case !!step.action:

DataMapper/v1/training/hbs/js/docker/botTrainedVersion.js

@@ -0,0 +1,21 @@
+import express from 'express';
+
+const router = express.Router();
+
+router.post('/', (req, res) => {
+    const { object, name } = req.body;
+
+    const serviceParameters = object.services['train-bot'].command;
+    const index = serviceParameters.indexOf('--fixed-model-name');
+
+    if(index !== -1 && index < serviceParameters.length -1) {
+        const fileName = serviceParameters[index + 1];
+        res.json(fileName.split(name)[1])
+    } else {
+        res.json('1_0');
+    }
+
+
+});
+
+export default router;

DataMapper/v1/training/hbs/js/docker/updateVersionForBot.js

@@ -0,0 +1,31 @@
+import express from 'express';
+
+const router = express.Router();
+
+router.post('/', (req, res) => {
+    const { object, servicesArray, newVersion } = req.body;
+    servicesArray.forEach((service) => {
+        if(service === 'train-bot') {
+            const selectedService = object.services[service];
+            selectedService.command = replaceNextElementInArray(selectedService.command,'--fixed-model-name',`${newVersion}`)
+        } else if(service === 'test-bot') {
+            const selectedService = object.services[service];
+            selectedService.command = replaceNextElementInArray(selectedService.command,'--out',`results/${newVersion}`)
+        } else if(service === 'test-bot-cv') {
+            const selectedService = object.services[service];
+            selectedService.command = replaceNextElementInArray(selectedService.command,'--out',`results/cross-${newVersion}`)
+        }
+    });
+    res.json(object);
+});
+
+const replaceNextElementInArray = (array, element, newInput) => {
+    const index = array.indexOf(element);
+
+    if(index !== -1 && index < array.length -1) {
+        array[index + 1] = newInput.toString();
+    }
+    return array;
+};
+
+export default router;

DataMapper/v1/training/hbs/js/file/deleteIntentFile.js

@@ -1,42 +1,42 @@
-import fs from 'fs';
+import fs from "fs";
 import express from "express";
 import path from "path";
 
 const router = express.Router();
 
 function customIsValidFilePath(file_path) {
-    const illegalCharacterPattern = /[\\?%*:|"<>.]/;
-    return illegalCharacterPattern.test(file_path);
+  const illegalCharacterPattern = /[\\?%*:|"<>.]/;
+  return illegalCharacterPattern.test(file_path);
 }
 
 function customBuildContentFilePath(file_path) {
-    return path.join(process.env.CONTENT_FOLDER || "", file_path);
+  return path.join(import.meta.env.CONTENT_FOLDER || "", file_path);
 }
 
-router.post('/', (req, res) => {
-    const filePath = req.body.file_path;
-    const deletePath = customBuildContentFilePath(filePath);
+router.post("/", (req, res) => {
+  const filePath = req.body.file_path;
+  const deletePath = customBuildContentFilePath(filePath);
 
-    if (!customIsValidFilePath(filePath)) {
-        return res.status(400).json({
-            error: true,
-            message: 'File path contains illegal characters',
-        });
+  if (!customIsValidFilePath(filePath)) {
+    return res.status(400).json({
+      error: true,
+      message: "File path contains illegal characters",
+    });
+  }
+
+  fs.unlink(deletePath, (err) => {
+    if (err) {
+      return res.status(500).json({
+        error: true,
+        message: "Unable to delete file",
+      });
     }
 
-    fs.unlink(deletePath, (err) => {
-        if (err) {
-            return res.status(500).json({
-                error: true,
-                message: 'Unable to delete file',
-            });
-        }
-
-        return res.status(200).json({
-            error: false,
-            message: 'File deleted successfully',
-        });
+    return res.status(200).json({
+      error: false,
+      message: "File deleted successfully",
     });
+  });
 });
 
 export default router;

DataMapper/v1/training/hbs/js/util/incrementDoubleDigitStringVersion.js

@@ -0,0 +1,14 @@
+import express from 'express';
+
+const router = express.Router();
+
+router.post('/', (req, res) => {
+    const { version } = req.body;
+    const splitVersion = version.split('_');
+    const majorV = splitVersion[0];
+    let minorV = parseInt(splitVersion[1]);
+    minorV += 1;
+    res.json(`${majorV}_${minorV}`);
+});
+
+export default router;

DataMapper/v1/training/hbs/js/util/removeServicesConnectedToIntent.js

@@ -0,0 +1,13 @@
+import express from 'express';
+
+const router = express.Router();
+
+router.post('/', (req, res) => {
+  const { connections, services } = req.body;
+  const usedServiceIds = new Set(connections.map(x => x.service));
+  const unusedServices = services.filter(x => !usedServiceIds.has(x.serviceId));
+
+  return res.status(200).send(unusedServices);
+});
+
+export default router;

DataMapper/v1/training/hbs/js/util/utils.js

@@ -26,7 +26,7 @@ export const mapSecretToJson = (secrets) => {
 };
 
 export const buildContentFilePath = (fileName) => {
-  return path.join(process.env.CONTENT_FOLDER || "data", fileName);
+  return path.join(import.meta.env.CONTENT_FOLDER || "data", fileName);
 };
 
 export const isValidFilename = (fileName) => {
@@ -42,7 +42,7 @@ export const isValidFilePath = (filePath) => {
 };
 
 export const getAllFiles = function (dirPath) {
-  const folder = path.join(process.env.CONTENT_FOLDER || "data", dirPath);
+  const folder = path.join(import.meta.env.CONTENT_FOLDER || "data", dirPath);
   return getAllFilesInsideFolder(folder);
 };
 

DataMapper/v1/training/hbs/js/validation/validateStoriesRules.js

@@ -12,17 +12,14 @@ function validateStepsForNoConsecutiveDuplicates(steps) {
         const currentStep = steps[i];
         const previousStep = steps[i - 1];
 
-        if (currentStep.entities && previousStep.entities && areConsecutive(currentStep, previousStep)) {
-            const currententities = currentstep.entities.map(entity => entity.entity);
-            const previousentities = previousstep.entities.map(entity => entity.entity);
-
-            if (hascommonelement(currententities, previousentities)) {
+        if (currentStep.intent && previousStep.intent) {
+            if (currentStep.intent === previousStep.intent) {
                 return false;
             }
         }
 
-        if (currentStep.intent && previousStep.intent && areConsecutive(currentStep, previousStep)) {
-            if (currentStep.intent === previousStep.intent) {
+        if (currentStep.entities && previousStep.entities) {
+            if (hasCommonElement(currentStep.entities, previousStep.entities)) {
                 return false;
             }
         }
@@ -32,15 +29,7 @@ function validateStepsForNoConsecutiveDuplicates(steps) {
 }
 
 function hasCommonElement(arr1, arr2) {
-    return arr1.some(element => arr2.includes(element));
-}
-
-function areConsecutive(currentStep, previousStep) {
-    if (currentStep.start !== undefined && previousStep.end !== undefined) {
-        return currentStep.start === previousStep.end + 1;
-    }
-
-    return false;
+    return arr1.some(element => arr2.some(item => item.entity === element.entity));
 }
 
 export default router;

DataMapper/v1/training/hbs/lib/helpers.js

@@ -8,6 +8,10 @@ Handlebars.registerHelper('eq', function(a, b) {
     return a == b;
 });
 
+Handlebars.registerHelper('jsonParse', function(obj) {
+    return JSON.parse(obj);
+});
+
 Handlebars.registerHelper('arrayIsNotEmpty', function(array) {
     return !(!Array.isArray(array) || !array.length);
 });
@@ -62,6 +66,12 @@ Handlebars.registerHelper('isInModel', function(intentTitle, intents) {
     return Array.isArray(inModelIntents) ? inModelIntents.includes(intentTitle) : false;
 });
 
+Handlebars.registerHelper('findConnectedServiceId', function(intentTitle, intents) {
+  const name = intentTitle.replace('_', ' ');
+  const service = intents?.connections.find(x => x.intent === name);
+  return service?.service ?? "";
+});
+
 Handlebars.registerHelper('getCount', function(intentTitle, intents) {
     const intentCounts = intents.count;
     const intentCount = intentCounts?.find(intent => intent.key === intentTitle)?.examples_counts?.value;

DataMapper/v1/training/hbs/server.js

@@ -44,46 +44,60 @@ import validateStoriesRules from "./js/validation/validateStoriesRules.js";
 import replaceNextElementInArray from "./js/util/replaceNextElementInArray.js";
 import updateParametersByKey from "./js/docker/updateParametersByKey.js";
 import createExpressionFromDateDays from "./js/cron/createExpressionFromDateDays.js";
+import incrementDoubleDigitStringVersion from "./js/util/incrementDoubleDigitStringVersion.js";
+import updateVersionForBot from "./js/docker/updateVersionForBot.js";
+import botTrainedVersion from "./js/docker/botTrainedVersion.js";
+import removeServicesConnectedToIntent from "./js/util/removeServicesConnectedToIntent.js";
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const { publicKey, privateKey } = crypto.generateKeyPairSync("rsa", {
   modulusLength: 2048,
 });
 
-const PORT = process.env.PORT || 3000;
+const PORT = import.meta.env.PORT || 3000;
 const app = express();
 const hbs = create({ helpers });
 
 app.use(express.json());
 app.use("/file-manager", files);
-app.use('/file/read', fileRead);
-app.use('/file/read-directory', fileReadDir);
-app.use('/file/write', fileWrite);
-app.use('/file/delete', fileDelete);
-app.use('/file/delete-intent', deleteIntentFile);
-app.use('/file/check', fileCheck);
-app.use('/merge', merge);
-app.use('/forms/detailed-information', formDetails);
-app.use('/merge/objects', mergeObjects);
-app.use('/replace/key-value-in-obj', replaceKeyValueObj);
-app.use('/merge/remove-key', mergeRemoveKey);
-app.use('/merge/remove-array-value', mergeRemoveArrayValue);
-app.use('/merge/replace-array-element', mergeReplaceArrayElement);
-app.use('/validate/array-elements-length', validate);
-app.use('/convert/yaml-to-json', yamlToJson)
-app.use('/convert/json-to-yaml', jsonToYaml)
-app.use('/convert/json-to-yaml-stories', jsonToYamlStories)
-app.use('/convert/csv-to-json', csvToJson)
-app.use('/convert/string/split', stringSplit)
-app.use('/convert/string/replace', stringReplace)
-app.use('/convert/string/toArray', stringToArray)
-app.use('/rules/remove-by-intent-name', removeRulesByIntentName);
-app.use('/array/replace-next-element', replaceNextElementInArray);
-app.use('/docker/update-parameter-by-key', updateParametersByKey);
-app.use('/cron/generate-expression-date-days', createExpressionFromDateDays);
-app.use('/domain/update-existing-response', domainUpdateExistingResponse)
-app.use('/util/objectListContainsId', objectListContainsId)
-app.use('/validate/validate-stories-rules', validateStoriesRules)
+app.use("/file/read", fileRead);
+app.use("/file/read-directory", fileReadDir);
+app.use("/file/write", fileWrite);
+app.use("/file/delete", fileDelete);
+app.use("/file/delete-intent", deleteIntentFile);
+app.use("/file/check", fileCheck);
+app.use("/merge", merge);
+app.use("/forms/detailed-information", formDetails);
+app.use("/merge/objects", mergeObjects);
+app.use("/replace/key-value-in-obj", replaceKeyValueObj);
+app.use("/merge/remove-key", mergeRemoveKey);
+app.use("/merge/remove-array-value", mergeRemoveArrayValue);
+app.use("/merge/replace-array-element", mergeReplaceArrayElement);
+app.use("/validate/array-elements-length", validate);
+app.use("/convert/yaml-to-json", yamlToJson);
+app.use("/convert/json-to-yaml", jsonToYaml);
+app.use("/convert/json-to-yaml-stories", jsonToYamlStories);
+app.use("/convert/csv-to-json", csvToJson);
+app.use("/convert/string/split", stringSplit);
+app.use("/convert/string/replace", stringReplace);
+app.use("/convert/string/toArray", stringToArray);
+app.use("/rules/remove-by-intent-name", removeRulesByIntentName);
+app.use("/array/replace-next-element", replaceNextElementInArray);
+app.use("/docker/update-parameter-by-key", updateParametersByKey);
+app.use("/docker/update-version-for-bot", updateVersionForBot);
+app.use("/docker/bot-trained-version", botTrainedVersion);
+app.use("/cron/generate-expression-date-days", createExpressionFromDateDays);
+app.use("/domain/update-existing-response", domainUpdateExistingResponse);
+app.use("/util/objectListContainsId", objectListContainsId);
+app.use(
+  "/util/increase-double-digit-version",
+  incrementDoubleDigitStringVersion
+);
+app.use("/validate/validate-stories-rules", validateStoriesRules);
+app.use(
+  "/filter_out_services_connected_to_intent",
+  removeServicesConnectedToIntent
+);
 app.use(express.urlencoded({ extended: true }));
 
 app.use(
@@ -102,7 +116,7 @@ app.use(
 );
 
 app.engine("handlebars", hbs.engine);
-app.use("/dmapper", mapper)
+app.use("/dmapper", mapper);
 
 app.set("view engine", "handlebars");
 app.set("views", "./views");

DataMapper/v1/training/hbs/views/training/get_intents_full.hbs

@@ -9,6 +9,7 @@
     {{/each}}
     ],
     "inmodel": {{isInModel title ../intents}},
+    "serviceId": "{{findConnectedServiceId title ../intents}}",
     "count": {{getCount title ../intents}}
     }{{#unless @last}},{{/unless}}
 {{/each}}

DataMapper/v1/training/hbs/views/training/get_models.hbs

@@ -0,0 +1,10 @@
+[
+    {{#each records}}
+        {
+            "id": "{{ this.fileName }}",
+            "name": "{{ this.fileName }}",
+            "lastTrained": "{{ this.trainedDate }}",
+            "state": "{{ this.state }}"
+        }{{#unless @last}},{{/unless}}
+    {{/each}}
+]