From 129c216e28bc639cdfea748912516047cc1a5c1a Mon Sep 17 00:00:00 2001 From: Oriol Raventos Date: Thu, 22 Feb 2024 13:01:59 +0100 Subject: [PATCH] refactor(intent): update intent using node v20 --- intent/babel.config.js | 5 +- intent/jest.config.js | 18 +++ intent/package.json | 89 ++++++------- intent/src/nlp/data/en/.gitkeep | 0 .../intent-classification/models/en/.gitkeep | 0 .../nlp/tasks/intent-classification/train.ts | 68 ---------- intent/src/plugins.js | 121 ++++++++++++++++-- intent/tests/app.test.js | 15 +-- intent/webpack.config.js | 54 ++------ 9 files changed, 190 insertions(+), 180 deletions(-) create mode 100644 intent/jest.config.js delete mode 100644 intent/src/nlp/data/en/.gitkeep delete mode 100644 intent/src/nlp/tasks/intent-classification/models/en/.gitkeep delete mode 100644 intent/src/nlp/tasks/intent-classification/train.ts diff --git a/intent/babel.config.js b/intent/babel.config.js index 7325b98..ce0c91d 100644 --- a/intent/babel.config.js +++ b/intent/babel.config.js @@ -23,8 +23,7 @@ module.exports = { ], ], plugins: [ - '@babel/plugin-proposal-object-rest-spread', - '@babel/plugin-proposal-class-properties', - '@babel/plugin-transform-runtime', + require('@babel/plugin-transform-modules-commonjs'), + require('@babel/plugin-transform-runtime'), ], } diff --git a/intent/jest.config.js b/intent/jest.config.js new file mode 100644 index 0000000..d2514ba --- /dev/null +++ b/intent/jest.config.js @@ -0,0 +1,18 @@ +const path = require('path') + +module.exports = { + rootDir: "tests", + transform: { + "^.+\\.jsx?$": [ + "babel-jest", + { "configFile": path.resolve(__dirname, "babel.config.js") }, + ], + }, + transformIgnorePatterns: [ + "/node_modules/(?!@botonic).+\\.(js|jsx|ts|tsx)$" + ], + moduleNameMapper: { + "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "/__mocks__/fileMock.js", + "\\.(scss|css|less)$": "/__mocks__/styleMock.js" + } +} \ No newline at end of file diff --git a/intent/package.json b/intent/package.json index 48f62f1..54a248f 100644 --- a/intent/package.json +++ b/intent/package.json @@ -3,61 +3,52 @@ "version": "1.0.0", "scripts": { "build": "webpack --env target=all --mode=production", - "start": "webpack serve --env target=dev --mode=development", - "test": "jest", - "train:ner": "ts-node src/nlp/tasks/ner/train.ts", - "train:intent-classification": "ts-node src/nlp/tasks/intent-classification/train.ts" - }, - "jest": { - "rootDir": "tests", - "transformIgnorePatterns": [ - "/node_modules/(?!@botonic).+\\.(js|jsx|ts|tsx|mjs)$" - ], - "moduleNameMapper": { - "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "/__mocks__/fileMock.js", - "\\.(scss|css|less)$": "/__mocks__/styleMock.js" - } + "start": "webpack-dev-server --env target=dev --mode=development", + "deploy": "botonic deploy -c build", + "test": "jest" }, "dependencies": { - "@botonic/plugin-dialogflow": "~0.21.0", - "@babel/runtime": "^7.12.5", - "@botonic/react": "~0.21.0" + "@botonic/plugin-dialogflow": "0.25.0-alpha.2", + "@babel/runtime": "^7.23.9", + "@botonic/react": "0.25.0-alpha.5" }, "devDependencies": { - "@babel/core": "^7.12.10", - "@babel/plugin-proposal-class-properties": "^7.12.1", - "@babel/plugin-transform-runtime": "^7.12.10", - "@babel/preset-env": "^7.12.11", - "@babel/preset-react": "^7.12.10", - "@hot-loader/react-dom": "^17.0.1", - "analytics-node": "^3.4.0-beta.3", - "babel-jest": "^26.6.3", - "babel-loader": "^8.2.2", - "chokidar": "^3.4.3", - "clean-webpack-plugin": "^3.0.0", - "copy-webpack-plugin": "^7.0.0", - "css-loader": "^5.0.1", + "@babel/core": "^7.23.9", + "@babel/plugin-transform-modules-commonjs": "^7.23.3", + "@babel/plugin-transform-runtime": "^7.23.9", + "@babel/preset-env": "^7.23.9", + "@babel/preset-react": "^7.23.3", + "@hot-loader/react-dom": "^16.14.0", + "analytics-node": "^3.5.0", + "babel-jest": "^29.7.0", + "babel-loader": "^9.1.3", + "chokidar": "^3.6.0", + "clean-webpack-plugin": "^4.0.0", + "css-loader": "^6.10.0", "file-loader": "^6.2.0", - "html-webpack-plugin": "^5.0.0-alpha.17", - "imagemin-gifsicle": "^6.0.0", - "imagemin-jpegtran": "^6.0.0", - "imagemin-optipng": "^7.0.0", - "imagemin-svgo": "^7.0.0", - "imagemin-webpack": "^5.0.0", - "jest": "^26.6.3", - "node-sass": "^8.0.0", + "html-webpack-plugin": "^5.6.0", + "image-minimizer-webpack-plugin": "^4.0.0", + "imagemin": "^8.0.1", + "imagemin-gifsicle": "^7.0.0", + "imagemin-jpegtran": "^7.0.0", + "imagemin-optipng": "^8.0.0", + "imagemin-svgo": "^10.0.1", + "jest": "^29.7.0", "null-loader": "^4.0.1", "process": "^0.11.10", - "react-hot-loader": "^4.13.0", - "sass": "^1.30.0", - "sass-loader": "^10.1.0", - "style-loader": "^2.0.0", - "terser": "^5.5.1", - "terser-webpack-plugin": "^5.0.3", - "ts-node": "^10.4.0", - "typescript": "^4.4.4", - "webpack": "^5.10.3", - "webpack-cli": "^4.2.0", - "webpack-dev-server": "4.13.3" + "react-hot-loader": "4.12.21", + "sass": "^1.71.1", + "sass-loader": "^14.1.1", + "style-loader": "^3.3.4", + "svgo": "^3.2.0", + "terser": "^5.27.2", + "terser-webpack-plugin": "^5.3.10", + "webpack": "^5.90.3", + "webpack-cli": "^5.1.4", + "webpack-dev-server": "5.0.2" + }, + "engines": { + "node": ">=20.0.0" } } + diff --git a/intent/src/nlp/data/en/.gitkeep b/intent/src/nlp/data/en/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/intent/src/nlp/tasks/intent-classification/models/en/.gitkeep b/intent/src/nlp/tasks/intent-classification/models/en/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/intent/src/nlp/tasks/intent-classification/train.ts b/intent/src/nlp/tasks/intent-classification/train.ts deleted file mode 100644 index ff9693c..0000000 --- a/intent/src/nlp/tasks/intent-classification/train.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { - BotonicIntentClassifier, - DatabaseStorage, - Dataset, - INTENT_CLASSIFIER_TEMPLATE, - Preprocessor, -} from '@botonic/nlp' // eslint-disable-line node/no-missing-import, import/no-unresolved -import { join } from 'path' - -const LOCALE = 'en' - -const DATASET_DIR_PATH = join(process.cwd(), 'src', 'nlp', 'data', LOCALE) -const MODEL_DIR_PATH = join( - process.cwd(), - 'src', - 'nlp', - 'tasks', - 'intent-classification', - 'models' -) - -const MAX_SEQUENCE_LENGTH = 12 -const EMBEDDINGS_DIMENSION = 50 -const EMBEDDINGS_TYPE = 'glove' -const EPOCHS = 8 -const BATCH_SIZE = 8 - -const dataset = Dataset.load(DATASET_DIR_PATH) - -console.log(`Dataset size: ${dataset.length}`) - -const { trainSet, testSet } = dataset.split() -console.log(`Train set size: ${trainSet.length}`) -console.log(`Test set size: ${testSet.length}`) - -const preprocessor = new Preprocessor(LOCALE, MAX_SEQUENCE_LENGTH) - -const vocabulary = trainSet.extractVocabulary(preprocessor) - -const trainModel = async () => { - const classifier = new BotonicIntentClassifier( - { - locale: LOCALE, - maxLength: MAX_SEQUENCE_LENGTH, - intents: dataset.intents, - vocabulary, - }, - preprocessor - ) - - const model = await classifier.createModel( - INTENT_CLASSIFIER_TEMPLATE.SIMPLE_NN, - await DatabaseStorage.with(LOCALE, EMBEDDINGS_TYPE, EMBEDDINGS_DIMENSION), - { units: 128, dropout: 0.6 } - ) - - classifier.setModel(model) - - await classifier.train(trainSet, EPOCHS, BATCH_SIZE) - - const { accuracy, loss } = await classifier.evaluate(testSet) - console.log(`Test Accuracy: ${accuracy}`) - console.log(`Test loss: ${loss}`) - - await classifier.saveModel(MODEL_DIR_PATH) -} - -trainModel() diff --git a/intent/src/plugins.js b/intent/src/plugins.js index cec3a7d..92272ef 100644 --- a/intent/src/plugins.js +++ b/intent/src/plugins.js @@ -1,20 +1,119 @@ +import * as pluginDialogflow from '@botonic/plugin-dialogflow' + export const plugins = [ { id: 'dialogflow', - resolve: require('@botonic/plugin-dialogflow'), + resolve: pluginDialogflow, // Copy-past here the generated JSON: https://dialogflow.com/docs/reference/v2-auth-setup options: { credentials: { - type: 'service_account', - project_id: 'YOUR_PROJECT_ID', - private_key_id: 'YOUR_PRIVATE_KEY_ID', - private_key: 'YOUR_PRIVATE_KEY', - client_email: 'YOUR_CLIENT_EMAIL', - client_id: 'CLIENT_ID', - auth_uri: 'AUTH_URI', - token_uri: 'TOKEN_URI', - auth_provider_x509_cert_url: 'AUT_PROVIDER_X509_CERT_URL', - client_x509_cert_url: 'CLIENT_X509_CERT_URL', + + "id": "f6ceb22d-0351-4fd6-a6b6-97d707d7e4b8", + "name": "Default Welcome Intent", + "auto": true, + "condition": "", + "conditionalFollowupEvents": [], + "conditionalResponses": [], + "context": [], + "contexts": [], + "endInteraction": false, + "events": [ + { + "name": "WELCOME" + } + ], + "fallbackIntent": false, + "liveAgentHandoff": false, + "parentId": null, + "followUpIntents": [], + "priority": 500000, + "responses": [ + { + "action": "input.welcome", + "affectedContexts": [], + "parameters": [], + "defaultResponsePlatforms": {}, + "messages": [ + { + "type": "message", + "condition": "", + "speech": [ + "¡Hola!", + "¡Hey!", + "¡Buenos días!" + ] + } + ], + "resetContexts": false + } + ], + "rootParentId": null, + "templates": [], + "userSays": [ + { + "isTemplate": false, + "data": [ + { + "text": "hola", + "userDefined": false + } + ], + "count": 0, + "id": "968a548d-bc43-4922-b0f0-b4d95e4479b4", + "updated": null + }, + { + "isTemplate": false, + "data": [ + { + "text": "hey", + "userDefined": false + } + ], + "count": 0, + "id": "8e6201b2-5115-4503-b2b9-b64a57701507", + "updated": null + }, + { + "isTemplate": false, + "data": [ + { + "text": "saludos", + "userDefined": false + } + ], + "count": 0, + "id": "5094725c-196e-4f52-9eed-254cf0dcfc40", + "updated": null + }, + { + "isTemplate": false, + "data": [ + { + "text": "hey ho", + "userDefined": false + } + ], + "count": 0, + "id": "df1cc2ca-9f39-45d9-b56c-9049c6577475", + "updated": null + }, + { + "isTemplate": false, + "data": [ + { + "text": "chao", + "userDefined": false + } + ], + "count": 0, + "id": "68adecfd-5fdd-497e-94b4-51456f10a520", + "updated": null + } + ], + "webhookForSlotFilling": false, + "webhookUsed": false + } }, }, diff --git a/intent/tests/app.test.js b/intent/tests/app.test.js index bf9299b..91ccbcb 100644 --- a/intent/tests/app.test.js +++ b/intent/tests/app.test.js @@ -10,14 +10,13 @@ import { routes } from '../src/routes' const app = new NodeApp({ routes, locales, ...config }) -const i = new BotonicInputTester(app) -const o = new BotonicOutputTester(app) +const input = new BotonicInputTester(app) +const output = new BotonicOutputTester(app) test('TEST: (404) NOT FOUND', async () => { - await expect(i.text('whatever')).resolves.toBe( - o.text( - // replace with 'Try typing "hello" to start the bot.' after configuring dialogflow - `Enter the generated JSON key for dialogflowV2 in plugins.js to test the bot.` - ) - ) + const response = await input.text('whatever') + expect(response).toBe(output.text( + // replace with 'Try typing "hello" to start the bot.' after configuring dialogflow + `Enter the generated JSON key for dialogflowV2 in plugins.js to test the bot.` + )) }) diff --git a/intent/webpack.config.js b/intent/webpack.config.js index 4b545fa..03250b1 100644 --- a/intent/webpack.config.js +++ b/intent/webpack.config.js @@ -1,34 +1,15 @@ const path = require('path') const webpack = require('webpack') -const CopyPlugin = require('copy-webpack-plugin') const TerserPlugin = require('terser-webpack-plugin') const HtmlWebpackPlugin = require('html-webpack-plugin') const { CleanWebpackPlugin } = require('clean-webpack-plugin') -const ImageminPlugin = require('imagemin-webpack') +const ImageMinimizerPlugin = require('image-minimizer-webpack-plugin') const ROOT = path.resolve(__dirname, 'src') -const NLP_DIRNAME = 'nlp' const ASSETS_DIRNAME = 'assets' -const MODELS_DIRNAME = 'models' -const TASKS_DIRNAME = 'tasks' -const INTENT_CLASSIFICATION_DIRNAME = 'intent-classification' const OUTPUT_PATH = path.resolve(__dirname, 'dist') const WEBVIEWS_PATH = path.resolve(OUTPUT_PATH, 'webviews') -const TASKS_PATH = path.join(ROOT, NLP_DIRNAME, TASKS_DIRNAME) - -const INTENT_CLASSIFICATION_MODELS_PATH = path.join( - NLP_DIRNAME, - TASKS_DIRNAME, - INTENT_CLASSIFICATION_DIRNAME, - MODELS_DIRNAME -) -const INTENTS_ASSETS_MODELS_PATH = path.join( - ASSETS_DIRNAME, - TASKS_DIRNAME, - INTENT_CLASSIFICATION_DIRNAME, - MODELS_DIRNAME -) const BOTONIC_PATH = path.resolve( __dirname, @@ -121,8 +102,6 @@ const babelLoaderConfig = { ], ], plugins: [ - '@babel/plugin-proposal-object-rest-spread', - '@babel/plugin-proposal-class-properties', '@babel/plugin-transform-runtime', ], }, @@ -165,16 +144,17 @@ const stylesLoaderConfig = { ], } -const imageminPlugin = new ImageminPlugin({ - bail: false, - cache: false, - imageminOptions: { - plugins: [ - ['imagemin-gifsicle', { interlaced: true }], - ['imagemin-jpegtran', { progressive: true }], - ['imagemin-optipng', { optimizationLevel: 5 }], - ['imagemin-svgo', { removeViewBox: true }], - ], +const imageminPlugin = new ImageMinimizerPlugin({ + minimizer: { + implementation: ImageMinimizerPlugin.imageminMinify, + options: { + plugins: [ + "imagemin-gifsicle", + "imagemin-jpegtran", + "imagemin-optipng", + "imagemin-svgo", + ], + }, }, }) @@ -200,7 +180,7 @@ function botonicDevConfig(mode) { }, resolve: resolveConfig, devServer: { - static: [OUTPUT_PATH, TASKS_PATH], + static: [OUTPUT_PATH], liveReload: true, historyApiFallback: true, hot: true, @@ -333,14 +313,6 @@ function botonicNodeConfig(mode) { IS_NODE: true, HUBTYPE_API_URL: JSON.stringify(process.env.HUBTYPE_API_URL), }), - new CopyPlugin({ - patterns: [ - { - from: INTENT_CLASSIFICATION_MODELS_PATH, - to: INTENTS_ASSETS_MODELS_PATH, - }, - ], - }), ], } }