From 755d8fed322ac0b40f7e74b8adb25482e3281647 Mon Sep 17 00:00:00 2001 From: Oriol Raventos Date: Tue, 20 Feb 2024 15:15:39 +0100 Subject: [PATCH] refactor(blank): remove nlp folder and fix dependencies --- blank/babel.config.js | 2 - blank/package.json | 63 ++++++++--------- blank/src/nlp/data/en/.gitkeep | 0 .../intent-classification/models/en/.gitkeep | 0 .../nlp/tasks/intent-classification/train.ts | 68 ------------------- blank/webpack.config.js | 2 - 6 files changed, 30 insertions(+), 105 deletions(-) delete mode 100644 blank/src/nlp/data/en/.gitkeep delete mode 100644 blank/src/nlp/tasks/intent-classification/models/en/.gitkeep delete mode 100644 blank/src/nlp/tasks/intent-classification/train.ts diff --git a/blank/babel.config.js b/blank/babel.config.js index 09ae315..55fa46e 100644 --- a/blank/babel.config.js +++ b/blank/babel.config.js @@ -23,8 +23,6 @@ module.exports = { ], ], plugins: [ - require('@babel/plugin-proposal-object-rest-spread'), - require('@babel/plugin-proposal-class-properties'), require('@babel/plugin-transform-runtime'), ], } diff --git a/blank/package.json b/blank/package.json index 4919207..f5b39aa 100644 --- a/blank/package.json +++ b/blank/package.json @@ -3,10 +3,9 @@ "version": "1.0.0", "scripts": { "build": "webpack --env target=all --mode=production", - "start": "webpack serve --env target=dev --mode=development", - "test": "jest", - "train:ner": "ts-node src/nlp/tasks/ner/train.ts", - "train:intent-classification": "ts-node src/nlp/tasks/intent-classification/train.ts" + "start": "webpack-dev-server --env target=dev --mode=development", + "deploy": "botonic deploy -c build", + "test": "jest" }, "jest": { "rootDir": "tests", @@ -20,43 +19,41 @@ }, "dependencies": { "@babel/runtime": "^7.12.5", - "@botonic/react": "~0.21.0" + "@botonic/react": "0.25.0-alpha.4" }, "devDependencies": { - "@babel/core": "^7.12.10", - "@babel/plugin-proposal-class-properties": "^7.12.1", - "@babel/plugin-transform-runtime": "^7.12.10", - "@babel/preset-env": "^7.12.11", - "@babel/preset-react": "^7.12.10", - "@hot-loader/react-dom": "^17.0.1", - "analytics-node": "^3.4.0-beta.3", - "babel-jest": "^26.6.3", - "babel-loader": "^8.2.2", - "chokidar": "^3.4.3", + "@babel/core": "^7.23.9", + "@babel/plugin-transform-runtime": "^7.23.9", + "@babel/preset-env": "^7.23.9", + "@babel/preset-react": "^7.23.3", + "@hot-loader/react-dom": "^16.14.0", + "analytics-node": "^3.5.0", + "babel-jest": "^29.7.0", + "babel-loader": "^8.3.0", + "chokidar": "^3.6.0", "clean-webpack-plugin": "^3.0.0", "copy-webpack-plugin": "^7.0.0", - "css-loader": "^5.0.1", + "css-loader": "^5.2.7", "file-loader": "^6.2.0", - "html-webpack-plugin": "^5.0.0-alpha.17", - "imagemin-gifsicle": "^6.0.0", + "html-webpack-plugin": "^5.6.0", + "imagemin-gifsicle": "^6.0.1", "imagemin-jpegtran": "^6.0.0", - "imagemin-optipng": "^7.0.0", - "imagemin-svgo": "^7.0.0", - "imagemin-webpack": "^5.0.0", - "jest": "^26.6.3", - "node-sass": "^8.0.0", + "imagemin-optipng": "^7.1.0", + "imagemin-svgo": "^7.1.0", + "imagemin-webpack": "^5.1.1", + "jest": "^29.7.0", "null-loader": "^4.0.1", "process": "^0.11.10", - "react-hot-loader": "^4.13.0", - "sass": "^1.30.0", - "sass-loader": "^10.1.0", + "react-hot-loader": "4.12.21", + "sass": "^1.71.0", + "sass-loader": "^10.5.2", "style-loader": "^2.0.0", - "terser": "^5.5.1", - "terser-webpack-plugin": "^5.0.3", - "ts-node": "^10.4.0", - "typescript": "^4.4.4", - "webpack": "^5.10.3", - "webpack-cli": "^4.2.0", - "webpack-dev-server": "4.13.3" + "terser": "^5.27.2", + "terser-webpack-plugin": "^5.3.10", + "ts-node": "^10.9.2", + "typescript": "^4.9.5", + "webpack": "^5.90.3", + "webpack-cli": "^5.1.4", + "webpack-dev-server": "5.0.2" } } diff --git a/blank/src/nlp/data/en/.gitkeep b/blank/src/nlp/data/en/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/blank/src/nlp/tasks/intent-classification/models/en/.gitkeep b/blank/src/nlp/tasks/intent-classification/models/en/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/blank/src/nlp/tasks/intent-classification/train.ts b/blank/src/nlp/tasks/intent-classification/train.ts deleted file mode 100644 index ff9693c..0000000 --- a/blank/src/nlp/tasks/intent-classification/train.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { - BotonicIntentClassifier, - DatabaseStorage, - Dataset, - INTENT_CLASSIFIER_TEMPLATE, - Preprocessor, -} from '@botonic/nlp' // eslint-disable-line node/no-missing-import, import/no-unresolved -import { join } from 'path' - -const LOCALE = 'en' - -const DATASET_DIR_PATH = join(process.cwd(), 'src', 'nlp', 'data', LOCALE) -const MODEL_DIR_PATH = join( - process.cwd(), - 'src', - 'nlp', - 'tasks', - 'intent-classification', - 'models' -) - -const MAX_SEQUENCE_LENGTH = 12 -const EMBEDDINGS_DIMENSION = 50 -const EMBEDDINGS_TYPE = 'glove' -const EPOCHS = 8 -const BATCH_SIZE = 8 - -const dataset = Dataset.load(DATASET_DIR_PATH) - -console.log(`Dataset size: ${dataset.length}`) - -const { trainSet, testSet } = dataset.split() -console.log(`Train set size: ${trainSet.length}`) -console.log(`Test set size: ${testSet.length}`) - -const preprocessor = new Preprocessor(LOCALE, MAX_SEQUENCE_LENGTH) - -const vocabulary = trainSet.extractVocabulary(preprocessor) - -const trainModel = async () => { - const classifier = new BotonicIntentClassifier( - { - locale: LOCALE, - maxLength: MAX_SEQUENCE_LENGTH, - intents: dataset.intents, - vocabulary, - }, - preprocessor - ) - - const model = await classifier.createModel( - INTENT_CLASSIFIER_TEMPLATE.SIMPLE_NN, - await DatabaseStorage.with(LOCALE, EMBEDDINGS_TYPE, EMBEDDINGS_DIMENSION), - { units: 128, dropout: 0.6 } - ) - - classifier.setModel(model) - - await classifier.train(trainSet, EPOCHS, BATCH_SIZE) - - const { accuracy, loss } = await classifier.evaluate(testSet) - console.log(`Test Accuracy: ${accuracy}`) - console.log(`Test loss: ${loss}`) - - await classifier.saveModel(MODEL_DIR_PATH) -} - -trainModel() diff --git a/blank/webpack.config.js b/blank/webpack.config.js index 4b545fa..06d7d87 100644 --- a/blank/webpack.config.js +++ b/blank/webpack.config.js @@ -121,8 +121,6 @@ const babelLoaderConfig = { ], ], plugins: [ - '@babel/plugin-proposal-object-rest-spread', - '@babel/plugin-proposal-class-properties', '@babel/plugin-transform-runtime', ], },