From 0a21c6b583257e8f2ad2f0b12821e40e9fb01315 Mon Sep 17 00:00:00 2001 From: Lars Liden Date: Fri, 25 Oct 2019 14:10:18 -0700 Subject: [PATCH 01/14] feat: cosmos log storage --- package-lock.json | 149 +++++++++++++++++++++++---- package.json | 1 + src/CosmosLogStorage.test.ts | 152 +++++++++++++++++++++++++++ src/CosmosLogStorage.ts | 194 +++++++++++++++++++++++++++++++++++ src/Memory/ILogStorage.ts | 39 +++++++ 5 files changed, 516 insertions(+), 19 deletions(-) create mode 100644 src/CosmosLogStorage.test.ts create mode 100644 src/CosmosLogStorage.ts create mode 100644 src/Memory/ILogStorage.ts diff --git a/package-lock.json b/package-lock.json index 8baee443..56b18338 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,43 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@azure/cosmos": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@azure/cosmos/-/cosmos-3.3.6.tgz", + "integrity": "sha512-HCPIa4lgBQnetH8VVwLJBsBipAyCt1iQRoojgtLpZBfu4pKvg2z2Vj3oyxxc5lYaab7VzLLSMynpWt/8Bixhmg==", + "requires": { + "@types/debug": "^4.1.4", + "debug": "^4.1.1", + "fast-json-stable-stringify": "^2.0.0", + "node-abort-controller": "^1.0.4", + "node-fetch": "^2.6.0", + "priorityqueuejs": "^1.0.0", + "semaphore": "^1.0.5", + "tslib": "^1.9.3", + "universal-user-agent": "^4.0.0", + "uuid": "^3.3.2" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "node-fetch": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" + }, + "uuid": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.3.tgz", + "integrity": "sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ==" + } + } + }, "@azure/ms-rest-js": { "version": "1.2.6", "resolved": "https://registry.npmjs.org/@azure/ms-rest-js/-/ms-rest-js-1.2.6.tgz", @@ -1437,6 +1474,11 @@ "@types/express": "*" } }, + "@types/debug": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.5.tgz", + "integrity": "sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ==" + }, "@types/events": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@types/events/-/events-1.2.0.tgz", @@ -3852,7 +3894,6 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==", - "dev": true, "requires": { "once": "^1.4.0" } @@ -3976,6 +4017,42 @@ "integrity": "sha512-9sLAvzhI5nc8TpuQUh4ahMdCrWT00wPWz7j47/emR5+2qEfoZP5zzUXvx+vdx+H6ohhnsYC31iX04QLYJK8zTg==", "dev": true }, + "execa": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz", + "integrity": "sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==", + "requires": { + "cross-spawn": "^6.0.0", + "get-stream": "^4.0.0", + "is-stream": "^1.1.0", + "npm-run-path": "^2.0.0", + "p-finally": "^1.0.0", + "signal-exit": "^3.0.0", + "strip-eof": "^1.0.0" + }, + "dependencies": { + "cross-spawn": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", + "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", + "requires": { + "nice-try": "^1.0.4", + "path-key": "^2.0.1", + "semver": "^5.5.0", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + } + }, + "get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "requires": { + "pump": "^3.0.0" + } + } + } + }, "exit": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", @@ -6325,8 +6402,7 @@ "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", - "dev": true + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" }, "isobject": { "version": "3.0.1", @@ -8488,6 +8564,11 @@ "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.6.tgz", "integrity": "sha512-swStvEyDqQ85MGpABCMBclZcLI/pBIlu8FFDtmX197+oEgKloJ67QnB+Tidh0340HmLMs39c4GrkPY3cmkXp6Q==" }, + "macos-release": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.3.0.tgz", + "integrity": "sha512-OHhSbtcviqMPt7yfw5ef5aghS2jzFVKEFyCJndQt2YpSQ9qRVSEv2axSJI1paVThEu+FFGs584h/1YhxjVqajA==" + }, "make-dir": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", @@ -8785,8 +8866,7 @@ "nice-try": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", - "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", - "dev": true + "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==" }, "nock": { "version": "10.0.6", @@ -8814,6 +8894,11 @@ } } }, + "node-abort-controller": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-1.0.4.tgz", + "integrity": "sha512-7cNtLKTAg0LrW3ViS2C7UfIzbL3rZd8L0++5MidbKqQVJ8yrH6+1VRSHl33P0ZjBTbOJd37d9EYekvHyKkB0QQ==" + }, "node-fetch": { "version": "1.7.3", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", @@ -8917,7 +9002,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=", - "dev": true, "requires": { "path-key": "^2.0.0" } @@ -9120,6 +9204,15 @@ } } }, + "os-name": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/os-name/-/os-name-3.1.0.tgz", + "integrity": "sha512-h8L+8aNjNcMpo/mAIBPn5PXCM16iyPGjHNWo6U1YO8sJTMHtEtyczI6QJnLoplswm6goopQkqc7OAnjhWcugVg==", + "requires": { + "macos-release": "^2.2.0", + "windows-release": "^3.1.0" + } + }, "os-tmpdir": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", @@ -9144,8 +9237,7 @@ "p-finally": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=", - "dev": true + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" }, "p-is-promise": { "version": "2.0.0", @@ -9270,8 +9362,7 @@ "path-key": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", - "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=", - "dev": true + "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=" }, "path-parse": { "version": "1.0.5", @@ -9449,6 +9540,11 @@ } } }, + "priorityqueuejs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/priorityqueuejs/-/priorityqueuejs-1.0.0.tgz", + "integrity": "sha1-LuTyPCVgkT4IwHzlzN1t498sWvg=" + }, "process-nextick-args": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", @@ -9516,7 +9612,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", - "dev": true, "requires": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -10283,6 +10378,11 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, + "semaphore": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/semaphore/-/semaphore-1.1.0.tgz", + "integrity": "sha512-O4OZEaNtkMd/K0i6js9SL+gqy0ZCBMgUvlSqHKi4IBdjhe7wB8pwztUk1BbZ1fmrvpwFrPbHzqd2w5pTcJH6LA==" + }, "semver": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/semver/-/semver-5.5.0.tgz", @@ -10383,7 +10483,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", - "dev": true, "requires": { "shebang-regex": "^1.0.0" } @@ -10391,8 +10490,7 @@ "shebang-regex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", - "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=", - "dev": true + "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=" }, "shelljs": { "version": "0.7.6", @@ -10414,8 +10512,7 @@ "signal-exit": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", - "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", - "dev": true + "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" }, "sisteransi": { "version": "1.0.0", @@ -10754,8 +10851,7 @@ "strip-eof": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", - "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", - "dev": true + "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=" }, "strip-indent": { "version": "2.0.0", @@ -11393,6 +11489,14 @@ "crypto-random-string": "^1.0.0" } }, + "universal-user-agent": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-4.0.0.tgz", + "integrity": "sha512-eM8knLpev67iBDizr/YtqkJsF3GK8gzDc6st/WKzrTuPtcsOKW/0IdL4cnMBsU69pOx0otavLWBDGTwg+dB0aA==", + "requires": { + "os-name": "^3.1.0" + } + }, "universalify": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", @@ -11669,7 +11773,6 @@ "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", - "dev": true, "requires": { "isexe": "^2.0.0" } @@ -11722,6 +11825,14 @@ } } }, + "windows-release": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/windows-release/-/windows-release-3.2.0.tgz", + "integrity": "sha512-QTlz2hKLrdqukrsapKsINzqMgOUpQW268eJ0OaOpJN32h272waxR9fkB9VoWRtK7uKHG5EHJcTXQBD8XZVJkFA==", + "requires": { + "execa": "^1.0.0" + } + }, "word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", diff --git a/package.json b/package.json index 949c5165..5d09be7c 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "author": "Microsoft Conversation Learner Team", "license": "MIT", "dependencies": { + "@azure/cosmos": "^3.3.6", "@conversationlearner/models": "0.213.0", "@conversationlearner/ui": "0.399.0", "@types/supertest": "2.0.4", diff --git a/src/CosmosLogStorage.test.ts b/src/CosmosLogStorage.test.ts new file mode 100644 index 00000000..fd7c235f --- /dev/null +++ b/src/CosmosLogStorage.test.ts @@ -0,0 +1,152 @@ +/** + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. + */ +//LARSimport * as utils from './Utils' +import * as CLM from '@conversationlearner/models'; +import { CosmosLogStorage } from './CosmosLogStorage'; + +const options = { + endpoint: "", + key: "" +} + +function makeLogDialog(logDialogId: string): CLM.LogDialog { + return { + ...CLM.makeLogDialog(), + logDialogId + } +} + +describe('CosmosLogStorage', () => { + + // Before each test clear the database + beforeEach(async () => { + const cls = await CosmosLogStorage.Get(options) + await cls.DeleteAll() + }) + + + test('CreateAsync', async () => { + + const cls = await CosmosLogStorage.Get(options) + + const appId = CLM.ModelUtils.generateGUID() + const logDialog1Id = CLM.ModelUtils.generateGUID() + const logDialog1 = makeLogDialog(logDialog1Id) + await cls.Add(appId, logDialog1) + + let queryResult = await cls.GetMany(appId, undefined) + expect(queryResult.logDialogs.length).toBe(1) + }) + + test('GetAsync', async () => { + + const cls = await CosmosLogStorage.Get(options) + + const appId = CLM.ModelUtils.generateGUID() + const logDialog1Id = CLM.ModelUtils.generateGUID() + const logDialog1 = makeLogDialog(logDialog1Id) + await cls.Add(appId, logDialog1) + + let logDialog = await cls.Get(appId, logDialog1Id) + expect(logDialog).not.toBe(undefined) + }) + + test('DeleteAsync', async () => { + + const cls = await CosmosLogStorage.Get(options) + + const appId = CLM.ModelUtils.generateGUID() + const logDialog1Id = CLM.ModelUtils.generateGUID() + const logDialog1 = makeLogDialog(logDialog1Id) + await cls.Add(appId, logDialog1) + + let queryResult = await cls.GetMany(appId, undefined) + expect(queryResult.logDialogs.length).toBe(1) + + await cls.Delete(appId, logDialog1.logDialogId) + + queryResult = await cls.GetMany(appId, undefined) + expect(queryResult.logDialogs.length).toBe(0) + + }) + + test('DeleteAll', async () => { + + const cls = await CosmosLogStorage.Get(options) + await cls.DeleteAll() + + let queryResult = await cls.GetMany() + expect(queryResult.logDialogs.length).toBe(0) + expect(queryResult.continuationToken).toBe(undefined) + }) + + + test('GetMany', async () => { + + const cls = await CosmosLogStorage.Get(options) + + // Create 10 items + for (let i = 0; i < 8; i = i + 1) { + const appId = CLM.ModelUtils.generateGUID() + const logDialogId = CLM.ModelUtils.generateGUID() + const logDialog = makeLogDialog(logDialogId) + await cls.Add(appId, logDialog) + } + + // Get the first 5 + let queryResult = await cls.GetMany(undefined, undefined, undefined, 5) + expect(queryResult.logDialogs.length).toBe(5) + expect(queryResult.continuationToken).not.toBe(undefined) + + // Get the next + queryResult = await cls.GetMany(undefined, undefined, queryResult.continuationToken) + expect(queryResult.logDialogs.length).toBe(3) + expect(queryResult.continuationToken).toBe(undefined) + }) + + test('AppendScorerStep', async () => { + + const cls = await CosmosLogStorage.Get(options) + + const appId = CLM.ModelUtils.generateGUID() + const logDialog1Id = CLM.ModelUtils.generateGUID() + const logDialog1 = makeLogDialog(logDialog1Id) + await cls.Add(appId, logDialog1) + + const scorerStep = CLM.makeLogScorerStep() + await cls.AppendScorerStep(appId, logDialog1.logDialogId, scorerStep) + + let logDialog = await cls.Get(appId, logDialog1Id) + expect(logDialog).not.toBe(undefined) + + if (logDialog) { + const lastRound = logDialog.rounds[logDialog.rounds.length - 1] + const lastScorerStep = lastRound.scorerSteps[lastRound.scorerSteps.length - 1] + expect(lastScorerStep).toEqual(scorerStep) + } + }) + + test('AppendExtractorStep', async () => { + + const cls = await CosmosLogStorage.Get(options) + + const appId = CLM.ModelUtils.generateGUID() + const logDialog1Id = CLM.ModelUtils.generateGUID() + const logDialog1 = makeLogDialog(logDialog1Id) + await cls.Add(appId, logDialog1) + + const extractorStep = CLM.makeLogExtractorStep() + await cls.AppendExtractorStep(appId, logDialog1.logDialogId, extractorStep) + + let logDialog = await cls.Get(appId, logDialog1Id) + expect(logDialog).not.toBe(undefined) + + if (logDialog) { + const lastRound = logDialog.rounds[logDialog.rounds.length - 1] + expect(lastRound.extractorStep).toEqual(extractorStep) + expect(lastRound.scorerSteps.length).toBe(0) + } + }) +}) diff --git a/src/CosmosLogStorage.ts b/src/CosmosLogStorage.ts new file mode 100644 index 00000000..e0cd8759 --- /dev/null +++ b/src/CosmosLogStorage.ts @@ -0,0 +1,194 @@ +/** + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. + */ +import * as Cosmos from '@azure/cosmos' +import { CLDebug } from './CLDebug' +import { ILogStorage, LogQueryResult } from './Memory/ILogStorage' +import * as CLM from '@conversationlearner/models' + +const DATABASE_ID = "LOG_DIALOGS" +const COLLECTION_ID = "LOG_DIALOGS" +const MAX_PAGE_SIZE = 100 +const PARTITION_KEY = { kind: 'Hash', paths: ['/appId'] } + +interface StoredLogDialog extends CLM.LogDialog { + // CosmosId + id: string, + appId: string +} + +export class CosmosLogStorage implements ILogStorage { + private client: Cosmos.CosmosClient + private database: Cosmos.Database | undefined + private container: Cosmos.Container | undefined + + /** + * + * @param options endpoint, key + * endpoint = Cosmos server (i.e. "https://your-account.documents.azure.com") + * key = Master key of the endpoint + */ + constructor(options: Cosmos.CosmosClientOptions) { + this.client = new Cosmos.CosmosClient(options) + } + + static async Get(options: Cosmos.CosmosClientOptions): Promise { + + const storage = new CosmosLogStorage(options) + + const dbResponse = await storage.client.databases.createIfNotExists({ id: DATABASE_ID }) + storage.database = dbResponse.database + + const coResponse = await storage.database.containers.createIfNotExists({ id: COLLECTION_ID, partitionKey: PARTITION_KEY }) + storage.container = coResponse.container + + return storage + } + + /** Add a log dialog to storage */ + public async Add(appId: string, logDialog: CLM.LogDialog) { + if (this.container) { + const storedLog = logDialog as StoredLogDialog + storedLog.id = this.GetDialogDocumentId(appId, logDialog.logDialogId) + storedLog.appId = appId + await this.container.items.create(storedLog) + } + } + + /** Append a scorer step to already existing log dialog */ + public async AppendScorerStep(appId: string, logDialogId: string, scorerStep: CLM.LogScorerStep): Promise { + if (!this.container) { + throw new Error("Cosmos Constainer Doesn't exist") + } + const { resource: logDialog } = await this.container.item(this.GetDialogDocumentId(appId, logDialogId), appId).read() + if (!logDialog) { + throw new Error(`Log Dialog does not exist App:${appId} Log:${logDialogId}`) + } + const lastRound = logDialog.rounds[logDialog.rounds.length - 1] + if (!lastRound || lastRound.scorerSteps.length === 0) { + throw new Error(`Log Dialogs has no Extractor Step Log:${logDialogId}`) + } + lastRound.scorerSteps.push(scorerStep) + await this.container.items.upsert(logDialog) + return logDialog + } + + /** Append an extractor step to already existing log dialog */ + public async AppendExtractorStep(appId: string, logDialogId: string, extractorStep: CLM.LogExtractorStep): Promise { + if (!this.container) { + throw new Error("Cosmos Constainer Doesn't exist") + } + const { resource: logDialog } = await this.container.item(this.GetDialogDocumentId(appId, logDialogId), appId).read() + if (!logDialog) { + throw new Error(`Log Dialog does not exist App:${appId} Log:${logDialogId}`) + } + const newRound: CLM.LogRound = { + extractorStep: extractorStep, + scorerSteps: [] + } + logDialog.rounds.push(newRound) + await this.container.items.upsert(logDialog) + return logDialog + } + + /** + * Get all log dialogs matching parameters + * @param appId Filer by appId if set + * @param packageId Filter by Package if set + * @param continuationToken Continuation token + * @param pageSize Number to retrieve (max 100) + */ + public async GetMany(appId?: string, packageId?: string, continuationToken?: string, pageSize = MAX_PAGE_SIZE): Promise { + + let querySpec: Cosmos.SqlQuerySpec + if (!appId && !packageId) { + querySpec = { + query: `SELECT * FROM c` + } + } + else if (appId) { + querySpec = { + query: `SELECT * FROM c WHERE c.appId = @appId`, + parameters: [ + { + name: "@appId", + value: appId + } + ] + } + } + else { + querySpec = { + query: `SELECT * FROM c WHERE c.packageId = @packageId`, + parameters: [ + { + name: "@packageId", + value: packageId! + } + ] + } + } + + const feedOptions: Cosmos.FeedOptions = { + maxItemCount: Math.min(pageSize, MAX_PAGE_SIZE), + continuation: continuationToken + } + + if (this.container) { + const feedResponse = await this.container.items.query(querySpec, feedOptions).fetchNext(); + return { + logDialogs: feedResponse.resources, + continuationToken: feedResponse.continuation + } + } + throw new Error("Contained undefined") + } + + /** Retrieve a log dialog from storage */ + public async Get(appId: string, logDialogId: string): Promise { + if (this.container) { + // this.container. + const { resource } = await this.container.item(this.GetDialogDocumentId(appId, logDialogId), appId).read() + return resource + } + return undefined + } + + /** Delete a log dialog in storage */ + public async Delete(appId: string, logDialogId: string): Promise { + if (this.container) { + await this.container.item(this.GetDialogDocumentId(appId, logDialogId), appId).delete() + CLDebug.Log(`Deleted ${logDialogId} / ${this.GetDialogDocumentId(appId, logDialogId)}`) + } + } + + /** Delete all log dialogs in storage */ + public async DeleteAll() { + if (!this.container) { + throw new Error("Continer is undefined") + } + const querySpec: Cosmos.SqlQuerySpec = { + query: `SELECT * FROM c` + } + + let continuationToken: string | undefined + do { + const feedOptions: Cosmos.FeedOptions = { + maxItemCount: 5, + continuation: continuationToken + } + const feedResponse = await this.container.items.query(querySpec, feedOptions).fetchNext() + for (const logDialog of feedResponse.resources) { + await this.container.item(logDialog.id, logDialog.appId).delete(logDialog) + } + continuationToken = feedResponse.continuation + } + while (continuationToken); + } + + /** Generate document id used in cosmos */ + private GetDialogDocumentId(appId: string, dialogId: string): string { + return `${appId}_${dialogId}`; + } +} diff --git a/src/Memory/ILogStorage.ts b/src/Memory/ILogStorage.ts new file mode 100644 index 00000000..9d574e8b --- /dev/null +++ b/src/Memory/ILogStorage.ts @@ -0,0 +1,39 @@ +/** + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. + */ +import * as CLM from '@conversationlearner/models' + +export interface LogQueryResult { + logDialogs: CLM.LogDialog[] + continuationToken: string | undefined +} + +export interface ILogStorage { + /** Add a log dialog to storage */ + Add(appId: string, logDialog: CLM.LogDialog): Promise + + /** Retrieve a log dialog from storage */ + Get(appId: string, logDialogId: string): Promise + + /** Delete a log dialog in storage */ + Delete(appId: string, lodDialogId: string): Promise + + /** + * Get all log dialogs matching parameters + * @param appId Filer by appId if set + * @param packageId Filter by Package if set + * @param continuationToken Continuation token + * @param pageSize Number to retrieve (max 100) + */ + GetMany(appId: string, packageId: string, continuationToken?: string, pageSize?: number): Promise + + /** Append an extractor step to already existing log dialog */ + AppendExtractorStep(appId: string, logDialogId: string, extractorStep: CLM.LogExtractorStep): Promise + + /** Append a scorer step to already existing log dialog */ + AppendScorerStep(appId: string, logDialogId: string, scorerStep: CLM.LogScorerStep): Promise + + /** Delete all log dialogs in storage */ + DeleteAll(): Promise +} From 0af588416498c7ef10225b94eee204fb35cb4c53 Mon Sep 17 00:00:00 2001 From: Lars Liden Date: Fri, 25 Oct 2019 14:10:56 -0700 Subject: [PATCH 02/14] fix: remove comment --- src/CosmosLogStorage.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/CosmosLogStorage.test.ts b/src/CosmosLogStorage.test.ts index fd7c235f..7dc930f1 100644 --- a/src/CosmosLogStorage.test.ts +++ b/src/CosmosLogStorage.test.ts @@ -2,7 +2,6 @@ * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. */ -//LARSimport * as utils from './Utils' import * as CLM from '@conversationlearner/models'; import { CosmosLogStorage } from './CosmosLogStorage'; From 5144b220edb917a981e838bbeaa68c43028a0164 Mon Sep 17 00:00:00 2001 From: Lars Liden Date: Mon, 4 Nov 2019 15:56:17 -0800 Subject: [PATCH 03/14] feat: log building --- src/CLClient.ts | 6 +++ src/CLRunner.ts | 87 ++++++++++++++++++++++++++++++++++---- src/ConversationLearner.ts | 5 ++- src/CosmosLogStorage.ts | 31 +++++++++----- src/Memory/CLState.ts | 8 +++- src/Memory/ILogStorage.ts | 7 ++- src/http/router.ts | 39 +++++++++++++---- src/index.ts | 8 +++- 8 files changed, 156 insertions(+), 35 deletions(-) diff --git a/src/CLClient.ts b/src/CLClient.ts index 13caf13e..99277d4b 100644 --- a/src/CLClient.ts +++ b/src/CLClient.ts @@ -237,6 +237,12 @@ export class CLClient { // Log Dialogs //============================================================================= + public GetLogDialogs(appId: string, packageIds: string[]): Promise { + const packages = packageIds.map(p => `package=${p}`).join("&") + const apiPath = `app/${appId}/logdialogs?includeDefinitions=false&${packages}` + return this.send('GET', this.MakeURL(apiPath)) + } + /** Runs entity extraction (prediction). */ public LogDialogExtract( appId: string, diff --git a/src/CLRunner.ts b/src/CLRunner.ts index 31c968c0..d4198d63 100644 --- a/src/CLRunner.ts +++ b/src/CLRunner.ts @@ -237,7 +237,7 @@ export class CLRunner { packageId: packageId, initialFilledEntities: [] } - await this.StartSessionAsync(state, BB.TurnContext.getConversationReference(activity), app.appId, SessionStartFlags.NONE, sessionCreateParams) + await this.CreateSessionAsync(state, BB.TurnContext.getConversationReference(activity), app.appId, SessionStartFlags.NONE, sessionCreateParams) } } } @@ -291,7 +291,7 @@ export class CLRunner { } } - public async StartSessionAsync(state: CLState, conversationRef: Partial | null, appId: string, sessionStartFlags: SessionStartFlags, createParams: CLM.SessionCreateParams | CLM.CreateTeachParams): Promise { + public async CreateSessionAsync(state: CLState, conversationRef: Partial | null, appId: string, sessionStartFlags: SessionStartFlags, createParams: CLM.SessionCreateParams | CLM.CreateTeachParams): Promise { const inTeach = ((sessionStartFlags & SessionStartFlags.IN_TEACH) > 0) let entityList = await this.clClient.GetEntities(appId) @@ -304,7 +304,7 @@ export class CLRunner { } // check that this works = should it be inside edit continue above - // Check if StartSession call is required + // Check if StartSessionCallback is required await this.CheckSessionStartCallback(state, entityList.entities); let startSessionEntities = await state.EntityState.FilledEntitiesAsync() startSessionEntities = [...createParams.initialFilledEntities || [], ...startSessionEntities] @@ -323,7 +323,7 @@ export class CLRunner { logDialogId = null } else { - startResponse = await this.clClient.StartSession(appId, createParams as CLM.SessionCreateParams) + startResponse = await this.StartSession(appId, createParams as CLM.SessionCreateParams) sessionId = startResponse.sessionId logDialogId = startResponse.logDialogId } @@ -336,6 +336,70 @@ export class CLRunner { return startResponse } + private async StartSession(appId: string, createParams: CLM.SessionCreateParams): Promise { + + // Don't save logs on server if custom storage was provided + if (CLState.logStorage) { + createParams.saveToLog = false + } + + const session = await this.clClient.StartSession(appId, createParams as CLM.SessionCreateParams) + + // If using customer storage add to log storage + if (CLState.logStorage) { + // For self-hosted log storage logDialogId is hash of sessionId + session.logDialogId = CLM.hashText(session.logDialogId) + const logDialog: CLM.LogDialog = { + logDialogId: session.logDialogId, + packageId: session.packageId, + rounds: [], + initialFilledEntities: [], + targetTrainDialogIds: [], + createdDateTime: session.createdDatetime, + dialogBeginDatetime: new Date().getTime().toString(), + dialogEndDatetime: new Date().getTime().toString(), + lastModifiedDateTime: new Date().getTime().toString(), + metrics: "LARS" + } + CLState.logStorage.NewSession(appId, logDialog) + } + return session + } + + private async SessionExtract(appId: string, sessionId: string, userInput: CLM.UserInput): Promise { + const extractResponse = await this.clClient.SessionExtract(appId, sessionId, userInput) + + // Add to dev's log storage account (if it exists) + if (CLState.logStorage) { + const logDialogId = CLM.hashText(sessionId) + CLState.logStorage.AppendExtractorStep(appId, logDialogId, extractResponse) + } + return extractResponse + } + + private async SessionScore(appId: string, sessionId: string, scoreInput: CLM.ScoreInput): Promise { + const stepBeginDatetime = new Date().getTime().toString() + const scoreResponse = await this.clClient.SessionScore(appId, sessionId, scoreInput) + + // Add to dev's log storage account (if it exists) + if (CLState.logStorage) { + const logDialogId = CLM.hashText(sessionId) + const predictedAction = scoreResponse.scoredActions[0] ? scoreResponse.scoredActions[0].actionId : "" + const logScorerStep: CLM.LogScorerStep = { + input: scoreInput, + predictedAction, + logicResult: undefined, // LARS what should this be + predictionDetails: scoreResponse, + stepBeginDatetime, + stepEndDatetime: new Date().getTime().toString(), + metrics: scoreResponse.metrics + } + + CLState.logStorage.AppendScorerStep(appId, logDialogId, logScorerStep) + } + return scoreResponse + } + // Get the currently running app private async GetRunningApp(state: CLState, inEditingUI: boolean): Promise { let app = await state.BotState.GetApp() @@ -466,7 +530,7 @@ export class CLRunner { } // Start a new session - let session = await this.StartSessionAsync(state, conversationReference, app.appId, SessionStartFlags.NONE, sessionCreateParams) as CLM.Session + let session = await this.CreateSessionAsync(state, conversationReference, app.appId, SessionStartFlags.NONE, sessionCreateParams) as CLM.Session sessionId = session.sessionId } // Otherwise update last access time @@ -496,7 +560,7 @@ export class CLRunner { initialFilledEntities: [] } let sessionStartFlags = uiMode === UIMode.TEST ? SessionStartFlags.IN_TEST : SessionStartFlags.NONE - let session = await this.StartSessionAsync(state, BB.TurnContext.getConversationReference(activity), app.appId, sessionStartFlags, sessionCreateParams) as CLM.Session + let session = await this.CreateSessionAsync(state, BB.TurnContext.getConversationReference(activity), app.appId, sessionStartFlags, sessionCreateParams) as CLM.Session sessionId = session.sessionId } @@ -507,8 +571,13 @@ export class CLRunner { // Generate result errComponent = 'Extract Entities' - let userInput: CLM.UserInput = { text: buttonResponse || activity.text || ' ' } - let extractResponse = await this.clClient.SessionExtract(app.appId, sessionId, userInput) + const logDialogId = await state.BotState.GetLogDialogId() + if (!logDialogId) { + throw new Error("No logDialogId") + } + const userInput: CLM.UserInput = { text: buttonResponse || activity.text || ' ' } + const extractResponse = await this.SessionExtract(app.appId, sessionId, userInput) + entities = extractResponse.definitions.entities errComponent = 'Score Actions' const scoredAction = await this.Score( @@ -613,7 +682,7 @@ export class CLRunner { // Return top scoring action return scoreResponse.scoredActions[0] } else { - scoreResponse = await this.clClient.SessionScore(appId, sessionId, scoreInput) + scoreResponse = await this.SessionScore(appId, sessionId, scoreInput) // Return top scoring action return scoreResponse.scoredActions[0] diff --git a/src/ConversationLearner.ts b/src/ConversationLearner.ts index 52ef722f..21244c97 100644 --- a/src/ConversationLearner.ts +++ b/src/ConversationLearner.ts @@ -12,18 +12,19 @@ import { CLDebug } from './CLDebug' import { CLClient } from './CLClient' import { CLRecognizerResult } from './CLRecognizeResult' import { CLModelOptions } from '.' +import { ILogStorage } from './Memory/ILogStorage' export class ConversationLearner { public static options: CLOptions | null = null public static clClient: CLClient public clRunner: CLRunner - public static Init(options: CLOptions, storage?: BB.Storage): express.Router { + public static Init(options: CLOptions, storage?: BB.Storage, logStorage?: ILogStorage): express.Router { ConversationLearner.options = options try { this.clClient = new CLClient(options) - CLState.Init(storage) + CLState.Init(storage, logStorage) } catch (error) { CLDebug.Error(error, 'Conversation Learner Initialization') } diff --git a/src/CosmosLogStorage.ts b/src/CosmosLogStorage.ts index e0cd8759..d9050f5c 100644 --- a/src/CosmosLogStorage.ts +++ b/src/CosmosLogStorage.ts @@ -10,7 +10,7 @@ import * as CLM from '@conversationlearner/models' const DATABASE_ID = "LOG_DIALOGS" const COLLECTION_ID = "LOG_DIALOGS" const MAX_PAGE_SIZE = 100 -const PARTITION_KEY = { kind: 'Hash', paths: ['/appId'] } +const PARTITION_KEY = { kind: 'Hash', paths: ['/appId', '/packageId'] } interface StoredLogDialog extends CLM.LogDialog { // CosmosId @@ -47,19 +47,26 @@ export class CosmosLogStorage implements ILogStorage { } /** Add a log dialog to storage */ - public async Add(appId: string, logDialog: CLM.LogDialog) { - if (this.container) { - const storedLog = logDialog as StoredLogDialog - storedLog.id = this.GetDialogDocumentId(appId, logDialog.logDialogId) - storedLog.appId = appId - await this.container.items.create(storedLog) + public async Add(appId: string, logDialog: CLM.LogDialog): Promise { + if (!this.container) { + throw new Error("Cosmos Container Doesn't exist") } + + const storedLog = logDialog as StoredLogDialog + storedLog.id = this.GetDialogDocumentId(appId, logDialog.logDialogId) + storedLog.appId = appId + await this.container.items.create(storedLog) + return storedLog + } + + public async NewSession(appId: string, logDialog: CLM.LogDialog): Promise { + return await this.Add(appId, logDialog) } /** Append a scorer step to already existing log dialog */ public async AppendScorerStep(appId: string, logDialogId: string, scorerStep: CLM.LogScorerStep): Promise { if (!this.container) { - throw new Error("Cosmos Constainer Doesn't exist") + throw new Error("Cosmos Container Doesn't exist") } const { resource: logDialog } = await this.container.item(this.GetDialogDocumentId(appId, logDialogId), appId).read() if (!logDialog) { @@ -120,11 +127,13 @@ export class CosmosLogStorage implements ILogStorage { } else { querySpec = { - query: `SELECT * FROM c WHERE c.packageId = @packageId`, + //query: `SELECT * FROM c WHERE c.packageId = @packageId`,LARS + query: `SELECT * FROM c WHERE ARRAY_CONTAINS(@packageList, c.packageId)`, parameters: [ { - name: "@packageId", - value: packageId! + //name: "@packageId", + //value: packageId! + } ] } diff --git a/src/Memory/CLState.ts b/src/Memory/CLState.ts index fa936453..a04ce8be 100644 --- a/src/Memory/CLState.ts +++ b/src/Memory/CLState.ts @@ -11,6 +11,7 @@ import { BotState } from './BotState' import { InProcessMessageState as MessageState } from './InProcessMessageState' import { CLStorage } from './CLStorage'; import { BrowserSlotState } from './BrowserSlot'; +import { ILogStorage } from './ILogStorage' /** * CLState is a container all states (BotState, EntityState, MessageState) and can be passed around as a single access point. @@ -26,6 +27,7 @@ import { BrowserSlotState } from './BrowserSlot'; */ export class CLState { private static bbStorage: BB.Storage + public static logStorage: ILogStorage public readonly turnContext?: BB.TurnContext BotState: BotState @@ -48,13 +50,17 @@ export class CLState { this.turnContext = turnContext } - public static Init(storage?: BB.Storage): void { + public static Init(storage?: BB.Storage, logStorage?: ILogStorage): void { // If memory storage not defined use disk storage if (!storage) { CLDebug.Log('Storage not defined. Defaulting to in-memory storage.') storage = new BB.MemoryStorage() } + // Is developer providing their own log storage + if (logStorage) { + CLState.logStorage = logStorage + } CLState.bbStorage = storage } diff --git a/src/Memory/ILogStorage.ts b/src/Memory/ILogStorage.ts index 9d574e8b..70161406 100644 --- a/src/Memory/ILogStorage.ts +++ b/src/Memory/ILogStorage.ts @@ -10,8 +10,9 @@ export interface LogQueryResult { } export interface ILogStorage { + /** Add a log dialog to storage */ - Add(appId: string, logDialog: CLM.LogDialog): Promise + Add(appId: string, logDialog: CLM.LogDialog): Promise /** Retrieve a log dialog from storage */ Get(appId: string, logDialogId: string): Promise @@ -19,6 +20,8 @@ export interface ILogStorage { /** Delete a log dialog in storage */ Delete(appId: string, lodDialogId: string): Promise + NewSession(appId: string, logDialog: CLM.LogDialog): Promise + /** * Get all log dialogs matching parameters * @param appId Filer by appId if set @@ -29,7 +32,7 @@ export interface ILogStorage { GetMany(appId: string, packageId: string, continuationToken?: string, pageSize?: number): Promise /** Append an extractor step to already existing log dialog */ - AppendExtractorStep(appId: string, logDialogId: string, extractorStep: CLM.LogExtractorStep): Promise + AppendExtractorStep(appId: string, logDialogId: string, extractorStep: Partial): Promise /** Append a scorer step to already existing log dialog */ AppendScorerStep(appId: string, logDialogId: string, scorerStep: CLM.LogScorerStep): Promise diff --git a/src/http/router.ts b/src/http/router.ts index f4a813d7..2b49b568 100644 --- a/src/http/router.ts +++ b/src/http/router.ts @@ -485,7 +485,7 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. // LogDialogs //======================================================== /** - * RUN EXTRACTOR: Runs entity extraction on a log dialog + * RUN EXTRACTOR: Runs entity extraction on an existing log dialog for the specified turn */ router.put('/app/:appId/logdialog/:logDialogId/extractor/:turnIndex', async (req, res, next) => { try { @@ -493,8 +493,8 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. const { appId, logDialogId, turnIndex } = req.params const userInput: CLM.UserInput = req.body const extractResponse = await client.LogDialogExtract(appId, logDialogId, turnIndex, userInput) - const state = CLState.Get(key) + const memories = await state.EntityState.DumpMemory() const uiExtractResponse: CLM.UIExtractResponse = { extractResponse, memories } res.send(uiExtractResponse) @@ -503,6 +503,26 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. } }) + router.get('/app/:appId/logdialogs', async (req, res, next) => { + const { appId, continuationToken, pageSize } = req.params + + try { + const { packageId } = getQuery(req) + const packageIds = packageId.split(",") + let logDialogs + if (CLState.logStorage) { + // LARS paging and multiple package ids + logDialogs = CLState.logStorage.GetMany(appId, packageId[0], continuationToken, pageSize) + } + else { + logDialogs = await client.GetLogDialogs(appId, packageIds) + } + res.send(logDialogs) + } catch (error) { + HandleError(res, error) + } + }) + //======================================================== // TrainDialogs //======================================================== @@ -548,7 +568,7 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. // Start new teach session from the old train dialog const createTeachParams = CLM.ModelUtils.ToCreateTeachParams(trainDialog) - teachWithActivities.teach = await clRunner.StartSessionAsync(state, null, appId, SessionStartFlags.IN_TEACH | SessionStartFlags.IS_EDIT_CONTINUE, createTeachParams) as CLM.Teach + teachWithActivities.teach = await clRunner.CreateSessionAsync(state, null, appId, SessionStartFlags.IN_TEACH | SessionStartFlags.IS_EDIT_CONTINUE, createTeachParams) as CLM.Teach res.send(teachWithActivities) } catch (error) { @@ -578,7 +598,8 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. // Clear memory when running Log from UI state.EntityState.ClearAsync() - const session = await clRunner.StartSessionAsync(state, null, appId, SessionStartFlags.NONE, sessionCreateParams) as CLM.Session + const session = await clRunner.CreateSessionAsync(state, null, appId, SessionStartFlags.NONE, sessionCreateParams) as CLM.Session + res.send(session) } catch (error) { @@ -665,7 +686,7 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. // TeachSession always starts with a clear the memory (no saved entities) await state.EntityState.ClearAsync() - const teachResponse = await clRunner.StartSessionAsync(state, null, appId, SessionStartFlags.IN_TEACH, createTeachParams) as CLM.TeachResponse + const teachResponse = await clRunner.CreateSessionAsync(state, null, appId, SessionStartFlags.IN_TEACH, createTeachParams) as CLM.TeachResponse res.send(teachResponse) @@ -723,7 +744,7 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. const createTeachParams = CLM.ModelUtils.ToCreateTeachParams(cleanTrainDialog) // NOTE: Todo - pass in filteredDialogId so start sesssion doesn't find conflicts with existing dialog being edited - teachWithActivities.teach = await clRunner.StartSessionAsync(state, null, appId, SessionStartFlags.IN_TEACH | SessionStartFlags.IS_EDIT_CONTINUE, createTeachParams) as CLM.Teach + teachWithActivities.teach = await clRunner.CreateSessionAsync(state, null, appId, SessionStartFlags.IN_TEACH | SessionStartFlags.IS_EDIT_CONTINUE, createTeachParams) as CLM.Teach // If last action wasn't terminal then score if (teachWithActivities.dialogMode === CLM.DialogMode.Scorer) { @@ -783,7 +804,7 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. // Start new teach session from the old train dialog const createTeachParams = CLM.ModelUtils.ToCreateTeachParams(newTrainDialog) - const teach = await clRunner.StartSessionAsync(state, null, appId, SessionStartFlags.IN_TEACH | SessionStartFlags.IS_EDIT_CONTINUE, createTeachParams) as CLM.Teach + const teach = await clRunner.CreateSessionAsync(state, null, appId, SessionStartFlags.IN_TEACH | SessionStartFlags.IS_EDIT_CONTINUE, createTeachParams) as CLM.Teach // Get entities from my memory const filledEntities = await state.EntityState.FilledEntitiesAsync() @@ -834,7 +855,7 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. // Start new teach session from the old train dialog const createTeachParams = CLM.ModelUtils.ToCreateTeachParams(newTrainDialog) - const teach = await clRunner.StartSessionAsync(state, null, appId, SessionStartFlags.IN_TEACH | SessionStartFlags.IS_EDIT_CONTINUE, createTeachParams) as CLM.Teach + const teach = await clRunner.CreateSessionAsync(state, null, appId, SessionStartFlags.IN_TEACH | SessionStartFlags.IS_EDIT_CONTINUE, createTeachParams) as CLM.Teach // Do extraction const extractResponse = await client.TeachExtract(appId, teach.teachId, userInput, null) @@ -1183,7 +1204,7 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. packageId, initialFilledEntities: [] } - const session = await clRunner.StartSessionAsync(state, null, appId, SessionStartFlags.IN_TEST, sessionCreateParams) as CLM.Session + const session = await clRunner.CreateSessionAsync(state, null, appId, SessionStartFlags.IN_TEST, sessionCreateParams) as CLM.Session const logDialogId = session.logDialogId const conversation: BB.ConversationAccount = { diff --git a/src/index.ts b/src/index.ts index e66ae436..d4fc68f7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,6 +11,8 @@ import { FileStorage } from './FileStorage' import uiRouter from './uiRouter' import { SessionEndState, MemoryValue } from '@conversationlearner/models' import { EntityDetectionCallback, OnSessionStartCallback, OnSessionEndCallback, LogicCallback, RenderCallback, ICallbackInput } from './CLRunner' +import { ILogStorage } from './Memory/ILogStorage' +import { CosmosLogStorage } from './CosmosLogStorage' export { uiRouter, @@ -28,5 +30,9 @@ export { LogicCallback, MemoryValue, RenderCallback, - ICallbackInput + ICallbackInput, + // Interface for custom log storage + ILogStorage, + // Sample implementation of ILogStorage using CosmosDB + CosmosLogStorage } From 77943c28af157fc1f94aef69184d63ef0f2cf0e6 Mon Sep 17 00:00:00 2001 From: Lars Liden Date: Tue, 5 Nov 2019 09:14:10 -0800 Subject: [PATCH 04/14] feat: namespace mock data and multi-package query --- package-lock.json | 41 ++++++++++++++++++++++++++---------- package.json | 2 +- src/CosmosLogStorage.test.ts | 6 +++--- src/CosmosLogStorage.ts | 27 +++++++++++++++++++----- src/Memory/ILogStorage.ts | 2 +- 5 files changed, 57 insertions(+), 21 deletions(-) diff --git a/package-lock.json b/package-lock.json index d7a5ad9f..2ea10933 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4602,7 +4602,8 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "aproba": { "version": "1.2.0", @@ -4623,12 +4624,14 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -4643,17 +4646,20 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "core-util-is": { "version": "1.0.2", @@ -4770,7 +4776,8 @@ "inherits": { "version": "2.0.3", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "ini": { "version": "1.3.5", @@ -4782,6 +4789,7 @@ "version": "1.0.0", "bundled": true, "dev": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -4796,6 +4804,7 @@ "version": "3.0.4", "bundled": true, "dev": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -4803,12 +4812,14 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "minipass": { "version": "2.3.5", "bundled": true, "dev": true, + "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -4827,6 +4838,7 @@ "version": "0.5.1", "bundled": true, "dev": true, + "optional": true, "requires": { "minimist": "0.0.8" } @@ -4914,7 +4926,8 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "object-assign": { "version": "4.1.1", @@ -4926,6 +4939,7 @@ "version": "1.4.0", "bundled": true, "dev": true, + "optional": true, "requires": { "wrappy": "1" } @@ -5011,7 +5025,8 @@ "safe-buffer": { "version": "5.1.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "safer-buffer": { "version": "2.1.2", @@ -5047,6 +5062,7 @@ "version": "1.0.2", "bundled": true, "dev": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -5066,6 +5082,7 @@ "version": "3.0.1", "bundled": true, "dev": true, + "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -5109,12 +5126,14 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "yallist": { "version": "3.0.3", "bundled": true, - "dev": true + "dev": true, + "optional": true } } }, diff --git a/package.json b/package.json index 480e5b52..e84e59b9 100644 --- a/package.json +++ b/package.json @@ -115,4 +115,4 @@ "commit-msg": "commitlint -E HUSKY_GIT_PARAMS" } } -} \ No newline at end of file +} diff --git a/src/CosmosLogStorage.test.ts b/src/CosmosLogStorage.test.ts index 7dc930f1..abbc6cf6 100644 --- a/src/CosmosLogStorage.test.ts +++ b/src/CosmosLogStorage.test.ts @@ -12,7 +12,7 @@ const options = { function makeLogDialog(logDialogId: string): CLM.LogDialog { return { - ...CLM.makeLogDialog(), + ...CLM.MockData.makeLogDialog(), logDialogId } } @@ -114,7 +114,7 @@ describe('CosmosLogStorage', () => { const logDialog1 = makeLogDialog(logDialog1Id) await cls.Add(appId, logDialog1) - const scorerStep = CLM.makeLogScorerStep() + const scorerStep = CLM.MockData.makeLogScorerStep() await cls.AppendScorerStep(appId, logDialog1.logDialogId, scorerStep) let logDialog = await cls.Get(appId, logDialog1Id) @@ -136,7 +136,7 @@ describe('CosmosLogStorage', () => { const logDialog1 = makeLogDialog(logDialog1Id) await cls.Add(appId, logDialog1) - const extractorStep = CLM.makeLogExtractorStep() + const extractorStep = CLM.MockData.makeLogExtractorStep() await cls.AppendExtractorStep(appId, logDialog1.logDialogId, extractorStep) let logDialog = await cls.Get(appId, logDialog1Id) diff --git a/src/CosmosLogStorage.ts b/src/CosmosLogStorage.ts index d9050f5c..e26b2e32 100644 --- a/src/CosmosLogStorage.ts +++ b/src/CosmosLogStorage.ts @@ -102,19 +102,19 @@ export class CosmosLogStorage implements ILogStorage { /** * Get all log dialogs matching parameters * @param appId Filer by appId if set - * @param packageId Filter by Package if set + * @param packageIds Filter by Package if set * @param continuationToken Continuation token * @param pageSize Number to retrieve (max 100) */ - public async GetMany(appId?: string, packageId?: string, continuationToken?: string, pageSize = MAX_PAGE_SIZE): Promise { + public async GetMany(appId?: string, packageIds?: string[], continuationToken?: string, pageSize = MAX_PAGE_SIZE): Promise { let querySpec: Cosmos.SqlQuerySpec - if (!appId && !packageId) { + if (!appId && (!packageIds || packageIds.length == 0)) { querySpec = { query: `SELECT * FROM c` } } - else if (appId) { + else if (appId && (!packageIds || packageIds.length == 0)) { querySpec = { query: `SELECT * FROM c WHERE c.appId = @appId`, parameters: [ @@ -125,12 +125,14 @@ export class CosmosLogStorage implements ILogStorage { ] } } - else { + else if (!appId && (packageIds && packageIds.length > 0)) { querySpec = { //query: `SELECT * FROM c WHERE c.packageId = @packageId`,LARS query: `SELECT * FROM c WHERE ARRAY_CONTAINS(@packageList, c.packageId)`, parameters: [ { + name: '@packageList', + value: packageIds //name: "@packageId", //value: packageId! @@ -138,6 +140,21 @@ export class CosmosLogStorage implements ILogStorage { ] } } + else { + querySpec = { + query: `SELECT * FROM c WHERE ARRAY_CONTAINS(@packageList, c.packageId) AND c.appId = @appId`, + parameters: [ + { + name: "@appId", + value: appId! + }, + { + name: '@packageList', + value: packageIds! + } + ] + } + } const feedOptions: Cosmos.FeedOptions = { maxItemCount: Math.min(pageSize, MAX_PAGE_SIZE), diff --git a/src/Memory/ILogStorage.ts b/src/Memory/ILogStorage.ts index 70161406..1c50d10d 100644 --- a/src/Memory/ILogStorage.ts +++ b/src/Memory/ILogStorage.ts @@ -29,7 +29,7 @@ export interface ILogStorage { * @param continuationToken Continuation token * @param pageSize Number to retrieve (max 100) */ - GetMany(appId: string, packageId: string, continuationToken?: string, pageSize?: number): Promise + GetMany(appId: string, packageIds: string[], continuationToken?: string, pageSize?: number): Promise /** Append an extractor step to already existing log dialog */ AppendExtractorStep(appId: string, logDialogId: string, extractorStep: Partial): Promise From 7f3468037b9f5f63eb6c7436c8865b0e228c5e58 Mon Sep 17 00:00:00 2001 From: Lars Liden Date: Fri, 8 Nov 2019 11:15:45 -0800 Subject: [PATCH 05/14] feat: delete --- src/CLClient.ts | 18 +++- src/CLRunner.ts | 79 ++++++++++++--- src/CosmosLogStorage.test.ts | 82 +++++++++++++--- src/CosmosLogStorage.ts | 185 ++++++++++++++++------------------- src/Memory/ILogStorage.ts | 17 ++-- src/Utils.ts | 5 + src/http/router.ts | 128 ++++++++++++++++++------ 7 files changed, 344 insertions(+), 170 deletions(-) diff --git a/src/CLClient.ts b/src/CLClient.ts index 99277d4b..45f04be6 100644 --- a/src/CLClient.ts +++ b/src/CLClient.ts @@ -237,12 +237,28 @@ export class CLClient { // Log Dialogs //============================================================================= + public DeleteLogDialog(appId: string, logDialogId: string): Promise { + let apiPath = `app/${appId}/logdialog/${logDialogId}` + return this.send('DELETE', this.MakeURL(apiPath)) + } + + public DeleteLogDialogs(appId: string, logDialogIds: string[]): Promise { + const ids = logDialogIds.map(p => `id=${p}`).join("&") + let apiPath = `app/${appId}/logdialog&${ids}` + return this.send('DELETE', this.MakeURL(apiPath)) + } + + public GetLogDialog(appId: string, logDialogId: string): Promise { + const apiPath = `app/${appId}/logdialog/${logDialogId}` + return this.send('GET', this.MakeURL(apiPath)) + } + public GetLogDialogs(appId: string, packageIds: string[]): Promise { const packages = packageIds.map(p => `package=${p}`).join("&") const apiPath = `app/${appId}/logdialogs?includeDefinitions=false&${packages}` return this.send('GET', this.MakeURL(apiPath)) } - + /** Runs entity extraction (prediction). */ public LogDialogExtract( appId: string, diff --git a/src/CLRunner.ts b/src/CLRunner.ts index 0f71aabc..a800dafc 100644 --- a/src/CLRunner.ts +++ b/src/CLRunner.ts @@ -346,8 +346,8 @@ export class CLRunner { // If using customer storage add to log storage if (CLState.logStorage) { - // For self-hosted log storage logDialogId is hash of sessionId - session.logDialogId = CLM.hashText(session.logDialogId) + // For self-hosted log storage logDialogId is sessionId + session.logDialogId = session.sessionId const logDialog: CLM.LogDialog = { logDialogId: session.logDialogId, packageId: session.packageId, @@ -358,20 +358,35 @@ export class CLRunner { dialogBeginDatetime: new Date().getTime().toString(), dialogEndDatetime: new Date().getTime().toString(), lastModifiedDateTime: new Date().getTime().toString(), - metrics: "LARS" + metrics: "" } - CLState.logStorage.NewSession(appId, logDialog) + const ld = await CLState.logStorage.Add(appId, logDialog) + console.log(`${session.sessionId}:${session.logDialogId} / ${ld.logDialogId}`) } return session } private async SessionExtract(appId: string, sessionId: string, userInput: CLM.UserInput): Promise { + const stepBeginDatetime = new Date().getTime().toString() const extractResponse = await this.clClient.SessionExtract(appId, sessionId, userInput) + const stepEndDatetime = new Date().getTime().toString() // Add to dev's log storage account (if it exists) if (CLState.logStorage) { - const logDialogId = CLM.hashText(sessionId) - CLState.logStorage.AppendExtractorStep(appId, logDialogId, extractResponse) + // For local stroate logDialogId = sessionId + const logDialogId = sessionId + + // Append an extractor step to already existing log dialog + const logDialog: CLM.LogDialog | undefined = await CLState.logStorage.Get(appId, logDialogId) + if (!logDialog) { + throw new Error(`Log Dialog does not exist App:${appId} Id:${logDialogId}`) + } + const newRound: CLM.LogRound = { + extractorStep: { ...extractResponse, stepBeginDatetime, stepEndDatetime }, + scorerSteps: [] + } + logDialog.rounds.push(newRound) + await CLState.logStorage.Replace(appId, logDialog) } return extractResponse } @@ -382,19 +397,44 @@ export class CLRunner { // Add to dev's log storage account (if it exists) if (CLState.logStorage) { - const logDialogId = CLM.hashText(sessionId) + // For self-hosted storage logDialogId is sessionId + const logDialogId = sessionId const predictedAction = scoreResponse.scoredActions[0] ? scoreResponse.scoredActions[0].actionId : "" - const logScorerStep: CLM.LogScorerStep = { + + // Keep only needed data (drop payload, etc) + const scoredActions = scoreResponse.scoredActions.map(sa => { + return { + score: sa.score, + actionId: sa.actionId + } + }) + const unscoredActions = scoreResponse.unscoredActions.map(sa => { + return { + reason: sa.reason, + actionId: sa.actionId + } + }) + // Need to use recursive partial as scored and unscored have only partial data + const logScorerStep: Utils.RecursivePartial = { input: scoreInput, predictedAction, logicResult: undefined, // LARS what should this be - predictionDetails: scoreResponse, + predictionDetails: { scoredActions, unscoredActions }, stepBeginDatetime, stepEndDatetime: new Date().getTime().toString(), metrics: scoreResponse.metrics } - CLState.logStorage.AppendScorerStep(appId, logDialogId, logScorerStep) + const logDialog: CLM.LogDialog | undefined = await CLState.logStorage.Get(appId, logDialogId) + if (!logDialog) { + throw new Error(`Log Dialog does not exist App:${appId} Log:${logDialogId}`) + } + const lastRound = logDialog.rounds[logDialog.rounds.length - 1] + if (!lastRound || !lastRound.extractorStep) { + throw new Error(`Log Dialogs has no Extractor Step Id:${logDialogId}`) + } + lastRound.scorerSteps.push(logScorerStep as any) + await CLState.logStorage.Replace(appId, logDialog) } return scoreResponse } @@ -472,7 +512,7 @@ export class CLRunner { if (!app) { let error = "ERROR: AppId not specified. When running in a channel (i.e. Skype) or the Bot Framework Emulator, CONVERSATION_LEARNER_MODEL_ID must be specified in your Bot's .env file or Application Settings on the server" await this.SendMessage(state, error, activity.id) - return null; + return null } let sessionId = await state.BotState.GetSessionIdAndSetConversationId(conversationReference) @@ -541,14 +581,14 @@ export class CLRunner { // Handle any other non-message input, filter out empty messages if (activity.type !== BB.ActivityTypes.Message || !activity.text || activity.text === "") { await InputQueue.MessageHandled(state.MessageState, activity.id); - return null; + return null } // PackageId: Use live package id if not in editing UI, default to devPackage if no active package set let packageId = (inEditingUI ? await state.BotState.GetEditingPackageForApp(app.appId) : app.livePackageId) || app.devPackageId if (!packageId) { await this.SendMessage(state, "ERROR: No PackageId has been set", activity.id) - return null; + return null } // If no session for this conversation, create a new one @@ -574,7 +614,15 @@ export class CLRunner { if (!logDialogId) { throw new Error("No logDialogId") } - const userInput: CLM.UserInput = { text: buttonResponse || activity.text || ' ' } + + if (activity.text.length > Utils.CL_MAX_USER_UTTERANCE) { + CLDebug.Verbose(`Trimming user input to ${Utils.CL_MAX_USER_UTTERANCE} chars`) + } + + const userInput: CLM.UserInput = { + text: buttonResponse || activity.text.substr(0, Utils.CL_MAX_USER_UTTERANCE) || ' ' + } + const extractResponse = await this.SessionExtract(app.appId, sessionId, userInput) entities = extractResponse.definitions.entities @@ -605,7 +653,8 @@ export class CLRunner { CLDebug.Log(`Failed to End Session`) } - CLDebug.Error(error, errorContext) + const errMessage = error.body ? JSON.stringify(error.body) : JSON.stringify(error) + CLDebug.Error(errMessage, errorContext) return null } } diff --git a/src/CosmosLogStorage.test.ts b/src/CosmosLogStorage.test.ts index abbc6cf6..db15881d 100644 --- a/src/CosmosLogStorage.test.ts +++ b/src/CosmosLogStorage.test.ts @@ -17,19 +17,59 @@ function makeLogDialog(logDialogId: string): CLM.LogDialog { } } +async function getStorage(): Promise { + if (!options.endpoint || !options.key) { + console.log("Skipping Test. Cosmos credentials not defined.") + return undefined + } + return await CosmosLogStorage.Get(options) +} + +async function appendScorerStep(appId: string, logDialogId: string, logScorerStep: CLM.LogScorerStep, cls: CosmosLogStorage) { + const logDialog: CLM.LogDialog | undefined = await cls.Get(appId, logDialogId) + if (!logDialog) { + throw new Error(`Log Dialog does not exist App:${appId} Log:${logDialogId}`) + } + const lastRound = logDialog.rounds[logDialog.rounds.length - 1] + if (!lastRound || !lastRound.extractorStep) { + throw new Error(`Log Dialogs has no Extractor Step Id:${logDialogId}`) + } + lastRound.scorerSteps.push(logScorerStep as any) + await cls.Replace(appId, logDialog) +} + +async function appendExtractorStep(appId: string, logDialogId: string, extractorStep: CLM.LogExtractorStep, cls: CosmosLogStorage) { + // Append an extractor step to already existing log dialog + const logDialog: CLM.LogDialog | undefined = await cls.Get(appId, logDialogId) + if (!logDialog) { + throw new Error(`Log Dialog does not exist App:${appId} Id:${logDialogId}`) + } + const newRound: CLM.LogRound = { + extractorStep: { ...extractorStep, stepBeginDatetime: "", stepEndDatetime: "" }, + scorerSteps: [] + } + logDialog.rounds.push(newRound) + await cls.Replace(appId, logDialog) +} + describe('CosmosLogStorage', () => { // Before each test clear the database beforeEach(async () => { - const cls = await CosmosLogStorage.Get(options) - await cls.DeleteAll() + + const cls = await getStorage() + if (cls) { + await cls.DeleteAll() + } }) test('CreateAsync', async () => { - const cls = await CosmosLogStorage.Get(options) - + const cls = await getStorage() + if (!cls) { + return + } const appId = CLM.ModelUtils.generateGUID() const logDialog1Id = CLM.ModelUtils.generateGUID() const logDialog1 = makeLogDialog(logDialog1Id) @@ -41,7 +81,10 @@ describe('CosmosLogStorage', () => { test('GetAsync', async () => { - const cls = await CosmosLogStorage.Get(options) + const cls = await getStorage() + if (!cls) { + return + } const appId = CLM.ModelUtils.generateGUID() const logDialog1Id = CLM.ModelUtils.generateGUID() @@ -54,7 +97,10 @@ describe('CosmosLogStorage', () => { test('DeleteAsync', async () => { - const cls = await CosmosLogStorage.Get(options) + const cls = await getStorage() + if (!cls) { + return + } const appId = CLM.ModelUtils.generateGUID() const logDialog1Id = CLM.ModelUtils.generateGUID() @@ -73,7 +119,10 @@ describe('CosmosLogStorage', () => { test('DeleteAll', async () => { - const cls = await CosmosLogStorage.Get(options) + const cls = await getStorage() + if (!cls) { + return + } await cls.DeleteAll() let queryResult = await cls.GetMany() @@ -84,7 +133,10 @@ describe('CosmosLogStorage', () => { test('GetMany', async () => { - const cls = await CosmosLogStorage.Get(options) + const cls = await getStorage() + if (!cls) { + return + } // Create 10 items for (let i = 0; i < 8; i = i + 1) { @@ -107,7 +159,10 @@ describe('CosmosLogStorage', () => { test('AppendScorerStep', async () => { - const cls = await CosmosLogStorage.Get(options) + const cls = await getStorage() + if (!cls) { + return + } const appId = CLM.ModelUtils.generateGUID() const logDialog1Id = CLM.ModelUtils.generateGUID() @@ -115,7 +170,7 @@ describe('CosmosLogStorage', () => { await cls.Add(appId, logDialog1) const scorerStep = CLM.MockData.makeLogScorerStep() - await cls.AppendScorerStep(appId, logDialog1.logDialogId, scorerStep) + await appendScorerStep(appId, logDialog1.logDialogId, scorerStep, cls) let logDialog = await cls.Get(appId, logDialog1Id) expect(logDialog).not.toBe(undefined) @@ -129,7 +184,10 @@ describe('CosmosLogStorage', () => { test('AppendExtractorStep', async () => { - const cls = await CosmosLogStorage.Get(options) + const cls = await getStorage() + if (!cls) { + return + } const appId = CLM.ModelUtils.generateGUID() const logDialog1Id = CLM.ModelUtils.generateGUID() @@ -137,7 +195,7 @@ describe('CosmosLogStorage', () => { await cls.Add(appId, logDialog1) const extractorStep = CLM.MockData.makeLogExtractorStep() - await cls.AppendExtractorStep(appId, logDialog1.logDialogId, extractorStep) + await appendExtractorStep(appId, logDialog1.logDialogId, extractorStep, cls) let logDialog = await cls.Get(appId, logDialog1Id) expect(logDialog).not.toBe(undefined) diff --git a/src/CosmosLogStorage.ts b/src/CosmosLogStorage.ts index e26b2e32..c9d06fb5 100644 --- a/src/CosmosLogStorage.ts +++ b/src/CosmosLogStorage.ts @@ -10,6 +10,7 @@ import * as CLM from '@conversationlearner/models' const DATABASE_ID = "LOG_DIALOGS" const COLLECTION_ID = "LOG_DIALOGS" const MAX_PAGE_SIZE = 100 +const DELETE_BATCH_SIZE = 10 const PARTITION_KEY = { kind: 'Hash', paths: ['/appId', '/packageId'] } interface StoredLogDialog extends CLM.LogDialog { @@ -22,6 +23,8 @@ export class CosmosLogStorage implements ILogStorage { private client: Cosmos.CosmosClient private database: Cosmos.Database | undefined private container: Cosmos.Container | undefined + // Queue of items to be deleted + private deleteQueue: string[] /** * @@ -31,6 +34,7 @@ export class CosmosLogStorage implements ILogStorage { */ constructor(options: Cosmos.CosmosClientOptions) { this.client = new Cosmos.CosmosClient(options) + this.deleteQueue = [] } static async Get(options: Cosmos.CosmosClientOptions): Promise { @@ -55,48 +59,30 @@ export class CosmosLogStorage implements ILogStorage { const storedLog = logDialog as StoredLogDialog storedLog.id = this.GetDialogDocumentId(appId, logDialog.logDialogId) storedLog.appId = appId - await this.container.items.create(storedLog) - return storedLog + const itemResponse = await this.container.items.create(storedLog) + const { resource: createdLog } = await itemResponse.item.read() + return createdLog } - public async NewSession(appId: string, logDialog: CLM.LogDialog): Promise { - return await this.Add(appId, logDialog) - } - - /** Append a scorer step to already existing log dialog */ - public async AppendScorerStep(appId: string, logDialogId: string, scorerStep: CLM.LogScorerStep): Promise { - if (!this.container) { - throw new Error("Cosmos Container Doesn't exist") - } - const { resource: logDialog } = await this.container.item(this.GetDialogDocumentId(appId, logDialogId), appId).read() - if (!logDialog) { - throw new Error(`Log Dialog does not exist App:${appId} Log:${logDialogId}`) - } - const lastRound = logDialog.rounds[logDialog.rounds.length - 1] - if (!lastRound || lastRound.scorerSteps.length === 0) { - throw new Error(`Log Dialogs has no Extractor Step Log:${logDialogId}`) + /** Append an extractor step to already existing log dialog */ + public async Replace(appId: string, logDialog: CLM.LogDialog): Promise { + if (this.container) { + await this.container.item(this.GetDialogDocumentId(appId, logDialog.logDialogId), appId).replace(logDialog) } - lastRound.scorerSteps.push(scorerStep) - await this.container.items.upsert(logDialog) - return logDialog } - /** Append an extractor step to already existing log dialog */ - public async AppendExtractorStep(appId: string, logDialogId: string, extractorStep: CLM.LogExtractorStep): Promise { - if (!this.container) { - throw new Error("Cosmos Constainer Doesn't exist") - } - const { resource: logDialog } = await this.container.item(this.GetDialogDocumentId(appId, logDialogId), appId).read() - if (!logDialog) { - throw new Error(`Log Dialog does not exist App:${appId} Log:${logDialogId}`) - } - const newRound: CLM.LogRound = { - extractorStep: extractorStep, - scorerSteps: [] + + /** Retrieve a log dialog from storage */ + public async Get(appId: string, logDialogId: string): Promise { + if (this.container) { + // Check if scheduled for deletion + if (this.deleteQueue.includes(logDialogId)) { + return undefined + } + const { resource } = await this.container.item(this.GetDialogDocumentId(appId, logDialogId), appId).read() + return resource } - logDialog.rounds.push(newRound) - await this.container.items.upsert(logDialog) - return logDialog + return undefined } /** @@ -108,89 +94,80 @@ export class CosmosLogStorage implements ILogStorage { */ public async GetMany(appId?: string, packageIds?: string[], continuationToken?: string, pageSize = MAX_PAGE_SIZE): Promise { - let querySpec: Cosmos.SqlQuerySpec - if (!appId && (!packageIds || packageIds.length == 0)) { - querySpec = { - query: `SELECT * FROM c` + if (this.container) { + let and = "" + const querySpec: Cosmos.SqlQuerySpec = { + query: `SELECT * FROM c`, + parameters: [] } - } - else if (appId && (!packageIds || packageIds.length == 0)) { - querySpec = { - query: `SELECT * FROM c WHERE c.appId = @appId`, - parameters: [ - { - name: "@appId", - value: appId - } - ] + if (appId) { + querySpec.query = querySpec.query.concat(' WHERE c.appId = @appId') + querySpec.parameters!.push({ name: "@appId", value: appId }) + and = " AND" } - } - else if (!appId && (packageIds && packageIds.length > 0)) { - querySpec = { - //query: `SELECT * FROM c WHERE c.packageId = @packageId`,LARS - query: `SELECT * FROM c WHERE ARRAY_CONTAINS(@packageList, c.packageId)`, - parameters: [ - { - name: '@packageList', - value: packageIds - //name: "@packageId", - //value: packageId! - - } - ] + if (packageIds && packageIds.length > 0) { + querySpec.query = querySpec.query.concat(`${and} ARRAY_CONTAINS(@packageList, c.packageId)`) + querySpec.parameters!.push({ name: '@packageList', value: packageIds }) + and = " AND" } - } - else { - querySpec = { - query: `SELECT * FROM c WHERE ARRAY_CONTAINS(@packageList, c.packageId) AND c.appId = @appId`, - parameters: [ - { - name: "@appId", - value: appId! - }, - { - name: '@packageList', - value: packageIds! - } - ] + if (this.deleteQueue && this.deleteQueue.length > 0) { + querySpec.query = querySpec.query.concat(`${and} NOT ARRAY_CONTAINS(@logIdList, c.logDialogId)`) + querySpec.parameters!.push({ name: '@logIdList', value: this.deleteQueue }) } - } - const feedOptions: Cosmos.FeedOptions = { - maxItemCount: Math.min(pageSize, MAX_PAGE_SIZE), - continuation: continuationToken - } + const feedOptions: Cosmos.FeedOptions = { + maxItemCount: Math.min(pageSize, MAX_PAGE_SIZE), + continuation: continuationToken + } + + const feedResponse = await this.container.items.query(querySpec, feedOptions).fetchNext() - if (this.container) { - const feedResponse = await this.container.items.query(querySpec, feedOptions).fetchNext(); return { - logDialogs: feedResponse.resources, + logDialogs: feedResponse.resources as CLM.LogDialog[], continuationToken: feedResponse.continuation } } throw new Error("Contained undefined") } - /** Retrieve a log dialog from storage */ - public async Get(appId: string, logDialogId: string): Promise { + /** Delete a log dialog in storage */ + public async Delete(appId: string, logDialogId: string): Promise { if (this.container) { - // this.container. - const { resource } = await this.container.item(this.GetDialogDocumentId(appId, logDialogId), appId).read() - return resource + try { + await this.container.item(this.GetDialogDocumentId(appId, logDialogId), appId).delete() + } + catch (err) { + // TODO: consider retry + CLDebug.Error(err) + } } - return undefined } - /** Delete a log dialog in storage */ - public async Delete(appId: string, logDialogId: string): Promise { + /** Delete a log dialog in storage LARS review comments */ + public async DeleteMany(appId: string, logDialogIds: string[]): Promise { if (this.container) { - await this.container.item(this.GetDialogDocumentId(appId, logDialogId), appId).delete() - CLDebug.Log(`Deleted ${logDialogId} / ${this.GetDialogDocumentId(appId, logDialogId)}`) + // Add items to existing delete queue + if (this.deleteQueue.length > 0) { + this.deleteQueue.push(...logDialogIds) + } + // Otherwise set queue and start deleting. + else { + this.deleteQueue.push(...logDialogIds) + while (this.deleteQueue.length > 0) { + // Batch in batches of DELETE_BATCH_SIZE + const logSet = this.deleteQueue.slice(0, DELETE_BATCH_SIZE) + const promises = logSet.map(lid => this.Delete(appId, lid)) + await Promise.all(promises) + + // Remove them from the delete queue w/o mutating the object + logSet.forEach(id => this.deleteQueue.splice(this.deleteQueue.indexOf(id), 1)) + } + } } } /** Delete all log dialogs in storage */ - public async DeleteAll() { + public async DeleteAll(appId?: string) { if (!this.container) { throw new Error("Continer is undefined") } @@ -198,16 +175,22 @@ export class CosmosLogStorage implements ILogStorage { query: `SELECT * FROM c` } + if (appId) { + querySpec.query = querySpec.query.concat(' WHERE c.appId = @appId`') + querySpec.parameters = [{ name: "@appId", value: appId }] + } + let continuationToken: string | undefined do { const feedOptions: Cosmos.FeedOptions = { - maxItemCount: 5, + maxItemCount: DELETE_BATCH_SIZE, continuation: continuationToken } const feedResponse = await this.container.items.query(querySpec, feedOptions).fetchNext() - for (const logDialog of feedResponse.resources) { - await this.container.item(logDialog.id, logDialog.appId).delete(logDialog) - } + const logDialogs: StoredLogDialog[] = feedResponse.resources + const promises = logDialogs.map(ld => this.Delete(ld.logDialogId, ld.appId)) + await Promise.all(promises) + continuationToken = feedResponse.continuation } while (continuationToken); diff --git a/src/Memory/ILogStorage.ts b/src/Memory/ILogStorage.ts index 1c50d10d..72c0c9c9 100644 --- a/src/Memory/ILogStorage.ts +++ b/src/Memory/ILogStorage.ts @@ -17,11 +17,6 @@ export interface ILogStorage { /** Retrieve a log dialog from storage */ Get(appId: string, logDialogId: string): Promise - /** Delete a log dialog in storage */ - Delete(appId: string, lodDialogId: string): Promise - - NewSession(appId: string, logDialog: CLM.LogDialog): Promise - /** * Get all log dialogs matching parameters * @param appId Filer by appId if set @@ -31,12 +26,14 @@ export interface ILogStorage { */ GetMany(appId: string, packageIds: string[], continuationToken?: string, pageSize?: number): Promise - /** Append an extractor step to already existing log dialog */ - AppendExtractorStep(appId: string, logDialogId: string, extractorStep: Partial): Promise + /** Delete a log dialog in storage */ + Delete(appId: string, lodDialogId: string): Promise - /** Append a scorer step to already existing log dialog */ - AppendScorerStep(appId: string, logDialogId: string, scorerStep: CLM.LogScorerStep): Promise + /** Delete multiple log dialogs */ + DeleteMany(appId: string, logDialogIds: string[]): Promise /** Delete all log dialogs in storage */ - DeleteAll(): Promise + DeleteAll(appID?: string): Promise + + Replace(appId: string, logDialog: CLM.LogDialog): Promise } diff --git a/src/Utils.ts b/src/Utils.ts index 5de1b52b..b9adf5eb 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -113,6 +113,10 @@ export const actionHasHash = (actionId: string, hash: string, actions: CLM.Actio return false } +// Create recursive partial of an object +export type RecursivePartial = { + [P in keyof T]?: RecursivePartial; +} export const addEntitiesById = (valuesByName: CLM.FilledEntityMap): CLM.FilledEntityMap => { const valuesById = convertToMapById(valuesByName) @@ -208,6 +212,7 @@ export async function EndSessionIfOpen(clClient: CLClient, appId: string, sessio export const CL_DEVELOPER = 'ConversationLearnerDeveloper'; export const UI_RUNNER_APPID = 'UIRunner_AppId' +export const CL_MAX_USER_UTTERANCE = 500 export const getSha256Hash: (id: string) => string = (id) => { return crypto.createHash('sha256').update(id, 'utf8').digest('base64') diff --git a/src/http/router.ts b/src/http/router.ts index fdf8f54e..90ce2612 100644 --- a/src/http/router.ts +++ b/src/http/router.ts @@ -503,18 +503,36 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. } }) + router.get('/app/:appId/logdialog/:logDialogId', async (req, res, next) => { + const { appId, logDialogId } = req.params + + try { + let logDialog + if (CLState.logStorage) { + logDialog = await CLState.logStorage.Get(appId, logDialogId) + } + else { + logDialog = await client.GetLogDialog(appId, logDialogId) + } + res.send(logDialog) + } catch (error) { + HandleError(res, error) + } + }) + + // Get log dialogs router.get('/app/:appId/logdialogs', async (req, res, next) => { const { appId, continuationToken, pageSize } = req.params try { - const { packageId } = getQuery(req) - const packageIds = packageId.split(",") + const { package: packages } = getQuery(req) + const packageIds = packages.split(",") let logDialogs if (CLState.logStorage) { - // LARS paging and multiple package ids - logDialogs = CLState.logStorage.GetMany(appId, packageId[0], continuationToken, pageSize) + logDialogs = await CLState.logStorage.GetMany(appId, packageIds, continuationToken, pageSize) } else { + // LARS add paging logDialogs = await client.GetLogDialogs(appId, packageIds) } res.send(logDialogs) @@ -523,6 +541,40 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. } }) + // Delete one log dialogs + router.delete('/app/:appId/logdialogs/:logDialogId', async (req, res, next) => { + const { appId, logDialogId } = req.params + + try { + if (CLState.logStorage) { + await CLState.logStorage.Delete(appId, logDialogId) + } + else { + await client.DeleteLogDialog(appId, logDialogId) + } + res.send(200) + } catch (error) { + HandleError(res, error) + } + }) + + // Delete a list of log dialogs + router.delete('/app/:appId/logdialog', async (req, res, next) => { + const { appId } = req.params + const { id: logDialogIds } = getQuery(req) + try { + if (CLState.logStorage) { + CLState.logStorage.DeleteMany(appId, logDialogIds) + } + else { + await client.DeleteLogDialogs(appId, logDialogIds) + } + res.send(200) + } catch (error) { + HandleError(res, error) + } + }) + //======================================================== // TrainDialogs //======================================================== @@ -1234,42 +1286,56 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. } const turnContext = new BB.TurnContext(clRunner.adapter!, activity) - const result = await clRunner.recognize(turnContext) - if (result) { + try { + const result = await clRunner.recognize(turnContext) - // Get conversation reference - const conversationReference = BB.TurnContext.getConversationReference(activity) - if (!conversationReference) { - res.send(null) - return - } + if (result) { - // Get session Id - const sessionId = await result.state.BotState.GetSessionIdAsync() - if (!sessionId) { - res.send(null) - return - } + // Get conversation reference + const conversationReference = BB.TurnContext.getConversationReference(activity) + if (!conversationReference) { + throw new Error("Missing Conversation Reference") + } - // Include apiResults when taking action so result will be the same when testing - await clRunner.TakeActionAsync(conversationReference, result, null, turnValidation.apiResults[0]) + // Get session Id + const sessionId = await result.state.BotState.GetSessionIdAsync() + if (!sessionId) { + throw new Error("Can't find sessionId") + } - // Trigger next action if non-terminal - let bestAction = result.scoredAction - let curHashIndex = 0 + // Include apiResults when taking action so result will be the same when testing + await clRunner.TakeActionAsync(conversationReference, result, null, turnValidation.apiResults[0]) - // Server enforces max number of non-terminal actions, so no endless loop here - while (!bestAction.isTerminal) { - curHashIndex = curHashIndex + 1 - bestAction = await clRunner.Score(appId, sessionId, result.state, '', [], result.clEntities, false, true) - result.scoredAction = bestAction + // Trigger next action if non-terminal + let bestAction = result.scoredAction + let curHashIndex = 0 - // Include apiResults when taking action so result will be the same when testing - await clRunner.TakeActionAsync(conversationReference, result, null, turnValidation.apiResults[curHashIndex]) + // Server enforces max number of non-terminal actions, so no endless loop here + while (!bestAction.isTerminal) { + curHashIndex = curHashIndex + 1 + bestAction = await clRunner.Score(appId, sessionId, result.state, '', [], result.clEntities, false, true) + result.scoredAction = bestAction + + // Include apiResults when taking action so result will be the same when testing + await clRunner.TakeActionAsync(conversationReference, result, null, turnValidation.apiResults[curHashIndex]) + } + } + else { + throw new Error("No recognize result") } } - else { + catch (e) { + const error: Error = e + CLDebug.Error(error.message) + + // Delete log dialog + if (CLState.logStorage) { + await CLState.logStorage.Delete(appId, logDialogId) + } + else { + await client.DeleteLogDialog(appId, logDialogId) + } res.send(null) return } From 4dd97f2dfabb060c19d22df2e973e4e400ef3425 Mon Sep 17 00:00:00 2001 From: Lars Liden Date: Fri, 8 Nov 2019 13:33:36 -0800 Subject: [PATCH 06/14] fix: code review comments --- src/CLRunner.ts | 34 +++--- src/ConversationLearner.ts | 10 +- src/CosmosLogStorage.test.ts | 2 + src/CosmosLogStorage.ts | 200 ++++++++++++++++++++--------------- src/Memory/CLState.ts | 8 +- src/Memory/ILogStorage.ts | 15 +-- src/http/router.ts | 20 ++-- 7 files changed, 163 insertions(+), 126 deletions(-) diff --git a/src/CLRunner.ts b/src/CLRunner.ts index a800dafc..f97be060 100644 --- a/src/CLRunner.ts +++ b/src/CLRunner.ts @@ -338,14 +338,14 @@ export class CLRunner { private async StartSession(appId: string, createParams: CLM.SessionCreateParams): Promise { // Don't save logs on server if custom storage was provided - if (CLState.logStorage) { + if (ConversationLearner.logStorage) { createParams.saveToLog = false } const session = await this.clClient.StartSession(appId, createParams as CLM.SessionCreateParams) // If using customer storage add to log storage - if (CLState.logStorage) { + if (ConversationLearner.logStorage) { // For self-hosted log storage logDialogId is sessionId session.logDialogId = session.sessionId const logDialog: CLM.LogDialog = { @@ -355,29 +355,29 @@ export class CLRunner { initialFilledEntities: [], targetTrainDialogIds: [], createdDateTime: session.createdDatetime, - dialogBeginDatetime: new Date().getTime().toString(), - dialogEndDatetime: new Date().getTime().toString(), - lastModifiedDateTime: new Date().getTime().toString(), + // Start out the same. End is updated when dialog is edited + dialogBeginDatetime: new Date().toJSON(), + dialogEndDatetime: new Date().toJSON(), + lastModifiedDateTime: new Date().toJSON(), metrics: "" } - const ld = await CLState.logStorage.Add(appId, logDialog) - console.log(`${session.sessionId}:${session.logDialogId} / ${ld.logDialogId}`) + await ConversationLearner.logStorage.Add(appId, logDialog) } return session } private async SessionExtract(appId: string, sessionId: string, userInput: CLM.UserInput): Promise { - const stepBeginDatetime = new Date().getTime().toString() + const stepBeginDatetime = new Date().toJSON() const extractResponse = await this.clClient.SessionExtract(appId, sessionId, userInput) - const stepEndDatetime = new Date().getTime().toString() + const stepEndDatetime = new Date().toJSON() // Add to dev's log storage account (if it exists) - if (CLState.logStorage) { + if (ConversationLearner.logStorage) { // For local stroate logDialogId = sessionId const logDialogId = sessionId // Append an extractor step to already existing log dialog - const logDialog: CLM.LogDialog | undefined = await CLState.logStorage.Get(appId, logDialogId) + const logDialog: CLM.LogDialog | undefined = await ConversationLearner.logStorage.Get(appId, logDialogId) if (!logDialog) { throw new Error(`Log Dialog does not exist App:${appId} Id:${logDialogId}`) } @@ -386,17 +386,17 @@ export class CLRunner { scorerSteps: [] } logDialog.rounds.push(newRound) - await CLState.logStorage.Replace(appId, logDialog) + await ConversationLearner.logStorage.Replace(appId, logDialog) } return extractResponse } private async SessionScore(appId: string, sessionId: string, scoreInput: CLM.ScoreInput): Promise { - const stepBeginDatetime = new Date().getTime().toString() + const stepBeginDatetime = new Date().toJSON() const scoreResponse = await this.clClient.SessionScore(appId, sessionId, scoreInput) // Add to dev's log storage account (if it exists) - if (CLState.logStorage) { + if (ConversationLearner.logStorage) { // For self-hosted storage logDialogId is sessionId const logDialogId = sessionId const predictedAction = scoreResponse.scoredActions[0] ? scoreResponse.scoredActions[0].actionId : "" @@ -421,11 +421,11 @@ export class CLRunner { logicResult: undefined, // LARS what should this be predictionDetails: { scoredActions, unscoredActions }, stepBeginDatetime, - stepEndDatetime: new Date().getTime().toString(), + stepEndDatetime: new Date().toJSON(), metrics: scoreResponse.metrics } - const logDialog: CLM.LogDialog | undefined = await CLState.logStorage.Get(appId, logDialogId) + const logDialog: CLM.LogDialog | undefined = await ConversationLearner.logStorage.Get(appId, logDialogId) if (!logDialog) { throw new Error(`Log Dialog does not exist App:${appId} Log:${logDialogId}`) } @@ -434,7 +434,7 @@ export class CLRunner { throw new Error(`Log Dialogs has no Extractor Step Id:${logDialogId}`) } lastRound.scorerSteps.push(logScorerStep as any) - await CLState.logStorage.Replace(appId, logDialog) + await ConversationLearner.logStorage.Replace(appId, logDialog) } return scoreResponse } diff --git a/src/ConversationLearner.ts b/src/ConversationLearner.ts index 719618e1..625b85aa 100644 --- a/src/ConversationLearner.ts +++ b/src/ConversationLearner.ts @@ -17,15 +17,21 @@ import { ILogStorage } from './Memory/ILogStorage' export class ConversationLearner { public static options: CLOptions | null = null public static clClient: CLClient + public static logStorage: ILogStorage public static models: ConversationLearner[] = [] public clRunner: CLRunner - public static Init(options: CLOptions, storage?: BB.Storage, logStorage?: ILogStorage): express.Router { + public static Init(options: CLOptions, stateStorage?: BB.Storage, logStorage?: ILogStorage): express.Router { ConversationLearner.options = options try { this.clClient = new CLClient(options) - CLState.Init(storage, logStorage) + CLState.Init(stateStorage) + + // Is developer providing their own log storage + if (logStorage) { + this.logStorage = logStorage + } } catch (error) { CLDebug.Error(error, 'Conversation Learner Initialization') } diff --git a/src/CosmosLogStorage.test.ts b/src/CosmosLogStorage.test.ts index db15881d..3a823b82 100644 --- a/src/CosmosLogStorage.test.ts +++ b/src/CosmosLogStorage.test.ts @@ -35,6 +35,7 @@ async function appendScorerStep(appId: string, logDialogId: string, logScorerSte throw new Error(`Log Dialogs has no Extractor Step Id:${logDialogId}`) } lastRound.scorerSteps.push(logScorerStep as any) + logDialog.dialogEndDatetime = new Date().toJSON() await cls.Replace(appId, logDialog) } @@ -49,6 +50,7 @@ async function appendExtractorStep(appId: string, logDialogId: string, extractor scorerSteps: [] } logDialog.rounds.push(newRound) + logDialog.dialogEndDatetime = new Date().toJSON() await cls.Replace(appId, logDialog) } diff --git a/src/CosmosLogStorage.ts b/src/CosmosLogStorage.ts index c9d06fb5..5600de6d 100644 --- a/src/CosmosLogStorage.ts +++ b/src/CosmosLogStorage.ts @@ -50,37 +50,40 @@ export class CosmosLogStorage implements ILogStorage { return storage } - /** Add a log dialog to storage */ - public async Add(appId: string, logDialog: CLM.LogDialog): Promise { + /** Add a new log dialog to storage */ + public async Add(appId: string, logDialog: CLM.LogDialog): Promise { if (!this.container) { throw new Error("Cosmos Container Doesn't exist") } - const storedLog = logDialog as StoredLogDialog - storedLog.id = this.GetDialogDocumentId(appId, logDialog.logDialogId) - storedLog.appId = appId - const itemResponse = await this.container.items.create(storedLog) - const { resource: createdLog } = await itemResponse.item.read() - return createdLog - } - - /** Append an extractor step to already existing log dialog */ - public async Replace(appId: string, logDialog: CLM.LogDialog): Promise { - if (this.container) { - await this.container.item(this.GetDialogDocumentId(appId, logDialog.logDialogId), appId).replace(logDialog) + try { + const storedLog = logDialog as StoredLogDialog + storedLog.id = this.GetDialogDocumentId(appId, logDialog.logDialogId) + storedLog.appId = appId + const itemResponse = await this.container.items.create(storedLog) + const { resource: createdLog } = await itemResponse.item.read() + return createdLog + } + catch (err) { + CLDebug.Error(err) + return undefined } } - /** Retrieve a log dialog from storage */ public async Get(appId: string, logDialogId: string): Promise { if (this.container) { - // Check if scheduled for deletion - if (this.deleteQueue.includes(logDialogId)) { - return undefined + try { + // Check if scheduled for deletion + if (this.deleteQueue.includes(logDialogId)) { + return undefined + } + const { resource } = await this.container.item(this.GetDialogDocumentId(appId, logDialogId), appId).read() + return resource + } + catch (err) { + CLDebug.Error(err) } - const { resource } = await this.container.item(this.GetDialogDocumentId(appId, logDialogId), appId).read() - return resource } return undefined } @@ -95,41 +98,59 @@ export class CosmosLogStorage implements ILogStorage { public async GetMany(appId?: string, packageIds?: string[], continuationToken?: string, pageSize = MAX_PAGE_SIZE): Promise { if (this.container) { - let and = "" - const querySpec: Cosmos.SqlQuerySpec = { - query: `SELECT * FROM c`, - parameters: [] - } - if (appId) { - querySpec.query = querySpec.query.concat(' WHERE c.appId = @appId') - querySpec.parameters!.push({ name: "@appId", value: appId }) - and = " AND" - } - if (packageIds && packageIds.length > 0) { - querySpec.query = querySpec.query.concat(`${and} ARRAY_CONTAINS(@packageList, c.packageId)`) - querySpec.parameters!.push({ name: '@packageList', value: packageIds }) - and = " AND" - } - if (this.deleteQueue && this.deleteQueue.length > 0) { - querySpec.query = querySpec.query.concat(`${and} NOT ARRAY_CONTAINS(@logIdList, c.logDialogId)`) - querySpec.parameters!.push({ name: '@logIdList', value: this.deleteQueue }) - } + try { + let and = "" + const querySpec: Cosmos.SqlQuerySpec = { + query: `SELECT * FROM c`, + parameters: [] + } + if (appId) { + querySpec.query = querySpec.query.concat(' WHERE c.appId = @appId') + querySpec.parameters!.push({ name: "@appId", value: appId }) + and = " AND" + } + if (packageIds && packageIds.length > 0) { + querySpec.query = querySpec.query.concat(`${and} ARRAY_CONTAINS(@packageList, c.packageId)`) + querySpec.parameters!.push({ name: '@packageList', value: packageIds }) + and = " AND" + } + if (this.deleteQueue && this.deleteQueue.length > 0) { + querySpec.query = querySpec.query.concat(`${and} NOT ARRAY_CONTAINS(@logIdList, c.logDialogId)`) + querySpec.parameters!.push({ name: '@logIdList', value: this.deleteQueue }) + } - const feedOptions: Cosmos.FeedOptions = { - maxItemCount: Math.min(pageSize, MAX_PAGE_SIZE), - continuation: continuationToken - } + const feedOptions: Cosmos.FeedOptions = { + maxItemCount: Math.min(pageSize, MAX_PAGE_SIZE), + continuation: continuationToken + } - const feedResponse = await this.container.items.query(querySpec, feedOptions).fetchNext() + const feedResponse = await this.container.items.query(querySpec, feedOptions).fetchNext() - return { - logDialogs: feedResponse.resources as CLM.LogDialog[], - continuationToken: feedResponse.continuation + return { + logDialogs: feedResponse.resources as CLM.LogDialog[], + continuationToken: feedResponse.continuation + } + } + catch (err) { + CLDebug.Error(err) } } throw new Error("Contained undefined") } + + /** Replace and exisiting log dialog */ + public async Replace(appId: string, logDialog: CLM.LogDialog): Promise { + if (this.container) { + try { + await this.container.item(this.GetDialogDocumentId(appId, logDialog.logDialogId), appId).replace(logDialog) + } + catch (err) { + CLDebug.Error(err) + } + } + } + /** Delete a log dialog in storage */ public async Delete(appId: string, logDialogId: string): Promise { if (this.container) { @@ -137,63 +158,74 @@ export class CosmosLogStorage implements ILogStorage { await this.container.item(this.GetDialogDocumentId(appId, logDialogId), appId).delete() } catch (err) { - // TODO: consider retry CLDebug.Error(err) } } } - /** Delete a log dialog in storage LARS review comments */ + /** Delete multiple log dialogs */ public async DeleteMany(appId: string, logDialogIds: string[]): Promise { if (this.container) { - // Add items to existing delete queue - if (this.deleteQueue.length > 0) { - this.deleteQueue.push(...logDialogIds) - } - // Otherwise set queue and start deleting. - else { - this.deleteQueue.push(...logDialogIds) - while (this.deleteQueue.length > 0) { - // Batch in batches of DELETE_BATCH_SIZE - const logSet = this.deleteQueue.slice(0, DELETE_BATCH_SIZE) - const promises = logSet.map(lid => this.Delete(appId, lid)) - await Promise.all(promises) - - // Remove them from the delete queue w/o mutating the object - logSet.forEach(id => this.deleteQueue.splice(this.deleteQueue.indexOf(id), 1)) + try { + // Add items to existing delete queue + if (this.deleteQueue.length > 0) { + this.deleteQueue.push(...logDialogIds) + } + // Otherwise set queue and start deleting. + else { + this.deleteQueue.push(...logDialogIds) + while (this.deleteQueue.length > 0) { + // Batch in batches of DELETE_BATCH_SIZE + const logSet = this.deleteQueue.slice(0, DELETE_BATCH_SIZE) + const promises = logSet.map(lid => this.Delete(appId, lid)) + await Promise.all(promises) + + // Remove them from the delete queue w/o mutating the object + logSet.forEach(id => this.deleteQueue.splice(this.deleteQueue.indexOf(id), 1)) + } } } + catch (err) { + CLDebug.Error(err) + } } } - /** Delete all log dialogs in storage */ + /** Delete all log dialogs in storage + * @param appId If provided will only delete log dialogs from the given appId + */ public async DeleteAll(appId?: string) { if (!this.container) { throw new Error("Continer is undefined") } - const querySpec: Cosmos.SqlQuerySpec = { - query: `SELECT * FROM c` - } - - if (appId) { - querySpec.query = querySpec.query.concat(' WHERE c.appId = @appId`') - querySpec.parameters = [{ name: "@appId", value: appId }] - } + try { + const querySpec: Cosmos.SqlQuerySpec = { + query: `SELECT * FROM c` + } - let continuationToken: string | undefined - do { - const feedOptions: Cosmos.FeedOptions = { - maxItemCount: DELETE_BATCH_SIZE, - continuation: continuationToken + if (appId) { + querySpec.query = querySpec.query.concat(' WHERE c.appId = @appId`') + querySpec.parameters = [{ name: "@appId", value: appId }] } - const feedResponse = await this.container.items.query(querySpec, feedOptions).fetchNext() - const logDialogs: StoredLogDialog[] = feedResponse.resources - const promises = logDialogs.map(ld => this.Delete(ld.logDialogId, ld.appId)) - await Promise.all(promises) - continuationToken = feedResponse.continuation + let continuationToken: string | undefined + do { + const feedOptions: Cosmos.FeedOptions = { + maxItemCount: DELETE_BATCH_SIZE, + continuation: continuationToken + } + const feedResponse = await this.container.items.query(querySpec, feedOptions).fetchNext() + const logDialogs: StoredLogDialog[] = feedResponse.resources + const promises = logDialogs.map(ld => this.Delete(ld.logDialogId, ld.appId)) + await Promise.all(promises) + + continuationToken = feedResponse.continuation + } + while (continuationToken); + } + catch (err) { + CLDebug.Error(err) } - while (continuationToken); } /** Generate document id used in cosmos */ diff --git a/src/Memory/CLState.ts b/src/Memory/CLState.ts index 7e761f1e..6a99e38b 100644 --- a/src/Memory/CLState.ts +++ b/src/Memory/CLState.ts @@ -11,7 +11,6 @@ import { BotState } from './BotState' import { InProcessMessageState as MessageState } from './InProcessMessageState' import { CLStorage } from './CLStorage'; import { BrowserSlotState } from './BrowserSlot'; -import { ILogStorage } from './ILogStorage' /** * CLState is a container all states (BotState, EntityState, MessageState) and can be passed around as a single access point. @@ -27,7 +26,6 @@ import { ILogStorage } from './ILogStorage' */ export class CLState { private static bbStorage: BB.Storage - public static logStorage: ILogStorage public readonly turnContext?: BB.TurnContext BotState: BotState @@ -53,17 +51,13 @@ export class CLState { this.turnContext = turnContext } - public static Init(storage?: BB.Storage, logStorage?: ILogStorage): void { + public static Init(storage?: BB.Storage): void { // If memory storage not defined use disk storage if (!storage) { CLDebug.Log('Storage not defined. Defaulting to in-memory storage.') storage = new BB.MemoryStorage() } - // Is developer providing their own log storage - if (logStorage) { - CLState.logStorage = logStorage - } CLState.bbStorage = storage } diff --git a/src/Memory/ILogStorage.ts b/src/Memory/ILogStorage.ts index 72c0c9c9..6d0e7d70 100644 --- a/src/Memory/ILogStorage.ts +++ b/src/Memory/ILogStorage.ts @@ -11,8 +11,8 @@ export interface LogQueryResult { export interface ILogStorage { - /** Add a log dialog to storage */ - Add(appId: string, logDialog: CLM.LogDialog): Promise + /** Add a new log dialog to storage */ + Add(appId: string, logDialog: CLM.LogDialog): Promise /** Retrieve a log dialog from storage */ Get(appId: string, logDialogId: string): Promise @@ -26,14 +26,17 @@ export interface ILogStorage { */ GetMany(appId: string, packageIds: string[], continuationToken?: string, pageSize?: number): Promise + /** Replace and exisiting log dialog */ + Replace(appId: string, logDialog: CLM.LogDialog): Promise + /** Delete a log dialog in storage */ Delete(appId: string, lodDialogId: string): Promise /** Delete multiple log dialogs */ DeleteMany(appId: string, logDialogIds: string[]): Promise - /** Delete all log dialogs in storage */ - DeleteAll(appID?: string): Promise - - Replace(appId: string, logDialog: CLM.LogDialog): Promise + /** Delete all log dialogs in storage + * @param appId If provided will only delete log dialogs from the given appId + */ + DeleteAll(appId?: string): Promise } diff --git a/src/http/router.ts b/src/http/router.ts index 90ce2612..c13c3a19 100644 --- a/src/http/router.ts +++ b/src/http/router.ts @@ -508,8 +508,8 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. try { let logDialog - if (CLState.logStorage) { - logDialog = await CLState.logStorage.Get(appId, logDialogId) + if (ConversationLearner.logStorage) { + logDialog = await ConversationLearner.logStorage.Get(appId, logDialogId) } else { logDialog = await client.GetLogDialog(appId, logDialogId) @@ -528,8 +528,8 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. const { package: packages } = getQuery(req) const packageIds = packages.split(",") let logDialogs - if (CLState.logStorage) { - logDialogs = await CLState.logStorage.GetMany(appId, packageIds, continuationToken, pageSize) + if (ConversationLearner.logStorage) { + logDialogs = await ConversationLearner.logStorage.GetMany(appId, packageIds, continuationToken, pageSize) } else { // LARS add paging @@ -546,8 +546,8 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. const { appId, logDialogId } = req.params try { - if (CLState.logStorage) { - await CLState.logStorage.Delete(appId, logDialogId) + if (ConversationLearner.logStorage) { + await ConversationLearner.logStorage.Delete(appId, logDialogId) } else { await client.DeleteLogDialog(appId, logDialogId) @@ -563,8 +563,8 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. const { appId } = req.params const { id: logDialogIds } = getQuery(req) try { - if (CLState.logStorage) { - CLState.logStorage.DeleteMany(appId, logDialogIds) + if (ConversationLearner.logStorage) { + ConversationLearner.logStorage.DeleteMany(appId, logDialogIds) } else { await client.DeleteLogDialogs(appId, logDialogIds) @@ -1330,8 +1330,8 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. CLDebug.Error(error.message) // Delete log dialog - if (CLState.logStorage) { - await CLState.logStorage.Delete(appId, logDialogId) + if (ConversationLearner.logStorage) { + await ConversationLearner.logStorage.Delete(appId, logDialogId) } else { await client.DeleteLogDialog(appId, logDialogId) From fc17dc6e5412cf7e5c99c712988952eb03d45dc2 Mon Sep 17 00:00:00 2001 From: Lars Liden Date: Fri, 8 Nov 2019 13:40:16 -0800 Subject: [PATCH 07/14] fix: update models package --- package-lock.json | 6 +++--- package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index af0bac88..42655fb9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -536,9 +536,9 @@ } }, "@conversationlearner/models": { - "version": "0.216.0", - "resolved": "https://registry.npmjs.org/@conversationlearner/models/-/models-0.216.0.tgz", - "integrity": "sha512-3jbKa6mEFB0KSjJQzezmhMXCuEkqf9ZO+Tr2oY4HbxG8mMewWflk6QFXltEJFf/f0VdNbizeNv/U/8DSQ0hVzw==", + "version": "0.217.0", + "resolved": "https://registry.npmjs.org/@conversationlearner/models/-/models-0.217.0.tgz", + "integrity": "sha512-JU+3bCRv1Z8rUfFuIkZDiuIT1FmzN2F2hqA72K0vyrFYqt1h6HYvUW21c/0EDfEdAN9BxdgoKQc/lRCtqKpG1w==", "requires": { "jest-resolve": "^24.8.0" }, diff --git a/package.json b/package.json index d9dc25a5..38fd89ee 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "license": "MIT", "dependencies": { "@azure/cosmos": "^3.3.6", - "@conversationlearner/models": "0.216.0", + "@conversationlearner/models": "0.217.0", "@conversationlearner/ui": "0.407.0", "@types/supertest": "2.0.4", "async-file": "^2.0.2", From d89931d3dc72a1d9fd711efb612f16e06c053205 Mon Sep 17 00:00:00 2001 From: Lars Liden Date: Fri, 8 Nov 2019 14:26:24 -0800 Subject: [PATCH 08/14] fix: createdatetime --- src/CLRunner.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CLRunner.ts b/src/CLRunner.ts index 1bbeadd6..618336ec 100644 --- a/src/CLRunner.ts +++ b/src/CLRunner.ts @@ -354,7 +354,7 @@ export class CLRunner { rounds: [], initialFilledEntities: [], targetTrainDialogIds: [], - createdDateTime: session.createdDatetime, + createdDateTime: new Date().toJSON(), // Start out the same. End is updated when dialog is edited dialogBeginDatetime: new Date().toJSON(), dialogEndDatetime: new Date().toJSON(), From b7c0e6718f447e6cefe5ce76e3870ea910a8ffb8 Mon Sep 17 00:00:00 2001 From: Lars Liden Date: Mon, 11 Nov 2019 14:22:48 -0800 Subject: [PATCH 09/14] fix: multi-params --- src/http/router.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/http/router.ts b/src/http/router.ts index e075c1aa..3965a58f 100644 --- a/src/http/router.ts +++ b/src/http/router.ts @@ -525,8 +525,10 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. const { appId, continuationToken, pageSize } = req.params try { - const { package: packages } = getQuery(req) - const packageIds = packages.split(",") + let { package: packageIds } = getQuery(req) + if (typeof packageIds === "string") { + packageIds = [packageIds] + } let logDialogs if (ConversationLearner.logStorage) { logDialogs = await ConversationLearner.logStorage.GetMany(appId, packageIds, continuationToken, pageSize) @@ -561,7 +563,10 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. // Delete a list of log dialogs router.delete('/app/:appId/logdialog', async (req, res, next) => { const { appId } = req.params - const { id: logDialogIds } = getQuery(req) + let { id: logDialogIds } = getQuery(req) + if (typeof logDialogIds === "string") { + logDialogIds = [logDialogIds] + } try { if (ConversationLearner.logStorage) { ConversationLearner.logStorage.DeleteMany(appId, logDialogIds) From 62f7a961ea2b0b61000dc83f779d720497c19edb Mon Sep 17 00:00:00 2001 From: Lars Liden Date: Mon, 11 Nov 2019 14:40:33 -0800 Subject: [PATCH 10/14] fix: code review comments --- src/CLRunner.ts | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/src/CLRunner.ts b/src/CLRunner.ts index 618336ec..6feda9da 100644 --- a/src/CLRunner.ts +++ b/src/CLRunner.ts @@ -201,7 +201,7 @@ export class CLRunner { } public async InTrainingUI(turnContext: BB.TurnContext): Promise { - if (turnContext.activity.from ?.name === Utils.CL_DEVELOPER) { + if (turnContext.activity.from?.name === Utils.CL_DEVELOPER) { const state = CLState.GetFromContext(turnContext, this.configModelId) const app = await state.BotState.GetApp() // If no app selected in UI or no app set in config, or they don't match return true @@ -482,12 +482,12 @@ export class CLRunner { const conversationReference = BB.TurnContext.getConversationReference(activity) // Validate request - if (activity.from ?.id === undefined) { + if (activity.from?.id === undefined) { throw new Error(`Attempted to get current session for user, but user was not defined on bot request.`) } try { - const inEditingUI = conversationReference.user ?.name === Utils.CL_DEVELOPER + const inEditingUI = conversationReference.user?.name === Utils.CL_DEVELOPER // Validate setup if (!inEditingUI && !this.configModelId) { @@ -496,7 +496,7 @@ export class CLRunner { return null } - if (!ConversationLearner.options ?.LUIS_AUTHORING_KEY) { + if (!ConversationLearner.options?.LUIS_AUTHORING_KEY) { const msg = 'Options must specify luisAuthoringKey. Set the LUIS_AUTHORING_KEY.\n\n' CLDebug.Error(msg) return null @@ -607,11 +607,6 @@ export class CLRunner { // Generate result errorContext = 'Extract Entities' - const logDialogId = await state.BotState.GetLogDialogId() - if (!logDialogId) { - throw new Error("No logDialogId") - } - if (activity.text.length > Utils.CL_MAX_USER_UTTERANCE) { CLDebug.Verbose(`Trimming user input to ${Utils.CL_MAX_USER_UTTERANCE} chars`) } @@ -998,7 +993,7 @@ export class CLRunner { if (uiMode === UIMode.TEST) { placeHolderFilledEntities = testAPIResults } - else if (uiTrainScorerStep ?.trainScorerStep.logicResult) { + else if (uiTrainScorerStep?.trainScorerStep.logicResult) { placeHolderFilledEntities = uiTrainScorerStep.trainScorerStep.logicResult.changedFilledEntities } @@ -1339,7 +1334,7 @@ export class CLRunner { throw new Error(`Set Entity Action: ${action.actionId} referenced entity ${entity.entityName} but it is not an ENUM. Please update the action to reference the correct entity.`) } - const enumValueObj = entity.enumValues ?.find(ev => ev.enumValueId === action.enumValueId) + const enumValueObj = entity.enumValues?.find(ev => ev.enumValueId === action.enumValueId) if (!enumValueObj) { throw new Error(`Set Entity Action: ${action.actionId} which sets: ${entity.entityName} could not find the value with id: ${action.enumValueId}`) } @@ -1663,7 +1658,7 @@ export class CLRunner { for (let round of trainDialog.rounds) { let userText = round.extractorStep.textVariations[0].text - let filledEntities = round.scorerSteps[0] ?.input ?.filledEntities ?? [] + let filledEntities = round.scorerSteps[0]?.input?.filledEntities ?? [] // Check that entities exist for (let filledEntity of filledEntities) { @@ -1749,7 +1744,7 @@ export class CLRunner { } else { const filledEntity = filledEntityMap.map[entity.entityName] - if (filledEntity ?.values.length === 0) { + if (filledEntity?.values.length === 0) { missingEntities.push(entity.entityName) } } @@ -1783,7 +1778,7 @@ export class CLRunner { */ public async ReplayTrainDialogLogic(trainDialog: CLM.TrainDialog, state: CLState, cleanse: boolean): Promise { - if (!trainDialog ?.rounds) { + if (!trainDialog?.rounds) { return trainDialog } @@ -2008,7 +2003,7 @@ export class CLRunner { if (scoreIndex > 0) { const lastScoredAction = round.scorerSteps[scoreIndex - 1].labelAction let lastAction = actions.find(a => a.actionId == lastScoredAction) - if (lastAction ?.isTerminal) { + if (lastAction?.isTerminal) { replayError = new CLM.ReplayErrorActionAfterWait() replayErrors.push(replayError) } @@ -2029,7 +2024,7 @@ export class CLRunner { let entityList: CLM.EntityList = { entities } let prevMemories: CLM.Memory[] = [] - if (!trainDialog ?.rounds) { + if (!trainDialog?.rounds) { return null } @@ -2047,7 +2042,7 @@ export class CLRunner { for (let [roundIndex, round] of trainDialog.rounds.entries()) { // Use entities from first scorer step - const filledEntities = round.scorerSteps[0] ?.input ?.filledEntities ?? [] + const filledEntities = round.scorerSteps[0]?.input?.filledEntities ?? [] // Validate scorer step replayError = this.GetTrainDialogRoundErrors(round, roundIndex, curAction, trainDialog, entities, filledEntities, replayErrors) From 3dd40ad1c8c4dfcd9502f3282048f9f427744458 Mon Sep 17 00:00:00 2001 From: Lars Liden Date: Mon, 11 Nov 2019 15:11:59 -0800 Subject: [PATCH 11/14] fix: code review comments --- src/CLRunner.ts | 10 +-- src/CosmosLogStorage.test.ts | 7 +- src/CosmosLogStorage.ts | 168 ++++++++++++++++------------------- src/http/router.ts | 2 +- 4 files changed, 88 insertions(+), 99 deletions(-) diff --git a/src/CLRunner.ts b/src/CLRunner.ts index 6feda9da..df00be38 100644 --- a/src/CLRunner.ts +++ b/src/CLRunner.ts @@ -304,7 +304,7 @@ export class CLRunner { // check that this works = should it be inside edit continue above // Check if StartSessionCallback is required - await this.CheckSessionStartCallback(state, entityList.entities); + await this.CheckSessionStartCallback(state, entityList.entities) let startSessionEntities = await state.EntityState.FilledEntitiesAsync() startSessionEntities = [...createParams.initialFilledEntities ?? [], ...startSessionEntities] @@ -345,7 +345,7 @@ export class CLRunner { const session = await this.clClient.StartSession(appId, createParams as CLM.SessionCreateParams) // If using customer storage add to log storage - if (ConversationLearner.logStorage) { + if (ConversationLearner.logStorage && createParams.saveToLog) { // For self-hosted log storage logDialogId is sessionId session.logDialogId = session.sessionId const logDialog: CLM.LogDialog = { @@ -371,9 +371,9 @@ export class CLRunner { const extractResponse = await this.clClient.SessionExtract(appId, sessionId, userInput) const stepEndDatetime = new Date().toJSON() - // Add to dev's log storage account (if it exists) + // Add to dev's self-hosted log storage account (if it exists) if (ConversationLearner.logStorage) { - // For local stroate logDialogId = sessionId + // For self-holsted logDialogId = sessionId const logDialogId = sessionId // Append an extractor step to already existing log dialog @@ -399,7 +399,7 @@ export class CLRunner { if (ConversationLearner.logStorage) { // For self-hosted storage logDialogId is sessionId const logDialogId = sessionId - const predictedAction = scoreResponse.scoredActions[0] ? scoreResponse.scoredActions[0].actionId : "" + const predictedAction = scoreResponse.scoredActions[0]?.actionId ?? "" // Keep only needed data (drop payload, etc) const scoredActions = scoreResponse.scoredActions.map(sa => { diff --git a/src/CosmosLogStorage.test.ts b/src/CosmosLogStorage.test.ts index 3a823b82..f1656551 100644 --- a/src/CosmosLogStorage.test.ts +++ b/src/CosmosLogStorage.test.ts @@ -2,8 +2,8 @@ * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. */ -import * as CLM from '@conversationlearner/models'; -import { CosmosLogStorage } from './CosmosLogStorage'; +import * as CLM from '@conversationlearner/models' +import { CosmosLogStorage } from './CosmosLogStorage' const options = { endpoint: "", @@ -19,7 +19,7 @@ function makeLogDialog(logDialogId: string): CLM.LogDialog { async function getStorage(): Promise { if (!options.endpoint || !options.key) { - console.log("Skipping Test. Cosmos credentials not defined.") + console.log("Skipping Test. Cosmos credentials not defsined.") return undefined } return await CosmosLogStorage.Get(options) @@ -65,7 +65,6 @@ describe('CosmosLogStorage', () => { } }) - test('CreateAsync', async () => { const cls = await getStorage() diff --git a/src/CosmosLogStorage.ts b/src/CosmosLogStorage.ts index 5600de6d..c17d5693 100644 --- a/src/CosmosLogStorage.ts +++ b/src/CosmosLogStorage.ts @@ -50,17 +50,20 @@ export class CosmosLogStorage implements ILogStorage { return storage } - /** Add a new log dialog to storage */ - public async Add(appId: string, logDialog: CLM.LogDialog): Promise { + private get Container(): Cosmos.Container { if (!this.container) { throw new Error("Cosmos Container Doesn't exist") } + return this.container + } + /** Add a new log dialog to storage */ + public async Add(appId: string, logDialog: CLM.LogDialog): Promise { try { const storedLog = logDialog as StoredLogDialog storedLog.id = this.GetDialogDocumentId(appId, logDialog.logDialogId) storedLog.appId = appId - const itemResponse = await this.container.items.create(storedLog) + const itemResponse = await this.Container.items.create(storedLog) const { resource: createdLog } = await itemResponse.item.read() return createdLog } @@ -72,20 +75,17 @@ export class CosmosLogStorage implements ILogStorage { /** Retrieve a log dialog from storage */ public async Get(appId: string, logDialogId: string): Promise { - if (this.container) { - try { - // Check if scheduled for deletion - if (this.deleteQueue.includes(logDialogId)) { - return undefined - } - const { resource } = await this.container.item(this.GetDialogDocumentId(appId, logDialogId), appId).read() - return resource - } - catch (err) { - CLDebug.Error(err) + try { + // Check if scheduled for deletion + if (this.deleteQueue.includes(logDialogId)) { + return undefined } + const { resource } = await this.Container.item(this.GetDialogDocumentId(appId, logDialogId), appId).read() + return resource + } + catch (err) { + CLDebug.Error(err) } - return undefined } /** @@ -96,108 +96,98 @@ export class CosmosLogStorage implements ILogStorage { * @param pageSize Number to retrieve (max 100) */ public async GetMany(appId?: string, packageIds?: string[], continuationToken?: string, pageSize = MAX_PAGE_SIZE): Promise { + try { + let and = "" + const querySpec: Cosmos.SqlQuerySpec = { + query: `SELECT * FROM c`, + parameters: [] + } + if (appId) { + querySpec.query = querySpec.query.concat(' WHERE c.appId = @appId') + querySpec.parameters!.push({ name: "@appId", value: appId }) + and = " AND" + } + if (packageIds && packageIds.length > 0) { + querySpec.query = querySpec.query.concat(`${and} ARRAY_CONTAINS(@packageList, c.packageId)`) + querySpec.parameters!.push({ name: '@packageList', value: packageIds }) + and = " AND" + } + if (this.deleteQueue && this.deleteQueue.length > 0) { + querySpec.query = querySpec.query.concat(`${and} NOT ARRAY_CONTAINS(@logIdList, c.logDialogId)`) + querySpec.parameters!.push({ name: '@logIdList', value: this.deleteQueue }) + } - if (this.container) { - try { - let and = "" - const querySpec: Cosmos.SqlQuerySpec = { - query: `SELECT * FROM c`, - parameters: [] - } - if (appId) { - querySpec.query = querySpec.query.concat(' WHERE c.appId = @appId') - querySpec.parameters!.push({ name: "@appId", value: appId }) - and = " AND" - } - if (packageIds && packageIds.length > 0) { - querySpec.query = querySpec.query.concat(`${and} ARRAY_CONTAINS(@packageList, c.packageId)`) - querySpec.parameters!.push({ name: '@packageList', value: packageIds }) - and = " AND" - } - if (this.deleteQueue && this.deleteQueue.length > 0) { - querySpec.query = querySpec.query.concat(`${and} NOT ARRAY_CONTAINS(@logIdList, c.logDialogId)`) - querySpec.parameters!.push({ name: '@logIdList', value: this.deleteQueue }) - } - - const feedOptions: Cosmos.FeedOptions = { - maxItemCount: Math.min(pageSize, MAX_PAGE_SIZE), - continuation: continuationToken - } + const feedOptions: Cosmos.FeedOptions = { + maxItemCount: Math.min(pageSize, MAX_PAGE_SIZE), + continuation: continuationToken + } - const feedResponse = await this.container.items.query(querySpec, feedOptions).fetchNext() + const feedResponse = await this.Container.items.query(querySpec, feedOptions).fetchNext() - return { - logDialogs: feedResponse.resources as CLM.LogDialog[], - continuationToken: feedResponse.continuation - } + return { + logDialogs: feedResponse.resources as CLM.LogDialog[], + continuationToken: feedResponse.continuation } - catch (err) { - CLDebug.Error(err) + } + catch (err) { + CLDebug.Error(err) + return { + logDialogs: [], + continuationToken: undefined } } - throw new Error("Contained undefined") } - /** Replace and exisiting log dialog */ public async Replace(appId: string, logDialog: CLM.LogDialog): Promise { - if (this.container) { - try { - await this.container.item(this.GetDialogDocumentId(appId, logDialog.logDialogId), appId).replace(logDialog) - } - catch (err) { - CLDebug.Error(err) - } + try { + await this.Container.item(this.GetDialogDocumentId(appId, logDialog.logDialogId), appId).replace(logDialog) + } + catch (err) { + CLDebug.Error(err) } } /** Delete a log dialog in storage */ public async Delete(appId: string, logDialogId: string): Promise { - if (this.container) { - try { - await this.container.item(this.GetDialogDocumentId(appId, logDialogId), appId).delete() - } - catch (err) { - CLDebug.Error(err) - } + try { + await this.Container.item(this.GetDialogDocumentId(appId, logDialogId), appId).delete() + } + catch (err) { + CLDebug.Error(err) } } /** Delete multiple log dialogs */ public async DeleteMany(appId: string, logDialogIds: string[]): Promise { - if (this.container) { - try { - // Add items to existing delete queue - if (this.deleteQueue.length > 0) { - this.deleteQueue.push(...logDialogIds) - } - // Otherwise set queue and start deleting. - else { - this.deleteQueue.push(...logDialogIds) - while (this.deleteQueue.length > 0) { - // Batch in batches of DELETE_BATCH_SIZE - const logSet = this.deleteQueue.slice(0, DELETE_BATCH_SIZE) - const promises = logSet.map(lid => this.Delete(appId, lid)) - await Promise.all(promises) - - // Remove them from the delete queue w/o mutating the object - logSet.forEach(id => this.deleteQueue.splice(this.deleteQueue.indexOf(id), 1)) - } - } + try { + // Add items to existing delete queue + if (this.deleteQueue.length > 0) { + this.deleteQueue.push(...logDialogIds) } - catch (err) { - CLDebug.Error(err) + // Otherwise set queue and start deleting. + else { + this.deleteQueue.push(...logDialogIds) + while (this.deleteQueue.length > 0) { + // Batch in batches of DELETE_BATCH_SIZE + const logSet = this.deleteQueue.slice(0, DELETE_BATCH_SIZE) + const promises = logSet.map(lid => this.Delete(appId, lid)) + await Promise.all(promises) + + // Remove them from the delete queue w/o mutating the object + logSet.forEach(id => this.deleteQueue.splice(this.deleteQueue.indexOf(id), 1)) + } } } + catch (err) { + CLDebug.Error(err) + } } /** Delete all log dialogs in storage * @param appId If provided will only delete log dialogs from the given appId */ public async DeleteAll(appId?: string) { - if (!this.container) { - throw new Error("Continer is undefined") - } try { const querySpec: Cosmos.SqlQuerySpec = { query: `SELECT * FROM c` @@ -214,7 +204,7 @@ export class CosmosLogStorage implements ILogStorage { maxItemCount: DELETE_BATCH_SIZE, continuation: continuationToken } - const feedResponse = await this.container.items.query(querySpec, feedOptions).fetchNext() + const feedResponse = await this.Container.items.query(querySpec, feedOptions).fetchNext() const logDialogs: StoredLogDialog[] = feedResponse.resources const promises = logDialogs.map(ld => this.Delete(ld.logDialogId, ld.appId)) await Promise.all(promises) diff --git a/src/http/router.ts b/src/http/router.ts index 3965a58f..2e358faa 100644 --- a/src/http/router.ts +++ b/src/http/router.ts @@ -534,7 +534,7 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. logDialogs = await ConversationLearner.logStorage.GetMany(appId, packageIds, continuationToken, pageSize) } else { - // LARS add paging + // TODO: Add paging to server logDialogs = await client.GetLogDialogs(appId, packageIds) } res.send(logDialogs) From d1bb7966566b601e8110b1f3ea3cacf76f50d14a Mon Sep 17 00:00:00 2001 From: Lars Liden Date: Mon, 11 Nov 2019 15:55:05 -0800 Subject: [PATCH 12/14] fix: delete single, catch non-200s --- src/CLRunner.ts | 4 +++- src/CosmosLogStorage.ts | 22 ++++++++++++++++++---- src/http/router.ts | 4 ++-- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/CLRunner.ts b/src/CLRunner.ts index df00be38..a86ab800 100644 --- a/src/CLRunner.ts +++ b/src/CLRunner.ts @@ -337,6 +337,8 @@ export class CLRunner { private async StartSession(appId: string, createParams: CLM.SessionCreateParams): Promise { + const saveToLog = createParams.saveToLog + // Don't save logs on server if custom storage was provided if (ConversationLearner.logStorage) { createParams.saveToLog = false @@ -345,7 +347,7 @@ export class CLRunner { const session = await this.clClient.StartSession(appId, createParams as CLM.SessionCreateParams) // If using customer storage add to log storage - if (ConversationLearner.logStorage && createParams.saveToLog) { + if (ConversationLearner.logStorage && saveToLog) { // For self-hosted log storage logDialogId is sessionId session.logDialogId = session.sessionId const logDialog: CLM.LogDialog = { diff --git a/src/CosmosLogStorage.ts b/src/CosmosLogStorage.ts index c17d5693..d0407c84 100644 --- a/src/CosmosLogStorage.ts +++ b/src/CosmosLogStorage.ts @@ -64,7 +64,10 @@ export class CosmosLogStorage implements ILogStorage { storedLog.id = this.GetDialogDocumentId(appId, logDialog.logDialogId) storedLog.appId = appId const itemResponse = await this.Container.items.create(storedLog) - const { resource: createdLog } = await itemResponse.item.read() + const { resource: createdLog, statusCode } = await itemResponse.item.read() + if (statusCode < 200 || statusCode >= 300) { + throw new Error(`${statusCode}`) + } return createdLog } catch (err) { @@ -80,7 +83,10 @@ export class CosmosLogStorage implements ILogStorage { if (this.deleteQueue.includes(logDialogId)) { return undefined } - const { resource } = await this.Container.item(this.GetDialogDocumentId(appId, logDialogId), appId).read() + const { resource, statusCode } = await this.container!.item(this.GetDialogDocumentId(appId, logDialogId), appId).read() + if (statusCode < 200 || statusCode >= 300) { + throw new Error(`${statusCode}`) + } return resource } catch (err) { @@ -141,7 +147,10 @@ export class CosmosLogStorage implements ILogStorage { /** Replace and exisiting log dialog */ public async Replace(appId: string, logDialog: CLM.LogDialog): Promise { try { - await this.Container.item(this.GetDialogDocumentId(appId, logDialog.logDialogId), appId).replace(logDialog) + const { statusCode } = await this.Container.item(this.GetDialogDocumentId(appId, logDialog.logDialogId), appId).replace(logDialog) + if (statusCode < 200 || statusCode >= 300) { + throw new Error(`${statusCode}`) + } } catch (err) { CLDebug.Error(err) @@ -151,7 +160,10 @@ export class CosmosLogStorage implements ILogStorage { /** Delete a log dialog in storage */ public async Delete(appId: string, logDialogId: string): Promise { try { - await this.Container.item(this.GetDialogDocumentId(appId, logDialogId), appId).delete() + const { statusCode } = await this.Container.item(this.GetDialogDocumentId(appId, logDialogId), appId).delete() + if (statusCode < 200 || statusCode >= 300) { + throw new Error(`${statusCode}`) + } } catch (err) { CLDebug.Error(err) @@ -172,6 +184,7 @@ export class CosmosLogStorage implements ILogStorage { // Batch in batches of DELETE_BATCH_SIZE const logSet = this.deleteQueue.slice(0, DELETE_BATCH_SIZE) const promises = logSet.map(lid => this.Delete(appId, lid)) + // TODO: Handle 429's await Promise.all(promises) // Remove them from the delete queue w/o mutating the object @@ -207,6 +220,7 @@ export class CosmosLogStorage implements ILogStorage { const feedResponse = await this.Container.items.query(querySpec, feedOptions).fetchNext() const logDialogs: StoredLogDialog[] = feedResponse.resources const promises = logDialogs.map(ld => this.Delete(ld.logDialogId, ld.appId)) + // TODO: Handle 429's await Promise.all(promises) continuationToken = feedResponse.continuation diff --git a/src/http/router.ts b/src/http/router.ts index 2e358faa..1791b6d0 100644 --- a/src/http/router.ts +++ b/src/http/router.ts @@ -543,8 +543,8 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. } }) - // Delete one log dialogs - router.delete('/app/:appId/logdialogs/:logDialogId', async (req, res, next) => { + // Delete one log dialog + router.delete('/app/:appId/logdialog/:logDialogId', async (req, res, next) => { const { appId, logDialogId } = req.params try { From 588a37b990b69825587f0b86f5b7d1beb1686043 Mon Sep 17 00:00:00 2001 From: Lars Liden Date: Mon, 18 Nov 2019 09:00:40 -0800 Subject: [PATCH 13/14] feat: support continuation token --- src/CLClient.ts | 10 ++++++++-- src/CLRunner.ts | 2 ++ src/CosmosLogStorage.ts | 4 +++- src/http/router.ts | 13 ++++++------- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/CLClient.ts b/src/CLClient.ts index b12643ef..d812e9c9 100644 --- a/src/CLClient.ts +++ b/src/CLClient.ts @@ -253,9 +253,15 @@ export class CLClient { return this.send('GET', this.MakeURL(apiPath)) } - public GetLogDialogs(appId: string, packageIds: string[]): Promise { + public GetLogDialogs(appId: string, packageIds: string[], continuationToken?: string, maxPageSize?: string): Promise { const packages = packageIds.map(p => `package=${p}`).join("&") - const apiPath = `app/${appId}/logdialogs?includeDefinitions=false&${packages}` + let apiPath = `app/${appId}/logdialogs?includeDefinitions=false&${packages}` + if (continuationToken) { + apiPath = apiPath.concat(`&continuationToken=${encodeURIComponent(continuationToken)}`) + } + if (maxPageSize) { + apiPath = apiPath.concat(`&maxPageSize=${maxPageSize}`) + } return this.send('GET', this.MakeURL(apiPath)) } diff --git a/src/CLRunner.ts b/src/CLRunner.ts index a86ab800..9e72b96c 100644 --- a/src/CLRunner.ts +++ b/src/CLRunner.ts @@ -388,6 +388,7 @@ export class CLRunner { scorerSteps: [] } logDialog.rounds.push(newRound) + logDialog.lastModifiedDateTime = new Date().toJSON() await ConversationLearner.logStorage.Replace(appId, logDialog) } return extractResponse @@ -436,6 +437,7 @@ export class CLRunner { throw new Error(`Log Dialogs has no Extractor Step Id:${logDialogId}`) } lastRound.scorerSteps.push(logScorerStep as any) + logDialog.lastModifiedDateTime = new Date().toJSON() await ConversationLearner.logStorage.Replace(appId, logDialog) } return scoreResponse diff --git a/src/CosmosLogStorage.ts b/src/CosmosLogStorage.ts index d0407c84..032486b6 100644 --- a/src/CosmosLogStorage.ts +++ b/src/CosmosLogStorage.ts @@ -11,7 +11,7 @@ const DATABASE_ID = "LOG_DIALOGS" const COLLECTION_ID = "LOG_DIALOGS" const MAX_PAGE_SIZE = 100 const DELETE_BATCH_SIZE = 10 -const PARTITION_KEY = { kind: 'Hash', paths: ['/appId', '/packageId'] } +const PARTITION_KEY = { kind: 'Hash', paths: ['/appId'] } interface StoredLogDialog extends CLM.LogDialog { // CosmosId @@ -122,6 +122,8 @@ export class CosmosLogStorage implements ILogStorage { querySpec.query = querySpec.query.concat(`${and} NOT ARRAY_CONTAINS(@logIdList, c.logDialogId)`) querySpec.parameters!.push({ name: '@logIdList', value: this.deleteQueue }) } + // Return in reverse order so newest is on top + querySpec.query = querySpec.query.concat(' ORDER BY c.lastModifiedDateTime DESC') const feedOptions: Cosmos.FeedOptions = { maxItemCount: Math.min(pageSize, MAX_PAGE_SIZE), diff --git a/src/http/router.ts b/src/http/router.ts index 1791b6d0..31efd02a 100644 --- a/src/http/router.ts +++ b/src/http/router.ts @@ -522,22 +522,21 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express. // Get log dialogs router.get('/app/:appId/logdialogs', async (req, res, next) => { - const { appId, continuationToken, pageSize } = req.params + const { appId } = req.params try { - let { package: packageIds } = getQuery(req) + let { package: packageIds, continuationToken, maxPageSize } = getQuery(req) if (typeof packageIds === "string") { packageIds = [packageIds] } - let logDialogs + let logQueryResult: CLM.LogQueryResult if (ConversationLearner.logStorage) { - logDialogs = await ConversationLearner.logStorage.GetMany(appId, packageIds, continuationToken, pageSize) + logQueryResult = await ConversationLearner.logStorage.GetMany(appId, packageIds, continuationToken, maxPageSize) } else { - // TODO: Add paging to server - logDialogs = await client.GetLogDialogs(appId, packageIds) + logQueryResult = await client.GetLogDialogs(appId, packageIds, continuationToken, maxPageSize) } - res.send(logDialogs) + res.send(logQueryResult) } catch (error) { HandleError(res, error) } From 7d61b993b7cf6bef65dfc4674ccc566a98c9b1c4 Mon Sep 17 00:00:00 2001 From: Lars Liden Date: Mon, 18 Nov 2019 09:20:53 -0800 Subject: [PATCH 14/14] fix: update tslint to fix bug in tsutil https://github.com/ajafff/tsutils/issues/71 --- package-lock.json | 89 +++++++++---------------------- package.json | 4 +- src/AzureFunctions.ts | 3 ++ src/CLClient.ts | 3 ++ src/CLDebug.ts | 3 ++ src/CLRunner.ts | 3 ++ src/ConversationLearner.ts | 3 ++ src/CosmosLogStorage.ts | 3 ++ src/Memory/BotState.ts | 3 ++ src/Memory/ClientMemoryManager.ts | 6 +++ src/Memory/EntityState.ts | 3 ++ src/Memory/ILogStorage.ts | 5 +- src/Memory/InputQueue.ts | 3 ++ src/RedisStorage.ts | 3 ++ src/TemplateProvider.ts | 3 ++ src/Utils.ts | 3 ++ 16 files changed, 74 insertions(+), 66 deletions(-) diff --git a/package-lock.json b/package-lock.json index 42655fb9..13012e3f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -536,9 +536,9 @@ } }, "@conversationlearner/models": { - "version": "0.217.0", - "resolved": "https://registry.npmjs.org/@conversationlearner/models/-/models-0.217.0.tgz", - "integrity": "sha512-JU+3bCRv1Z8rUfFuIkZDiuIT1FmzN2F2hqA72K0vyrFYqt1h6HYvUW21c/0EDfEdAN9BxdgoKQc/lRCtqKpG1w==", + "version": "0.218.0", + "resolved": "https://registry.npmjs.org/@conversationlearner/models/-/models-0.218.0.tgz", + "integrity": "sha512-ogD9zt8WJwX0GqElrhQlkhxVzzrz2IbNdkglDenb1kgek+xKPqvzwyf5KzvKLCy8n6Lzmlx553c6CTcE6mfrEg==", "requires": { "jest-resolve": "^24.8.0" }, @@ -2330,44 +2330,6 @@ } } }, - "babel-code-frame": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz", - "integrity": "sha1-Y/1D99weO7fONZR9uP42mj9Yx0s=", - "dev": true, - "requires": { - "chalk": "^1.1.3", - "esutils": "^2.0.2", - "js-tokens": "^3.0.2" - }, - "dependencies": { - "ansi-styles": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", - "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=", - "dev": true - }, - "chalk": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", - "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", - "dev": true, - "requires": { - "ansi-styles": "^2.2.1", - "escape-string-regexp": "^1.0.2", - "has-ansi": "^2.0.0", - "strip-ansi": "^3.0.0", - "supports-color": "^2.0.0" - } - }, - "supports-color": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", - "dev": true - } - } - }, "babel-jest": { "version": "24.5.0", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-24.5.0.tgz", @@ -3230,9 +3192,9 @@ } }, "commander": { - "version": "2.15.1", - "resolved": "http://registry.npmjs.org/commander/-/commander-2.15.1.tgz", - "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag==", + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", "dev": true }, "commitizen": { @@ -3754,9 +3716,9 @@ "dev": true }, "diff": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", - "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.1.tgz", + "integrity": "sha512-s2+XdvhPCOF01LRQBC8hf4vhbVmI2CGS5aZnxLJlT5FtdhPCDFq80q++zK2KlrVorVDdL5BOGZ/VfLrVtYNF+Q==", "dev": true }, "diff-sequences": { @@ -5338,15 +5300,6 @@ "function-bind": "^1.1.1" } }, - "has-ansi": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", - "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", - "dev": true, - "requires": { - "ansi-regex": "^2.0.0" - } - }, "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", @@ -11218,23 +11171,24 @@ "integrity": "sha512-4krF8scpejhaOgqzBEcGM7yDIEfi0/8+8zDRZhNZZ2kjmHJ4hv3zCbQWxoJGz1iw5U0Jl0nma13xzHXcncMavQ==" }, "tslint": { - "version": "5.10.0", - "resolved": "https://registry.npmjs.org/tslint/-/tslint-5.10.0.tgz", - "integrity": "sha1-EeJrzLiK+gLdDZlWyuPUVAtfVMM=", + "version": "5.20.0", + "resolved": "https://registry.npmjs.org/tslint/-/tslint-5.20.0.tgz", + "integrity": "sha512-2vqIvkMHbnx8acMogAERQ/IuINOq6DFqgF8/VDvhEkBqQh/x6SP0Y+OHnKth9/ZcHQSroOZwUQSN18v8KKF0/g==", "dev": true, "requires": { - "babel-code-frame": "^6.22.0", + "@babel/code-frame": "^7.0.0", "builtin-modules": "^1.1.1", "chalk": "^2.3.0", "commander": "^2.12.1", - "diff": "^3.2.0", + "diff": "^4.0.1", "glob": "^7.1.1", - "js-yaml": "^3.7.0", + "js-yaml": "^3.13.1", "minimatch": "^3.0.4", + "mkdirp": "^0.5.1", "resolve": "^1.3.2", "semver": "^5.3.0", "tslib": "^1.8.0", - "tsutils": "^2.12.1" + "tsutils": "^2.29.0" }, "dependencies": { "js-yaml": { @@ -11246,6 +11200,15 @@ "argparse": "^1.0.7", "esprima": "^4.0.0" } + }, + "tsutils": { + "version": "2.29.0", + "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-2.29.0.tgz", + "integrity": "sha512-g5JVHCIJwzfISaXpXE1qvNalca5Jwob6FjI4AoPlqMusJ6ftFE7IkkFoMhVLRgK+4Kx3gkzb8UZK5t5yTTvEmA==", + "dev": true, + "requires": { + "tslib": "^1.8.1" + } } } }, diff --git a/package.json b/package.json index 38fd89ee..fbd97f69 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "license": "MIT", "dependencies": { "@azure/cosmos": "^3.3.6", - "@conversationlearner/models": "0.217.0", + "@conversationlearner/models": "0.218.0", "@conversationlearner/ui": "0.407.0", "@types/supertest": "2.0.4", "async-file": "^2.0.2", @@ -81,7 +81,7 @@ "jest": "24.5.0", "nodemon": "^1.11.0", "prettier": "^1.10.2", - "tslint": "^5.9.1", + "tslint": "^5.20.0", "tslint-config-prettier": "^1.12.0", "tslint-config-standard": "^7.0.0", "tslint-microsoft-contrib": "^5.0.3", diff --git a/src/AzureFunctions.ts b/src/AzureFunctions.ts index 785cceae..fc7b148a 100644 --- a/src/AzureFunctions.ts +++ b/src/AzureFunctions.ts @@ -5,6 +5,9 @@ import * as Request from 'request' import { CLDebug } from './CLDebug' +/** + * Support for calls to Azure Functions (currently disabled in UI) + */ export class AzureFunctions { public static Call(azureFunctionsUrl: string, azureFunctionsKey: string, funcName: string, args: string): Promise { let apiPath = 'app' diff --git a/src/CLClient.ts b/src/CLClient.ts index d812e9c9..8ca11368 100644 --- a/src/CLClient.ts +++ b/src/CLClient.ts @@ -26,6 +26,9 @@ export interface ICLClientOptions { LUIS_SUBSCRIPTION_KEY?: string } +/** + * Manages calls to Conversation Learner Service + */ export class CLClient { private options: ICLClientOptions diff --git a/src/CLDebug.ts b/src/CLDebug.ts index a4be307e..64b99064 100644 --- a/src/CLDebug.ts +++ b/src/CLDebug.ts @@ -25,6 +25,9 @@ interface LogMessage { logType: LogType } +/*** + * Handles debug output instead of console.log + */ export class CLDebug { private static adapter: BB.BotAdapter private static conversationReference: Partial diff --git a/src/CLRunner.ts b/src/CLRunner.ts index 9e72b96c..d5a8c0e7 100644 --- a/src/CLRunner.ts +++ b/src/CLRunner.ts @@ -106,6 +106,9 @@ export interface IActionResult { export type CallbackMap = { [name: string]: InternalCallback } +/** + * Runs Conversation Learnern for a given CL Model + */ export class CLRunner { /* Lookup table for CLRunners. One CLRunner per CL Model */ diff --git a/src/ConversationLearner.ts b/src/ConversationLearner.ts index 625b85aa..f5e9be69 100644 --- a/src/ConversationLearner.ts +++ b/src/ConversationLearner.ts @@ -14,6 +14,9 @@ import { CLRecognizerResult } from './CLRecognizeResult' import { CLModelOptions } from '.' import { ILogStorage } from './Memory/ILogStorage' +/** + * Main CL class used by Bot + */ export class ConversationLearner { public static options: CLOptions | null = null public static clClient: CLClient diff --git a/src/CosmosLogStorage.ts b/src/CosmosLogStorage.ts index 032486b6..2b8e4191 100644 --- a/src/CosmosLogStorage.ts +++ b/src/CosmosLogStorage.ts @@ -19,6 +19,9 @@ interface StoredLogDialog extends CLM.LogDialog { appId: string } +/** + * Custom Log Stoage account using Cosmos + */ export class CosmosLogStorage implements ILogStorage { private client: Cosmos.CosmosClient private database: Cosmos.Database | undefined diff --git a/src/Memory/BotState.ts b/src/Memory/BotState.ts index 82512aba..1b9d1ca5 100644 --- a/src/Memory/BotState.ts +++ b/src/Memory/BotState.ts @@ -64,6 +64,9 @@ export enum BotStateType { type GetKey = (datakey: string) => string export type ConvIdMapper = (ref: Partial | null) => string | null +/** + * Maintains state of Bot + */ export class BotState { private readonly storage: CLStorage private readonly getKey: GetKey diff --git a/src/Memory/ClientMemoryManager.ts b/src/Memory/ClientMemoryManager.ts index 08e44663..83b96f9f 100644 --- a/src/Memory/ClientMemoryManager.ts +++ b/src/Memory/ClientMemoryManager.ts @@ -8,6 +8,9 @@ import * as CLM from '@conversationlearner/models' export type MemoryManagerReturnType = T extends CLM.MemoryValue[] | CLM.MemoryValue ? T extends CLM.MemoryValue[] ? CLM.MemoryValue[] : CLM.MemoryValue : T +/** + * Used to read CL Memory in Bot code + */ export class ReadOnlyClientMemoryManager { protected allEntities: CLM.EntityBase[] = [] private sessionInfo: SessionInfo @@ -165,6 +168,9 @@ export class ReadOnlyClientMemoryManager { } } +/** + * Used to manipulate CL Memory in Bot code + */ export class ClientMemoryManager extends ReadOnlyClientMemoryManager { public constructor(prevMemories: CLM.FilledEntityMap, curMemories: CLM.FilledEntityMap, allEntities: CLM.EntityBase[], sessionInfo: SessionInfo) { diff --git a/src/Memory/EntityState.ts b/src/Memory/EntityState.ts index b461cea7..ee3397ab 100644 --- a/src/Memory/EntityState.ts +++ b/src/Memory/EntityState.ts @@ -11,6 +11,9 @@ const NEGATIVE_PREFIX = '~' export type GetKey = () => string +/** + * Memory for a given entity + */ export class EntityState { private storage: CLStorage private getKey: GetKey diff --git a/src/Memory/ILogStorage.ts b/src/Memory/ILogStorage.ts index 6d0e7d70..13c192fd 100644 --- a/src/Memory/ILogStorage.ts +++ b/src/Memory/ILogStorage.ts @@ -9,7 +9,10 @@ export interface LogQueryResult { continuationToken: string | undefined } -export interface ILogStorage { +/** + * Interface for generating custom LogStorage implimentations + */ +export declare class ILogStorage { /** Add a new log dialog to storage */ Add(appId: string, logDialog: CLM.LogDialog): Promise diff --git a/src/Memory/InputQueue.ts b/src/Memory/InputQueue.ts index 5cf9197c..fdb4a15b 100644 --- a/src/Memory/InputQueue.ts +++ b/src/Memory/InputQueue.ts @@ -14,6 +14,9 @@ export interface QueuedInput { callback: Function } +/** + * Used to queue up multiple user inputs when then come in a row + */ export class InputQueue { private static messageQueue: QueuedInput[] = []; diff --git a/src/RedisStorage.ts b/src/RedisStorage.ts index 02bf1039..7896ba14 100644 --- a/src/RedisStorage.ts +++ b/src/RedisStorage.ts @@ -18,6 +18,9 @@ export interface RedisStorageSettings { port?: number } +/** + * Bot storage implimentation using a Redis cache + */ export class RedisStorage implements Storage { private redisClient: Redis.RedisClient private _get: (...args: any[]) => Promise diff --git a/src/TemplateProvider.ts b/src/TemplateProvider.ts index 7d857be6..2d301249 100644 --- a/src/TemplateProvider.ts +++ b/src/TemplateProvider.ts @@ -7,6 +7,9 @@ import * as path from 'path' import { Template, TemplateVariable, RenderedActionArgument } from '@conversationlearner/models' import { CLDebug } from './CLDebug' +/** + * Provider for rendering templates + */ export class TemplateProvider { private static hasSubmitError = false diff --git a/src/Utils.ts b/src/Utils.ts index 493fa19a..b37ec70e 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -12,6 +12,9 @@ import * as CLM from '@conversationlearner/models' import * as HttpStatus from 'http-status-codes' import { CLClient } from './CLClient' +/** + * General utilities + */ export class Utils { public static SendTyping(adapter: BB.BotAdapter, address: any) { /* TODO