From d4b2446852b7f1cc9d279a71b33f3548e87422fe Mon Sep 17 00:00:00 2001 From: Tien Suwandy Date: Tue, 23 Mar 2021 10:34:32 -0700 Subject: [PATCH 01/18] Removed orchestratorRecognizer --- .../botbuilder-ai-orchestrator/src/index.ts | 1 - .../src/orchestratorAdaptiveRecognizer.ts | 27 ++++++- .../src/orchestratorRecognizer.ts | 72 ------------------- .../tests/mockResolver.js | 14 +++- .../orchestratorAdaptiveRecognizer.test.js | 2 +- libraries/tests.schema | 10 +-- 6 files changed, 43 insertions(+), 83 deletions(-) delete mode 100644 libraries/botbuilder-ai-orchestrator/src/orchestratorRecognizer.ts diff --git a/libraries/botbuilder-ai-orchestrator/src/index.ts b/libraries/botbuilder-ai-orchestrator/src/index.ts index f62c192c4e..a872467cb0 100644 --- a/libraries/botbuilder-ai-orchestrator/src/index.ts +++ b/libraries/botbuilder-ai-orchestrator/src/index.ts @@ -8,4 +8,3 @@ export { OrchestratorAdaptiveRecognizer } from './orchestratorAdaptiveRecognizer'; export { OrchestratorComponentRegistration } from './orchestratorComponentRegistration'; -export { OrchestratorRecognizer } from './orchestratorRecognizer'; diff --git a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts index 0c4ade21a2..b6497fa52a 100644 --- a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts +++ b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts @@ -34,6 +34,11 @@ export interface OrchestratorAdaptiveRecognizerConfiguration extends RecognizerC externalEntityRecognizer?: Recognizer; } +enum LabelType { + Intent = 1, + Entity = 2, +} + type LabelResolver = { score( text: string @@ -44,6 +49,21 @@ type LabelResolver = { name: string; }; }[]; + + score( + text: string, + labelType: number, + ): { + score: number; + closest_text: string; + label: { + name: string; + span: { + offset: number; + length: number; + } + }; + }[]; }; type Orchestrator = { @@ -56,7 +76,7 @@ type Orchestrator = { export class OrchestratorAdaptiveRecognizer extends AdaptiveRecognizer implements OrchestratorAdaptiveRecognizerConfiguration { - public static $kind = 'Microsoft.OrchestratorRecognizer'; + public static $kind = 'Microsoft.OrchestratorAdaptiveRecognizer'; /** * Path to Orchestrator base model folder. @@ -84,6 +104,11 @@ export class OrchestratorAdaptiveRecognizer */ public externalEntityRecognizer: Recognizer; + /** + * Enable entity detection if entity model exists inside modelFolder. Defaults to false. + */ + public scoreEntities: BoolExpression = new BoolExpression(false); + /** * Intent name if ambiguous intents are detected. */ diff --git a/libraries/botbuilder-ai-orchestrator/src/orchestratorRecognizer.ts b/libraries/botbuilder-ai-orchestrator/src/orchestratorRecognizer.ts deleted file mode 100644 index f241b70081..0000000000 --- a/libraries/botbuilder-ai-orchestrator/src/orchestratorRecognizer.ts +++ /dev/null @@ -1,72 +0,0 @@ -/** - * @module botbuilder-ai-orchestrator - */ -/** - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. - */ - -import { RecognizerResult, TurnContext } from 'botbuilder-core'; -import { Configurable, DialogContext, DialogSet, Recognizer } from 'botbuilder-dialogs'; -import { OrchestratorAdaptiveRecognizer } from './orchestratorAdaptiveRecognizer'; - -/** - * Class that represents an Orchestrator recognizer. - */ -export class OrchestratorRecognizer extends Configurable { - /** - * Full recognition results are available under this property - */ - public readonly resultProperty: string = 'result'; - - /** - * Recognizers unique ID. - */ - public id: string; - - /** - * Path to Orchestrator base model folder. - */ - public modelFolder: string; - - /** - * Path to the snapshot (.blu file) to load. - */ - public snapshotFile: string; - - /** - * The external entity recognizer. - */ - public externalEntityRecognizer: Recognizer; - - /** - * Threshold value to use for ambiguous intent detection. Defaults to 0.05. - * Recognizer returns ChooseIntent (disambiguation) if other intents are classified within this threshold of the top scoring intent. - */ - public disambiguationScoreThreshold = 0.05; - - /** - * Enable ambiguous intent detection. Defaults to false. - */ - public detectAmbiguousIntents = false; - - /** - * Returns recognition result. Also sends trace activity with recognition result. - * - * @param {TurnContext} turnContext Context for the current turn of conversation with the use. - * @returns {Promise} Recognizer result. - */ - public async recognize(turnContext: TurnContext): Promise { - const rec = new OrchestratorAdaptiveRecognizer().configure({ - id: this.id, - detectAmbiguousIntents: this.detectAmbiguousIntents, - modelFolder: this.modelFolder, - snapshotFile: this.snapshotFile, - disambiguationScoreThreshold: this.disambiguationScoreThreshold, - externalEntityRecognizer: this.externalEntityRecognizer, - }); - - const dc = new DialogContext(new DialogSet(), turnContext, { dialogStack: [] }); - return rec.recognize(dc, turnContext.activity); - } -} diff --git a/libraries/botbuilder-ai-orchestrator/tests/mockResolver.js b/libraries/botbuilder-ai-orchestrator/tests/mockResolver.js index ef86f676cc..53b5d4eeac 100644 --- a/libraries/botbuilder-ai-orchestrator/tests/mockResolver.js +++ b/libraries/botbuilder-ai-orchestrator/tests/mockResolver.js @@ -6,12 +6,20 @@ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ class MockResolver { - constructor(score) { + constructor(score, entityScore = null) { this._score = score; + this._entityScore } - score(_text) { - return this._score; + score(_text, labelType = 1) { + if (labelType == 1) { + return this._score; + } + else if (labelType == 2) { + return this._entityScore; + } + + throw new Error('Not supported!'); } } diff --git a/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js b/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js index b3d5d40f4a..e43c95c687 100644 --- a/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js +++ b/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js @@ -13,7 +13,7 @@ const { NumberEntityRecognizer } = require('botbuilder-dialogs-adaptive'); const sinon = require('sinon'); const { orchestratorIntentText, getLogPersonalInformation, validateTelemetry } = require('./recognizerTelemetryUtils'); -describe('OrchestratorAdpativeRecognizer tests', function () { +describe('OrchestratorAdaptiveRecognizer tests', function () { it('Expect initialize is called when orchestrator obj is null', async () => { const result = [ { diff --git a/libraries/tests.schema b/libraries/tests.schema index f27938bcde..ead91d1286 100644 --- a/libraries/tests.schema +++ b/libraries/tests.schema @@ -254,7 +254,7 @@ "$ref": "#/definitions/Microsoft.OnUnknownIntent" }, { - "$ref": "#/definitions/Microsoft.OrchestratorRecognizer" + "$ref": "#/definitions/Microsoft.OrchestratorAdaptiveRecognizer" }, { "$ref": "#/definitions/Microsoft.OrdinalEntityRecognizer" @@ -4269,7 +4269,7 @@ "$ref": "#/definitions/Microsoft.NumberRangeEntityRecognizer" }, { - "$ref": "#/definitions/Microsoft.OrchestratorRecognizer" + "$ref": "#/definitions/Microsoft.OrchestratorAdaptiveRecognizer" }, { "$ref": "#/definitions/Microsoft.OrdinalEntityRecognizer" @@ -7312,8 +7312,8 @@ } } }, - "Microsoft.OrchestratorRecognizer": { - "$role": "implements(Microsoft.IRecognizer)", + "Microsoft.OrchestratorAdaptiveRecognizer": { + "$role": "implements(Microsoft.OrchestratorAdaptiveRecognizerConfiguration)", "title": "Orchestrator recognizer", "description": "Orchestrator recognizer.", "type": "object", @@ -7383,7 +7383,7 @@ "description": "Defines the valid properties for the component you are configuring (from a dialog .schema file)", "type": "string", "pattern": "^[a-zA-Z][a-zA-Z0-9.]*$", - "const": "Microsoft.OrchestratorRecognizer" + "const": "Microsoft.OrchestratorAdaptiveRecognizer" }, "$designer": { "title": "Designer information", From 57ade541e948bf6b6dae1a899c8003c984a6d873 Mon Sep 17 00:00:00 2001 From: Tien Suwandy Date: Tue, 23 Mar 2021 13:45:10 -0700 Subject: [PATCH 02/18] check entity model folder and load orchestrator with entity model --- .../src/orchestratorAdaptiveRecognizer.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts index b6497fa52a..e289c2957c 100644 --- a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts +++ b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts @@ -107,7 +107,7 @@ export class OrchestratorAdaptiveRecognizer /** * Enable entity detection if entity model exists inside modelFolder. Defaults to false. */ - public scoreEntities: BoolExpression = new BoolExpression(false); + public scoreEntities: boolean = false; /** * Intent name if ambiguous intents are detected. @@ -293,8 +293,15 @@ export class OrchestratorAdaptiveRecognizer if (!existsSync(fullModelFolder)) { throw new Error(`Model folder does not exist at ${fullModelFolder}.`); } + + const entityModelFolder = resolve(this._modelFolder, 'entity'); + this.scoreEntities = existsSync(entityModelFolder); + const orchestrator = new oc.Orchestrator(); - if (!orchestrator.load(fullModelFolder)) { + if (this.scoreEntities && !orchestrator.load(fullModelFolder, entityModelFolder)) { + throw new Error(`Model load failed.`); + } + else if (!orchestrator.load(fullModelFolder)) { throw new Error(`Model load failed.`); } OrchestratorAdaptiveRecognizer.orchestrator = orchestrator; From 67110fcba81d165735f8f9d0f478065826440fd2 Mon Sep 17 00:00:00 2001 From: Dave Taniguchi Date: Tue, 23 Mar 2021 19:50:46 -0700 Subject: [PATCH 03/18] Add telemetry updates --- .../src/orchestratorAdaptiveRecognizer.ts | 73 +++++++++++++++++-- .../orchestratorAdaptiveRecognizer.test.js | 18 +++++ .../tests/recognizerTelemetryUtils.js | 13 +++- package.json | 1 + 4 files changed, 97 insertions(+), 8 deletions(-) diff --git a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts index 0c4ade21a2..3bf7e6176d 100644 --- a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts +++ b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts @@ -9,7 +9,7 @@ import { existsSync, readFileSync } from 'fs'; import { resolve } from 'path'; import { v4 as uuidv4 } from 'uuid'; - +import { omit } from 'lodash'; import { BoolExpression, BoolExpressionConverter, @@ -19,6 +19,7 @@ import { StringExpression, StringExpressionConverter, } from 'adaptive-expressions'; + import { AdaptiveRecognizer } from 'botbuilder-dialogs-adaptive'; import { Activity, RecognizerResult } from 'botbuilder-core'; import { Converter, ConverterFactory, DialogContext, Recognizer, RecognizerConfiguration } from 'botbuilder-dialogs'; @@ -199,10 +200,11 @@ export class OrchestratorAdaptiveRecognizer if (topScore < this.unknownIntentFilterScore) { recognizerResult.intents.None = { score: 1.0 }; } else { - // add top score - recognizerResult.intents[`${topScoringIntent}`] ??= { - score: topScore, - }; + // add all scores + recognizerResult.intents = results.reduce(function(intents, result){ + intents[result.label.name] = { score: result.score }; + return intents; + }, {}); // disambiguate if (detectAmbiguity) { @@ -253,6 +255,67 @@ export class OrchestratorAdaptiveRecognizer return recognizerResult; } + private getTopTwoIntents( + result: RecognizerResult + ): { name: string; score: number }[] { + if (!result || !result.intents) { + throw new Error('result is empty'); + } + const intents = Object.entries(result.intents) + .map((intent) => {return {name: intent[0], score: +intent[1].score} }) + .sort((a,b) => (b.score - a.score)) + intents.length = 2; + + return intents; + } + + /** + * Uses the RecognizerResult to create a list of properties to be included when tracking the result in telemetry. + * + * @param {RecognizerResult} recognizerResult Recognizer Result. + * @param {Record} telemetryProperties A list of properties to append or override the properties created using the RecognizerResult. + * @param {DialogContext} dialogContext Dialog Context. + * @returns {Record} A collection of properties that can be included when calling the TrackEvent method on the TelemetryClient. + */ + protected fillRecognizerResultTelemetryProperties( + recognizerResult: RecognizerResult, + telemetryProperties: Record, + dialogContext?: DialogContext + ): Record { + const topTwo = this.getTopTwoIntents(recognizerResult); + const intents = Object.entries(recognizerResult.intents); + const properties: Record = { + TopIntent: intents.length > 0 ? topTwo[0].name : undefined, + TopIntentScore: intents.length > 0 ? topTwo[0].score.toString() : undefined, + NextIntent: intents.length > 1 ? topTwo[1].name : undefined, + NextIntentScore: intents.length > 1 ? topTwo[1].score.toString() : undefined, + Intents: intents.length > 0 ? JSON.stringify(recognizerResult.intents) : undefined, + Entities: recognizerResult.entities ? JSON.stringify(recognizerResult.entities) : undefined, + AdditionalProperties: JSON.stringify( + omit(recognizerResult, ['text', 'alteredText', 'intents', 'entities']) + ), + }; + + var logPersonalInformation = + this.logPersonalInformation instanceof BoolExpression + ? this.logPersonalInformation.getValue(dialogContext.state) + : this.logPersonalInformation; + if (logPersonalInformation == undefined) + logPersonalInformation = false; + + if (logPersonalInformation) { + properties['Text'] = recognizerResult.text; + properties['AlteredText'] = recognizerResult.alteredText; + } + + // Additional Properties can override "stock" properties. + if (telemetryProperties) { + return Object.assign({}, properties, telemetryProperties); + } + + return properties; + } + private _initializeModel() { if (!this._modelFolder) { throw new Error(`Missing "ModelFolder" information.`); diff --git a/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js b/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js index b3d5d40f4a..e21ecc047d 100644 --- a/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js +++ b/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js @@ -143,6 +143,12 @@ describe('OrchestratorAdpativeRecognizer tests', function () { name: 'mockLabel', }, }, + { + score: 0.8, + label: { + name: 'mockLabel2', + }, + }, ]; const mockResolver = new MockResolver(result); const testPaths = 'test'; @@ -181,6 +187,12 @@ describe('OrchestratorAdpativeRecognizer tests', function () { name: 'mockLabel', }, }, + { + score: 0.8, + label: { + name: 'mockLabel2', + }, + }, ]; const mockResolver = new MockResolver(result); const testPaths = 'test'; @@ -219,6 +231,12 @@ describe('OrchestratorAdpativeRecognizer tests', function () { name: 'mockLabel', }, }, + { + score: 0.8, + label: { + name: 'mockLabel2', + }, + }, ]; const mockResolver = new MockResolver(result); const testPaths = 'test'; diff --git a/libraries/botbuilder-ai-orchestrator/tests/recognizerTelemetryUtils.js b/libraries/botbuilder-ai-orchestrator/tests/recognizerTelemetryUtils.js index 4e5d77debd..44ea2cf1c5 100644 --- a/libraries/botbuilder-ai-orchestrator/tests/recognizerTelemetryUtils.js +++ b/libraries/botbuilder-ai-orchestrator/tests/recognizerTelemetryUtils.js @@ -15,9 +15,12 @@ const spyOnTelemetryClientTrackEvent = (recognizer) => { }; const getLogPersonalInformation = (recognizer, dialogContext) => { - return recognizer.logPersonalInformation instanceof BoolExpression + var result = recognizer.logPersonalInformation instanceof BoolExpression ? recognizer.logPersonalInformation.getValue(dialogContext.state) : recognizer.logPersonalInformation; + if (result == undefined) + return false; + return result; }; const validateTelemetry = async ({ recognizer, dialogContext, spy, activity, result, callCount }) => { @@ -42,14 +45,17 @@ module.exports = { const getOrchestratorIntentProps = () => ({ TopIntent: 'mockLabel', TopIntentScore: '0.9', - Intents: JSON.stringify({ mockLabel: { score: 0.9 } }), + NextIntent: 'mockLabel2', + NextIntentScore: '0.8', + Intents: JSON.stringify({ mockLabel: { score: 0.9 }, mockLabel2: { score: 0.8 } } ), Entities: '{}', AdditionalProperties: JSON.stringify({ - result: [{ score: 0.9, label: { name: 'mockLabel' } }], + result: [{ score: 0.9, label: { name: 'mockLabel' } },{ score: 0.8, label: { name: 'mockLabel2' } } ], }), }); const getExpectedProps = (activity, result, logPersonalInformation) => { + const text = asMessageActivity(activity).text; const expectedProps = text === orchestratorIntentText ? getOrchestratorIntentProps() : {}; @@ -62,6 +68,7 @@ const getExpectedProps = (activity, result, logPersonalInformation) => { }; const hasValidTelemetryProps = (actual, expected) => { + if (Object.keys(actual).length !== Object.keys(expected).length) { return false; } diff --git a/package.json b/package.json index 3701584394..b50c85b37e 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "test:consumer": "yarn workspace consumer-test test", "test:devops": "npm-run-all test:teams test:mocha:junit test:nyc:cobertura", "test:github": "npm-run-all test:mocha:min test:runtime:min test:teams:min test:nyc:lcov", + "test:orchestrator": "wsrun -m -p \"botbuilder-ai-orchestrator\" -t test", "test:runtime": "wsrun -m -p \"botbuilder-runtime*\" -t test", "test:runtime:min": "wsrun -m -p \"botbuilder-runtime*\" -t test:min", "test:teams": "yarn workspace botbuilder-dialogs-adaptive-teams test", From 413aa8939120a7e727d161c909f35f26e7e0a6d9 Mon Sep 17 00:00:00 2001 From: Tien Suwandy Date: Wed, 24 Mar 2021 18:05:18 -0700 Subject: [PATCH 04/18] Update orchestratorAdaptiveRecognizer.ts --- .../src/orchestratorAdaptiveRecognizer.ts | 95 ++++++++++++++++--- 1 file changed, 83 insertions(+), 12 deletions(-) diff --git a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts index 2b8f60ce1c..6b3aaf5f03 100644 --- a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts +++ b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts @@ -40,6 +40,20 @@ enum LabelType { Entity = 2, } +type Label = { + name: string; + span: { + offset: number; + length: number; + } +}; + +type Result = { + score: number; + closest_text: string; + label: Label; +} + type LabelResolver = { score( text: string @@ -54,17 +68,7 @@ type LabelResolver = { score( text: string, labelType: number, - ): { - score: number; - closest_text: string; - label: { - name: string; - span: { - offset: number; - length: number; - } - }; - }[]; + ): Result[]; }; type Orchestrator = { @@ -116,10 +120,15 @@ export class OrchestratorAdaptiveRecognizer public readonly chooseIntent: string = 'ChooseIntent'; /** - * Full recognition results are available under this property + * Full intent recognition results are available under this property */ public readonly resultProperty: string = 'result'; + /** + * Full entity recognition results are available under this property + */ + public readonly entitiesProperty: string = 'entities'; + public getConverter(property: keyof OrchestratorAdaptiveRecognizerConfiguration): Converter | ConverterFactory { switch (property) { case 'modelFolder': @@ -264,6 +273,8 @@ export class OrchestratorAdaptiveRecognizer recognizerResult.intents.None = { score: 1.0 }; } + await this.tryScoreEntities(text, recognizerResult); + await dc.context.sendTraceActivity( 'OrchestratorAdaptiveRecognizer', recognizerResult, @@ -382,4 +393,64 @@ export class OrchestratorAdaptiveRecognizer this._resolver = OrchestratorAdaptiveRecognizer.orchestrator.createLabelResolver(snapshot); } } + + private async tryScoreEntities(text: string, recognizerResult: RecognizerResult) { + if (!this.scoreEntities) { + return; + } + + const results = await this._resolver.score(text, LabelType.Entity); + + // Add full entity recognition result as a 'result' property + recognizerResult[this.entitiesProperty] = results; + + if (results.length > 0) { + if (recognizerResult.entities === null) { + recognizerResult.entities = {}; + } + + results.forEach((result: Result) => { + const entityType = result.label.name; + + // add value + let values: any[] = recognizerResult.entities[entityType]; + if (!values) + { + values = recognizerResult.entities[entityType] = []; + } + + const span = result.label.span; + const entityText = text.substr(span.offset, span.length); + values.push({ + type: entityType, + score: result.score, + text: entityText, + start: span.offset, + end: span.offset + span.length + }); + + // get/create $instance + let instanceRoot: any = recognizerResult.entities['$instance']; + if (!instanceRoot) + { + instanceRoot = recognizerResult.entities['$instance'] = {}; + } + + // add instanceData + let instanceData: any[] = instanceRoot[entityType]; + if (!instanceData) + { + instanceData = instanceRoot[entityType] = []; + } + + instanceData.push({ + startIndex: span.offset, + endIndex: span.offset + span.length, + score: result.score, + text: entityText, + type: entityType, + }); + }); + } + } } From 0564524df140b2b0c2850597d0cd42b351d786de Mon Sep 17 00:00:00 2001 From: Tien Suwandy Date: Thu, 25 Mar 2021 00:10:27 -0700 Subject: [PATCH 05/18] added entity test --- .../botbuilder-ai-orchestrator/src/index.ts | 2 +- .../src/orchestratorAdaptiveRecognizer.ts | 23 +++++++------- .../tests/mockResolver.js | 11 ++++--- .../orchestratorAdaptiveRecognizer.test.js | 31 ++++++++++++++++--- 4 files changed, 44 insertions(+), 23 deletions(-) diff --git a/libraries/botbuilder-ai-orchestrator/src/index.ts b/libraries/botbuilder-ai-orchestrator/src/index.ts index a872467cb0..d8bb20f136 100644 --- a/libraries/botbuilder-ai-orchestrator/src/index.ts +++ b/libraries/botbuilder-ai-orchestrator/src/index.ts @@ -6,5 +6,5 @@ * Licensed under the MIT License. */ -export { OrchestratorAdaptiveRecognizer } from './orchestratorAdaptiveRecognizer'; +export { OrchestratorAdaptiveRecognizer, LabelType } from './orchestratorAdaptiveRecognizer'; export { OrchestratorComponentRegistration } from './orchestratorComponentRegistration'; diff --git a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts index 6b3aaf5f03..187fe0df99 100644 --- a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts +++ b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts @@ -35,7 +35,7 @@ export interface OrchestratorAdaptiveRecognizerConfiguration extends RecognizerC externalEntityRecognizer?: Recognizer; } -enum LabelType { +export enum LabelType { Intent = 1, Entity = 2, } @@ -127,7 +127,7 @@ export class OrchestratorAdaptiveRecognizer /** * Full entity recognition results are available under this property */ - public readonly entitiesProperty: string = 'entities'; + public readonly entityProperty: string = 'entityResult'; public getConverter(property: keyof OrchestratorAdaptiveRecognizerConfiguration): Converter | ConverterFactory { switch (property) { @@ -400,11 +400,13 @@ export class OrchestratorAdaptiveRecognizer } const results = await this._resolver.score(text, LabelType.Entity); + if (!results) { + throw new Error(`Failed scoring entities for: ${text}`); + } // Add full entity recognition result as a 'result' property - recognizerResult[this.entitiesProperty] = results; - - if (results.length > 0) { + recognizerResult[this.entityProperty] = results; + if (results.length) { if (recognizerResult.entities === null) { recognizerResult.entities = {}; } @@ -414,8 +416,7 @@ export class OrchestratorAdaptiveRecognizer // add value let values: any[] = recognizerResult.entities[entityType]; - if (!values) - { + if (!values) { values = recognizerResult.entities[entityType] = []; } @@ -427,19 +428,17 @@ export class OrchestratorAdaptiveRecognizer text: entityText, start: span.offset, end: span.offset + span.length - }); + }); // get/create $instance let instanceRoot: any = recognizerResult.entities['$instance']; - if (!instanceRoot) - { + if (!instanceRoot) { instanceRoot = recognizerResult.entities['$instance'] = {}; } // add instanceData let instanceData: any[] = instanceRoot[entityType]; - if (!instanceData) - { + if (!instanceData) { instanceData = instanceRoot[entityType] = []; } diff --git a/libraries/botbuilder-ai-orchestrator/tests/mockResolver.js b/libraries/botbuilder-ai-orchestrator/tests/mockResolver.js index 53b5d4eeac..22f31806fb 100644 --- a/libraries/botbuilder-ai-orchestrator/tests/mockResolver.js +++ b/libraries/botbuilder-ai-orchestrator/tests/mockResolver.js @@ -5,17 +5,18 @@ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ +const { OrchestratorAdaptiveRecognizer, LabelType } = require('../lib'); class MockResolver { - constructor(score, entityScore = null) { + constructor(score, entityScore) { this._score = score; - this._entityScore + this._entityScore = entityScore; } - score(_text, labelType = 1) { - if (labelType == 1) { + score(_text, labelType = LabelType.Intent) { + if (labelType === LabelType.Intent) { return this._score; } - else if (labelType == 2) { + else if (labelType === LabelType.Entity) { return this._entityScore; } diff --git a/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js b/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js index 573ed78bb0..98ed528604 100644 --- a/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js +++ b/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js @@ -4,7 +4,7 @@ */ const { MockResolver, TestAdapterSettings } = require('./mockResolver'); const { ok, rejects } = require('assert'); -const { OrchestratorAdaptiveRecognizer } = require('../lib'); +const { OrchestratorAdaptiveRecognizer, LabelType } = require('../lib'); const { DialogContext, DialogSet } = require('botbuilder-dialogs'); const { TurnContext, MessageFactory, NullTelemetryClient } = require('botbuilder-core'); const { BotFrameworkAdapter } = require('../../botbuilder/lib'); @@ -82,19 +82,40 @@ describe('OrchestratorAdaptiveRecognizer tests', function () { }, }, ]; - const mockResolver = new MockResolver(result); + const entityResult = [ + { + score: 0.75, + label: { + name: 'mockEntityLabel', + type: LabelType.Entity, + span: { + offset: 17, + length: 7 + }, + }, + }, + ]; + const mockResolver = new MockResolver(result, entityResult); const testPaths = 'test'; const rec = new OrchestratorAdaptiveRecognizer(testPaths, testPaths, mockResolver); + rec.scoreEntities = true; OrchestratorAdaptiveRecognizer.orchestrator = 'mock'; rec.modelFolder = new StringExpression(testPaths); rec.snapshotFile = new StringExpression(testPaths); rec.externalEntityRecognizer = new NumberEntityRecognizer(); - const { dc, activity } = createTestDcAndActivity('hello 123'); + const { dc, activity } = createTestDcAndActivity('turn on light in room 12'); const res = await rec.recognize(dc, activity); - ok(res.text, 'hello 123'); + ok(res.text, 'turn on light in room 12'); ok(res.intents.mockLabel.score, 0.9); - ok(res.entities.number[0], '123'); + ok(res.entities.number[0], '12'); + + ok(res['entityResult'] = entityResult); + ok(res.entities.mockEntityLabel); + ok(res.entities.mockEntityLabel[0].score, 0.75); + ok(res.entities.mockEntityLabel[0].text, 'room 12'); + ok(res.entities.mockEntityLabel[0].start, 17); + ok(res.entities.mockEntityLabel[0].end, 24); }); it('Test ambiguous intent recognition', async () => { From 26cb1768c74020851bf5a92cdcdab687c3ada299 Mon Sep 17 00:00:00 2001 From: Dave Taniguchi Date: Thu, 25 Mar 2021 10:36:18 -0700 Subject: [PATCH 06/18] Updates for CR feedback --- .../botbuilder-ai-orchestrator/package.json | 1 + .../src/orchestratorAdaptiveRecognizer.ts | 55 +++++++++---------- 2 files changed, 26 insertions(+), 30 deletions(-) diff --git a/libraries/botbuilder-ai-orchestrator/package.json b/libraries/botbuilder-ai-orchestrator/package.json index 18501f3b3e..d004c91349 100644 --- a/libraries/botbuilder-ai-orchestrator/package.json +++ b/libraries/botbuilder-ai-orchestrator/package.json @@ -33,6 +33,7 @@ "botbuilder-dialogs-adaptive": "4.1.6", "botbuilder-dialogs-declarative": "4.1.6", "orchestrator-core": "4.12.0-preview", + "lodash": "^4.17.21", "uuid": "^8.3.2" }, "scripts": { diff --git a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts index 187fe0df99..f71cc5e29d 100644 --- a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts +++ b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts @@ -9,7 +9,7 @@ import { existsSync, readFileSync } from 'fs'; import { resolve } from 'path'; import { v4 as uuidv4 } from 'uuid'; -import { omit } from 'lodash'; +import omit from 'lodash/omit'; import { BoolExpression, BoolExpressionConverter, @@ -40,21 +40,21 @@ export enum LabelType { Entity = 2, } -type Label = { +interface Label { name: string; span: { offset: number; length: number; - } -}; + }; +} -type Result = { +interface Result { score: number; closest_text: string; label: Label; } -type LabelResolver = { +interface LabelResolver { score( text: string ): { @@ -65,11 +65,8 @@ type LabelResolver = { }; }[]; - score( - text: string, - labelType: number, - ): Result[]; -}; + score(text: string, labelType: number): Result[]; +} type Orchestrator = { createLabelResolver(snapshot: Uint8Array): LabelResolver; @@ -112,22 +109,22 @@ export class OrchestratorAdaptiveRecognizer /** * Enable entity detection if entity model exists inside modelFolder. Defaults to false. */ - public scoreEntities: boolean = false; + public scoreEntities = false; /** * Intent name if ambiguous intents are detected. */ - public readonly chooseIntent: string = 'ChooseIntent'; + public readonly chooseIntent = 'ChooseIntent'; /** * Full intent recognition results are available under this property */ - public readonly resultProperty: string = 'result'; + public readonly resultProperty = 'result'; /** * Full entity recognition results are available under this property */ - public readonly entityProperty: string = 'entityResult'; + public readonly entityProperty = 'entityResult'; public getConverter(property: keyof OrchestratorAdaptiveRecognizerConfiguration): Converter | ConverterFactory { switch (property) { @@ -235,10 +232,10 @@ export class OrchestratorAdaptiveRecognizer recognizerResult.intents.None = { score: 1.0 }; } else { // add all scores - recognizerResult.intents = results.reduce(function(intents, result){ + recognizerResult.intents = results.reduce(function (intents, result) { intents[result.label.name] = { score: result.score }; return intents; - }, {}); + }, {}); // disambiguate if (detectAmbiguity) { @@ -291,15 +288,15 @@ export class OrchestratorAdaptiveRecognizer return recognizerResult; } - private getTopTwoIntents( - result: RecognizerResult - ): { name: string; score: number }[] { + private getTopTwoIntents(result: RecognizerResult): { name: string; score: number }[] { if (!result || !result.intents) { throw new Error('result is empty'); } const intents = Object.entries(result.intents) - .map((intent) => {return {name: intent[0], score: +intent[1].score} }) - .sort((a,b) => (b.score - a.score)) + .map((intent) => { + return { name: intent[0], score: +intent[1].score }; + }) + .sort((a, b) => b.score - a.score); intents.length = 2; return intents; @@ -307,7 +304,7 @@ export class OrchestratorAdaptiveRecognizer /** * Uses the RecognizerResult to create a list of properties to be included when tracking the result in telemetry. - * + * * @param {RecognizerResult} recognizerResult Recognizer Result. * @param {Record} telemetryProperties A list of properties to append or override the properties created using the RecognizerResult. * @param {DialogContext} dialogContext Dialog Context. @@ -332,12 +329,11 @@ export class OrchestratorAdaptiveRecognizer ), }; - var logPersonalInformation = + let logPersonalInformation = this.logPersonalInformation instanceof BoolExpression ? this.logPersonalInformation.getValue(dialogContext.state) : this.logPersonalInformation; - if (logPersonalInformation == undefined) - logPersonalInformation = false; + if (logPersonalInformation == undefined) logPersonalInformation = false; if (logPersonalInformation) { properties['Text'] = recognizerResult.text; @@ -374,8 +370,7 @@ export class OrchestratorAdaptiveRecognizer const orchestrator = new oc.Orchestrator(); if (this.scoreEntities && !orchestrator.load(fullModelFolder, entityModelFolder)) { throw new Error(`Model load failed.`); - } - else if (!orchestrator.load(fullModelFolder)) { + } else if (!orchestrator.load(fullModelFolder)) { throw new Error(`Model load failed.`); } OrchestratorAdaptiveRecognizer.orchestrator = orchestrator; @@ -427,8 +422,8 @@ export class OrchestratorAdaptiveRecognizer score: result.score, text: entityText, start: span.offset, - end: span.offset + span.length - }); + end: span.offset + span.length, + }); // get/create $instance let instanceRoot: any = recognizerResult.entities['$instance']; From 44fef572653126d35d901688ea8b52d51788dc2e Mon Sep 17 00:00:00 2001 From: Tien Suwandy Date: Thu, 25 Mar 2021 10:46:24 -0700 Subject: [PATCH 07/18] addressed review comments, fixed some issues from yarn lint --- .../src/orchestratorAdaptiveRecognizer.ts | 52 +++++-------------- .../tests/mockResolver.js | 16 +++--- .../orchestratorAdaptiveRecognizer.test.js | 42 +++++++-------- 3 files changed, 42 insertions(+), 68 deletions(-) diff --git a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts index 187fe0df99..cc11a029be 100644 --- a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts +++ b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts @@ -45,30 +45,18 @@ type Label = { span: { offset: number; length: number; - } + }; }; -type Result = { +type Result = { score: number; closest_text: string; label: Label; -} +}; type LabelResolver = { - score( - text: string - ): { - score: number; - closest_text: string; - label: { - name: string; - }; - }[]; - - score( - text: string, - labelType: number, - ): Result[]; + score(text: string): Result[]; + score(text: string, labelType: number): Result[]; }; type Orchestrator = { @@ -112,7 +100,7 @@ export class OrchestratorAdaptiveRecognizer /** * Enable entity detection if entity model exists inside modelFolder. Defaults to false. */ - public scoreEntities: boolean = false; + public scoreEntities = false; /** * Intent name if ambiguous intents are detected. @@ -227,7 +215,6 @@ export class OrchestratorAdaptiveRecognizer recognizerResult[this.resultProperty] = results; if (results.length) { - const topScoringIntent = results[0].label.name; const topScore = results[0].score; // if top scoring intent is less than threshold, return None @@ -235,11 +222,10 @@ export class OrchestratorAdaptiveRecognizer recognizerResult.intents.None = { score: 1.0 }; } else { // add all scores - recognizerResult.intents = results.reduce(function(intents, result){ + recognizerResult.intents = results.reduce(function(intents, result) { intents[result.label.name] = { score: result.score }; return intents; }, {}); - // disambiguate if (detectAmbiguity) { const disambiguationScoreThreshold = this.disambiguationScoreThreshold.getValue(dc.state); @@ -291,9 +277,7 @@ export class OrchestratorAdaptiveRecognizer return recognizerResult; } - private getTopTwoIntents( - result: RecognizerResult - ): { name: string; score: number }[] { + private getTopTwoIntents(result: RecognizerResult): { name: string; score: number }[] { if (!result || !result.intents) { throw new Error('result is empty'); } @@ -407,18 +391,15 @@ export class OrchestratorAdaptiveRecognizer // Add full entity recognition result as a 'result' property recognizerResult[this.entityProperty] = results; if (results.length) { - if (recognizerResult.entities === null) { - recognizerResult.entities = {}; - } + recognizerResult.entities ??= {}; results.forEach((result: Result) => { const entityType = result.label.name; // add value let values: any[] = recognizerResult.entities[entityType]; - if (!values) { - values = recognizerResult.entities[entityType] = []; - } + values ??= []; + recognizerResult.entities[entityType] = values; const span = result.label.span; const entityText = text.substr(span.offset, span.length); @@ -431,17 +412,10 @@ export class OrchestratorAdaptiveRecognizer }); // get/create $instance - let instanceRoot: any = recognizerResult.entities['$instance']; - if (!instanceRoot) { - instanceRoot = recognizerResult.entities['$instance'] = {}; - } + const instanceRoot = recognizerResult.entities['$instance'] ?? {}; // add instanceData - let instanceData: any[] = instanceRoot[entityType]; - if (!instanceData) { - instanceData = instanceRoot[entityType] = []; - } - + const instanceData = instanceRoot[entityType] ?? []; instanceData.push({ startIndex: span.offset, endIndex: span.offset + span.length, diff --git a/libraries/botbuilder-ai-orchestrator/tests/mockResolver.js b/libraries/botbuilder-ai-orchestrator/tests/mockResolver.js index 22f31806fb..7e4cc78919 100644 --- a/libraries/botbuilder-ai-orchestrator/tests/mockResolver.js +++ b/libraries/botbuilder-ai-orchestrator/tests/mockResolver.js @@ -5,7 +5,7 @@ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ -const { OrchestratorAdaptiveRecognizer, LabelType } = require('../lib'); +const { LabelType } = require('../lib'); class MockResolver { constructor(score, entityScore) { this._score = score; @@ -13,14 +13,14 @@ class MockResolver { } score(_text, labelType = LabelType.Intent) { - if (labelType === LabelType.Intent) { - return this._score; + switch (labelType) { + case LabelType.Intent: + return this._score; + case LabelType.Entity: + return this._entityScore; + default: + throw new Error('Label type not supported!'); } - else if (labelType === LabelType.Entity) { - return this._entityScore; - } - - throw new Error('Not supported!'); } } diff --git a/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js b/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js index 98ed528604..5f48f55177 100644 --- a/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js +++ b/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js @@ -3,7 +3,7 @@ * Licensed under the MIT License. */ const { MockResolver, TestAdapterSettings } = require('./mockResolver'); -const { ok, rejects } = require('assert'); +const { ok, rejects, strictEqual } = require('assert'); const { OrchestratorAdaptiveRecognizer, LabelType } = require('../lib'); const { DialogContext, DialogSet } = require('botbuilder-dialogs'); const { TurnContext, MessageFactory, NullTelemetryClient } = require('botbuilder-core'); @@ -33,9 +33,9 @@ describe('OrchestratorAdaptiveRecognizer tests', function () { const { dc, activity } = createTestDcAndActivity('hello'); const res = await rec.recognize(dc, activity); - ok(res.text, 'hello'); - ok(res.intents.mockLabel.score, 0.9); - ok(rec._initializeModel.calledOnce); + strictEqual(res.text, 'hello'); + strictEqual(res.intents.mockLabel.score, 0.9); + strictEqual(rec._initializeModel.calledOnce); }); it('Expect initialize is called when labelresolver is null', async () => { @@ -69,8 +69,8 @@ describe('OrchestratorAdaptiveRecognizer tests', function () { const { dc, activity } = createTestDcAndActivity('hello'); const res = await rec.recognize(dc, activity); - ok(res.text, 'hello'); - ok(res.intents.mockLabel.score, 0.9); + strictEqual(res.text, 'hello'); + strictEqual(res.intents.mockLabel.score, 0.9); }); it('Test entity recognition', async () => { @@ -85,12 +85,12 @@ describe('OrchestratorAdaptiveRecognizer tests', function () { const entityResult = [ { score: 0.75, - label: { - name: 'mockEntityLabel', - type: LabelType.Entity, - span: { - offset: 17, - length: 7 + label: { + name: 'mockEntityLabel', + type: LabelType.Entity, + span: { + offset: 17, + length: 7, }, }, }, @@ -106,16 +106,16 @@ describe('OrchestratorAdaptiveRecognizer tests', function () { const { dc, activity } = createTestDcAndActivity('turn on light in room 12'); const res = await rec.recognize(dc, activity); - ok(res.text, 'turn on light in room 12'); - ok(res.intents.mockLabel.score, 0.9); - ok(res.entities.number[0], '12'); + strictEqual(res.text, 'turn on light in room 12'); + strictEqual(res.intents.mockLabel.score, 0.9); + strictEqual(res.entities.number[0], '12'); - ok(res['entityResult'] = entityResult); - ok(res.entities.mockEntityLabel); - ok(res.entities.mockEntityLabel[0].score, 0.75); - ok(res.entities.mockEntityLabel[0].text, 'room 12'); - ok(res.entities.mockEntityLabel[0].start, 17); - ok(res.entities.mockEntityLabel[0].end, 24); + strictEqual(res['entityResult'], entityResult); + strictEqual(res.entities.mockEntityLabel); + strictEqual(res.entities.mockEntityLabel[0].score, 0.75); + strictEqual(res.entities.mockEntityLabel[0].text, 'room 12'); + strictEqual(res.entities.mockEntityLabel[0].start, 17); + strictEqual(res.entities.mockEntityLabel[0].end, 24); }); it('Test ambiguous intent recognition', async () => { From a44aee71be4a4c06ca2a0788fad8db48de57d857 Mon Sep 17 00:00:00 2001 From: Tien Suwandy Date: Thu, 25 Mar 2021 10:57:52 -0700 Subject: [PATCH 08/18] fixed tests --- .../src/orchestratorAdaptiveRecognizer.ts | 12 ++++++------ .../tests/orchestratorAdaptiveRecognizer.test.js | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts index 0c4a955e49..7610661ad5 100644 --- a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts +++ b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts @@ -46,22 +46,22 @@ interface Label { offset: number; length: number; }; -}; +} interface Result { score: number; closest_text: string; label: Label; -}; +} interface LabelResolver { score(text: string): Result[]; score(text: string, labelType: number): Result[]; -}; +} -type Orchestrator = { +interface Orchestrator { createLabelResolver(snapshot: Uint8Array): LabelResolver; -}; +} /** * Class that represents an adaptive Orchestrator recognizer. @@ -398,7 +398,7 @@ export class OrchestratorAdaptiveRecognizer const entityType = result.label.name; // add value - let values: any[] = recognizerResult.entities[entityType]; + let values = recognizerResult.entities[entityType]; values ??= []; recognizerResult.entities[entityType] = values; diff --git a/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js b/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js index 5f48f55177..9f741ed630 100644 --- a/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js +++ b/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js @@ -35,7 +35,7 @@ describe('OrchestratorAdaptiveRecognizer tests', function () { strictEqual(res.text, 'hello'); strictEqual(res.intents.mockLabel.score, 0.9); - strictEqual(rec._initializeModel.calledOnce); + ok(rec._initializeModel.calledOnce); }); it('Expect initialize is called when labelresolver is null', async () => { @@ -111,7 +111,7 @@ describe('OrchestratorAdaptiveRecognizer tests', function () { strictEqual(res.entities.number[0], '12'); strictEqual(res['entityResult'], entityResult); - strictEqual(res.entities.mockEntityLabel); + ok(res.entities.mockEntityLabel); strictEqual(res.entities.mockEntityLabel[0].score, 0.75); strictEqual(res.entities.mockEntityLabel[0].text, 'room 12'); strictEqual(res.entities.mockEntityLabel[0].start, 17); From 038dfe325da3251390caf0e306c35c6043841bbb Mon Sep 17 00:00:00 2001 From: Tien Suwandy Date: Thu, 25 Mar 2021 11:06:04 -0700 Subject: [PATCH 09/18] yarn lint fixes --- .../tests/recognizerTelemetryUtils.js | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/libraries/botbuilder-ai-orchestrator/tests/recognizerTelemetryUtils.js b/libraries/botbuilder-ai-orchestrator/tests/recognizerTelemetryUtils.js index 44ea2cf1c5..8ee86e894b 100644 --- a/libraries/botbuilder-ai-orchestrator/tests/recognizerTelemetryUtils.js +++ b/libraries/botbuilder-ai-orchestrator/tests/recognizerTelemetryUtils.js @@ -15,11 +15,11 @@ const spyOnTelemetryClientTrackEvent = (recognizer) => { }; const getLogPersonalInformation = (recognizer, dialogContext) => { - var result = recognizer.logPersonalInformation instanceof BoolExpression - ? recognizer.logPersonalInformation.getValue(dialogContext.state) - : recognizer.logPersonalInformation; - if (result == undefined) - return false; + const result = + recognizer.logPersonalInformation instanceof BoolExpression + ? recognizer.logPersonalInformation.getValue(dialogContext.state) + : recognizer.logPersonalInformation; + if (result == undefined) return false; return result; }; @@ -41,21 +41,22 @@ module.exports = { }; // **** PRIVATE **** // - const getOrchestratorIntentProps = () => ({ TopIntent: 'mockLabel', TopIntentScore: '0.9', NextIntent: 'mockLabel2', NextIntentScore: '0.8', - Intents: JSON.stringify({ mockLabel: { score: 0.9 }, mockLabel2: { score: 0.8 } } ), + Intents: JSON.stringify({ mockLabel: { score: 0.9 }, mockLabel2: { score: 0.8 } }), Entities: '{}', AdditionalProperties: JSON.stringify({ - result: [{ score: 0.9, label: { name: 'mockLabel' } },{ score: 0.8, label: { name: 'mockLabel2' } } ], + result: [ + { score: 0.9, label: { name: 'mockLabel' } }, + { score: 0.8, label: { name: 'mockLabel2' } }, + ], }), }); const getExpectedProps = (activity, result, logPersonalInformation) => { - const text = asMessageActivity(activity).text; const expectedProps = text === orchestratorIntentText ? getOrchestratorIntentProps() : {}; @@ -68,7 +69,6 @@ const getExpectedProps = (activity, result, logPersonalInformation) => { }; const hasValidTelemetryProps = (actual, expected) => { - if (Object.keys(actual).length !== Object.keys(expected).length) { return false; } From bd50f33369e6d0b06e7f13b2788090262355b040 Mon Sep 17 00:00:00 2001 From: Tien Suwandy Date: Thu, 25 Mar 2021 11:11:47 -0700 Subject: [PATCH 10/18] addressed more PR comments --- .../src/orchestratorAdaptiveRecognizer.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts index 7610661ad5..cb7eb127e4 100644 --- a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts +++ b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts @@ -55,8 +55,7 @@ interface Result { } interface LabelResolver { - score(text: string): Result[]; - score(text: string, labelType: number): Result[]; + score(text: string, labelType?: number): Result[]; } interface Orchestrator { @@ -398,8 +397,7 @@ export class OrchestratorAdaptiveRecognizer const entityType = result.label.name; // add value - let values = recognizerResult.entities[entityType]; - values ??= []; + const values = recognizerResult.entities[entityType] ?? []; recognizerResult.entities[entityType] = values; const span = result.label.span; From c1f1297743e3959c1d042b34ba6b7f4f7814039e Mon Sep 17 00:00:00 2001 From: Dave Taniguchi Date: Thu, 25 Mar 2021 11:19:37 -0700 Subject: [PATCH 11/18] Updates --- .../src/orchestratorAdaptiveRecognizer.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts index cb7eb127e4..2c60518325 100644 --- a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts +++ b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts @@ -305,16 +305,17 @@ export class OrchestratorAdaptiveRecognizer dialogContext?: DialogContext ): Record { const topTwo = this.getTopTwoIntents(recognizerResult); - const intents = Object.entries(recognizerResult.intents); + const intent = Object.entries(recognizerResult.intents); + const { text, alteredText, intents, entities, ...customRecognizerProps } = recognizerResult; const properties: Record = { - TopIntent: intents.length > 0 ? topTwo[0].name : undefined, - TopIntentScore: intents.length > 0 ? topTwo[0].score.toString() : undefined, - NextIntent: intents.length > 1 ? topTwo[1].name : undefined, - NextIntentScore: intents.length > 1 ? topTwo[1].score.toString() : undefined, - Intents: intents.length > 0 ? JSON.stringify(recognizerResult.intents) : undefined, + TopIntent: intent.length > 0 ? topTwo[0].name : undefined, + TopIntentScore: intent.length > 0 ? topTwo[0].score.toString() : undefined, + NextIntent: intent.length > 1 ? topTwo[1].name : undefined, + NextIntentScore: intent.length > 1 ? topTwo[1].score.toString() : undefined, + Intents: intent.length > 0 ? JSON.stringify(recognizerResult.intents) : undefined, Entities: recognizerResult.entities ? JSON.stringify(recognizerResult.entities) : undefined, - AdditionalProperties: JSON.stringify( - omit(recognizerResult, ['text', 'alteredText', 'intents', 'entities']) + AdditionalProperties: JSON.stringify( + customRecognizerProps ), }; From 1c74d5ad62897772366181dd4de246d8c84a1d70 Mon Sep 17 00:00:00 2001 From: Dave Taniguchi Date: Thu, 25 Mar 2021 11:21:23 -0700 Subject: [PATCH 12/18] Remove lodash/omit --- libraries/botbuilder-ai-orchestrator/package.json | 1 - .../src/orchestratorAdaptiveRecognizer.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/libraries/botbuilder-ai-orchestrator/package.json b/libraries/botbuilder-ai-orchestrator/package.json index d004c91349..18501f3b3e 100644 --- a/libraries/botbuilder-ai-orchestrator/package.json +++ b/libraries/botbuilder-ai-orchestrator/package.json @@ -33,7 +33,6 @@ "botbuilder-dialogs-adaptive": "4.1.6", "botbuilder-dialogs-declarative": "4.1.6", "orchestrator-core": "4.12.0-preview", - "lodash": "^4.17.21", "uuid": "^8.3.2" }, "scripts": { diff --git a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts index 2c60518325..c6019e1c55 100644 --- a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts +++ b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts @@ -9,7 +9,7 @@ import { existsSync, readFileSync } from 'fs'; import { resolve } from 'path'; import { v4 as uuidv4 } from 'uuid'; -import omit from 'lodash/omit'; + import { BoolExpression, BoolExpressionConverter, From 17382f725571e72523d41e7771efd17716ece3a6 Mon Sep 17 00:00:00 2001 From: Dave Taniguchi Date: Thu, 25 Mar 2021 11:44:38 -0700 Subject: [PATCH 13/18] Lint issues --- .../src/orchestratorAdaptiveRecognizer.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts index c6019e1c55..2566b2e2bc 100644 --- a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts +++ b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts @@ -306,6 +306,7 @@ export class OrchestratorAdaptiveRecognizer ): Record { const topTwo = this.getTopTwoIntents(recognizerResult); const intent = Object.entries(recognizerResult.intents); + // eslint-disable-next-line @typescript-eslint/no-unused-vars const { text, alteredText, intents, entities, ...customRecognizerProps } = recognizerResult; const properties: Record = { TopIntent: intent.length > 0 ? topTwo[0].name : undefined, @@ -314,9 +315,7 @@ export class OrchestratorAdaptiveRecognizer NextIntentScore: intent.length > 1 ? topTwo[1].score.toString() : undefined, Intents: intent.length > 0 ? JSON.stringify(recognizerResult.intents) : undefined, Entities: recognizerResult.entities ? JSON.stringify(recognizerResult.entities) : undefined, - AdditionalProperties: JSON.stringify( - customRecognizerProps - ), + AdditionalProperties: JSON.stringify(customRecognizerProps), }; let logPersonalInformation = From c4f1516ea4326cb6ae5f87e50af708de36babed8 Mon Sep 17 00:00:00 2001 From: Dave Taniguchi Date: Thu, 25 Mar 2021 13:23:01 -0700 Subject: [PATCH 14/18] Add comment for omit --- .../src/orchestratorAdaptiveRecognizer.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts index 2566b2e2bc..82424f9bd5 100644 --- a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts +++ b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts @@ -306,6 +306,8 @@ export class OrchestratorAdaptiveRecognizer ): Record { const topTwo = this.getTopTwoIntents(recognizerResult); const intent = Object.entries(recognizerResult.intents); + // customRecognizerProps = recognizerResult with following properties omitted: + // text, alteredText, intents, entities // eslint-disable-next-line @typescript-eslint/no-unused-vars const { text, alteredText, intents, entities, ...customRecognizerProps } = recognizerResult; const properties: Record = { From 5567d317cba8e81ccbb9dd55af4f27d956e36038 Mon Sep 17 00:00:00 2001 From: Dave Taniguchi Date: Thu, 25 Mar 2021 13:51:41 -0700 Subject: [PATCH 15/18] Fix test:orchestator command --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b50c85b37e..b9aa30eef5 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "test:consumer": "yarn workspace consumer-test test", "test:devops": "npm-run-all test:teams test:mocha:junit test:nyc:cobertura", "test:github": "npm-run-all test:mocha:min test:runtime:min test:teams:min test:nyc:lcov", - "test:orchestrator": "wsrun -m -p \"botbuilder-ai-orchestrator\" -t test", + "test:orchestrator": "yarn workspace botbuilder-ai-orchestrator test", "test:runtime": "wsrun -m -p \"botbuilder-runtime*\" -t test", "test:runtime:min": "wsrun -m -p \"botbuilder-runtime*\" -t test:min", "test:teams": "yarn workspace botbuilder-dialogs-adaptive-teams test", From 925732d04a286754fc941a41cfd3bcffafce1d9a Mon Sep 17 00:00:00 2001 From: Dave Taniguchi Date: Thu, 25 Mar 2021 15:31:14 -0700 Subject: [PATCH 16/18] Update package to latest R13. We'll update to rc and full R13 build later --- libraries/botbuilder-ai-orchestrator/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/botbuilder-ai-orchestrator/package.json b/libraries/botbuilder-ai-orchestrator/package.json index 18501f3b3e..09cd82bb4d 100644 --- a/libraries/botbuilder-ai-orchestrator/package.json +++ b/libraries/botbuilder-ai-orchestrator/package.json @@ -32,7 +32,7 @@ "botbuilder-dialogs": "4.1.6", "botbuilder-dialogs-adaptive": "4.1.6", "botbuilder-dialogs-declarative": "4.1.6", - "orchestrator-core": "4.12.0-preview", + "orchestrator-core": "next", "uuid": "^8.3.2" }, "scripts": { From 7b2da8eb4c5a61d43728e133c51f38c066053c63 Mon Sep 17 00:00:00 2001 From: Tien Suwandy Date: Thu, 25 Mar 2021 16:57:57 -0700 Subject: [PATCH 17/18] updated per PR comments, added tests --- .../src/orchestratorAdaptiveRecognizer.ts | 8 +++++--- .../tests/orchestratorAdaptiveRecognizer.test.js | 3 +++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts index 82424f9bd5..602b79c1bd 100644 --- a/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts +++ b/libraries/botbuilder-ai-orchestrator/src/orchestratorAdaptiveRecognizer.ts @@ -390,7 +390,7 @@ export class OrchestratorAdaptiveRecognizer throw new Error(`Failed scoring entities for: ${text}`); } - // Add full entity recognition result as a 'result' property + // Add full entity recognition result as a 'entityResult' property recognizerResult[this.entityProperty] = results; if (results.length) { recognizerResult.entities ??= {}; @@ -413,10 +413,12 @@ export class OrchestratorAdaptiveRecognizer }); // get/create $instance - const instanceRoot = recognizerResult.entities['$instance'] ?? {}; + recognizerResult.entities['$instance'] ??= {}; + const instanceRoot = recognizerResult.entities['$instance']; // add instanceData - const instanceData = instanceRoot[entityType] ?? []; + instanceRoot[entityType] ??= []; + const instanceData = instanceRoot[entityType]; instanceData.push({ startIndex: span.offset, endIndex: span.offset + span.length, diff --git a/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js b/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js index 9f741ed630..00f5072d14 100644 --- a/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js +++ b/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js @@ -116,6 +116,9 @@ describe('OrchestratorAdaptiveRecognizer tests', function () { strictEqual(res.entities.mockEntityLabel[0].text, 'room 12'); strictEqual(res.entities.mockEntityLabel[0].start, 17); strictEqual(res.entities.mockEntityLabel[0].end, 24); + strictEqual(Object.keys(res.entities).length, 3); + strictEqual(Object.keys(res.entities.$instance).length, 2); + console.log('ENTITIES ' + JSON.stringify(res.entities)); }); it('Test ambiguous intent recognition', async () => { From 678f192913f96b2765e44ff713314ad3b0f1d191 Mon Sep 17 00:00:00 2001 From: Tien Suwandy Date: Fri, 26 Mar 2021 11:43:17 -0700 Subject: [PATCH 18/18] updated schema --- ...r.schema => Microsoft.OrchestratorAdaptiveRecognizer.schema} | 0 libraries/tests.schema | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename libraries/botbuilder-ai-orchestrator/schemas/{Microsoft.OrchestratorRecognizer.schema => Microsoft.OrchestratorAdaptiveRecognizer.schema} (100%) diff --git a/libraries/botbuilder-ai-orchestrator/schemas/Microsoft.OrchestratorRecognizer.schema b/libraries/botbuilder-ai-orchestrator/schemas/Microsoft.OrchestratorAdaptiveRecognizer.schema similarity index 100% rename from libraries/botbuilder-ai-orchestrator/schemas/Microsoft.OrchestratorRecognizer.schema rename to libraries/botbuilder-ai-orchestrator/schemas/Microsoft.OrchestratorAdaptiveRecognizer.schema diff --git a/libraries/tests.schema b/libraries/tests.schema index 7f43b1fdec..310ab3ec15 100644 --- a/libraries/tests.schema +++ b/libraries/tests.schema @@ -7313,7 +7313,7 @@ } }, "Microsoft.OrchestratorAdaptiveRecognizer": { - "$role": "implements(Microsoft.OrchestratorAdaptiveRecognizerConfiguration)", + "$role": "implements(Microsoft.IRecognizer)", "title": "Orchestrator recognizer", "description": "Orchestrator recognizer.", "type": "object",