diff --git a/libraries/botbuilder-ai-orchestrator/etc/botbuilder-ai-orchestrator.api.md b/libraries/botbuilder-ai-orchestrator/etc/botbuilder-ai-orchestrator.api.md index b4bc1e3436..77188c0319 100644 --- a/libraries/botbuilder-ai-orchestrator/etc/botbuilder-ai-orchestrator.api.md +++ b/libraries/botbuilder-ai-orchestrator/etc/botbuilder-ai-orchestrator.api.md @@ -45,13 +45,13 @@ export class OrchestratorRecognizer extends AdaptiveRecognizer implements Orches // (undocumented) static $kind: string; // Warning: (ae-forgotten-export) The symbol "LabelResolver" needs to be exported by the entry point index.d.ts - constructor(modelFolder?: string, snapshotFile?: string, resolver?: LabelResolver); + constructor(modelFolder?: string, snapshotFile?: string, resolverExternal?: LabelResolver); readonly chooseIntent = "ChooseIntent"; detectAmbiguousIntents: BoolExpression; disambiguationScoreThreshold: NumberExpression; readonly entityProperty = "entityResult"; - externalEntityRecognizer: Recognizer; - protected fillRecognizerResultTelemetryProperties(recognizerResult: RecognizerResult, telemetryProperties: Record, dialogContext?: DialogContext): Record; + externalEntityRecognizer?: Recognizer; + protected fillRecognizerResultTelemetryProperties(recognizerResult: RecognizerResult, telemetryProperties?: Record, dialogContext?: DialogContext): Record; // (undocumented) getConverter(property: keyof OrchestratorRecognizerConfiguration): Converter | ConverterFactory; modelFolder: StringExpression; diff --git a/libraries/botbuilder-ai-orchestrator/src/orchestratorRecognizer.ts b/libraries/botbuilder-ai-orchestrator/src/orchestratorRecognizer.ts index 761132000c..b83f41a4b1 100644 --- a/libraries/botbuilder-ai-orchestrator/src/orchestratorRecognizer.ts +++ b/libraries/botbuilder-ai-orchestrator/src/orchestratorRecognizer.ts @@ -62,6 +62,11 @@ interface Orchestrator { createLabelResolver(snapshot: Uint8Array): LabelResolver; } +interface OrchestratorDictionaryEntry { + orchestrator: Orchestrator; + isEntityExtractionCapable: boolean; +} + /** * Class that represents an adaptive Orchestrator recognizer. */ @@ -92,12 +97,13 @@ export class OrchestratorRecognizer extends AdaptiveRecognizer implements Orches /** * The external entity recognizer. */ - externalEntityRecognizer: Recognizer; + externalEntityRecognizer?: Recognizer = undefined; /** - * Enable entity detection if entity model exists inside modelFolder. Defaults to false. + * Enable entity detection if entity model exists inside modelFolder. Defaults to true. + * NOTE: SHOULD consider removing this flag in the next major SDK release (V5). */ - scoreEntities = false; + scoreEntities = true; /** * Intent name if ambiguous intents are detected. @@ -130,27 +136,24 @@ export class OrchestratorRecognizer extends AdaptiveRecognizer implements Orches } private readonly unknownIntentFilterScore = 0.4; - private static orchestrator: Orchestrator; - private _resolver: LabelResolver; - private _modelFolder: string; - private _snapshotFile: string; + private static orchestratorMap = new Map(); + private _orchestrator?: OrchestratorDictionaryEntry = undefined; + private _resolver?: LabelResolver = undefined; + private _isResolverExternal = false; /** * Returns an OrchestratorRecognizer instance. * * @param {string} modelFolder Path to NLR model. * @param {string} snapshotFile Path to snapshot. - * @param {any} resolver Orchestrator resolver to use. + * @param {any} resolverExternal Orchestrator resolver to use. */ - constructor(modelFolder?: string, snapshotFile?: string, resolver?: LabelResolver) { + constructor(modelFolder?: string, snapshotFile?: string, resolverExternal?: LabelResolver) { super(); - if (modelFolder) { - this._modelFolder = modelFolder; - } - if (snapshotFile) { - this._snapshotFile = snapshotFile; + + if ((modelFolder && snapshotFile) || resolverExternal) { + this._initializeModel(modelFolder, snapshotFile, resolverExternal); } - this._resolver = resolver; } /** @@ -168,17 +171,13 @@ export class OrchestratorRecognizer extends AdaptiveRecognizer implements Orches telemetryProperties?: Record, telemetryMetrics?: Record ): Promise { - if (!this.modelFolder) { - throw new Error(`Missing "ModelFolder" information.`); - } - - if (!this.snapshotFile) { - throw new Error(`Missing "SnapshotFile" information.`); + if (!this._resolver) { + const modelFolder: string = this.modelFolder.getValue(dc.state); + const snapshotFile: string = this.snapshotFile.getValue(dc.state); + this._initializeModel(modelFolder, snapshotFile, undefined); } const text = activity.text ?? ''; - this._modelFolder = this.modelFolder.getValue(dc.state); - this._snapshotFile = this.snapshotFile.getValue(dc.state); const detectAmbiguity = this.detectAmbiguousIntents.getValue(dc.state); let recognizerResult: RecognizerResult = { @@ -187,31 +186,18 @@ export class OrchestratorRecognizer extends AdaptiveRecognizer implements Orches entities: {}, }; - if (text === '') { + if (!text) { // nothing to recognize, return empty result. return recognizerResult; } - this._initializeModel(); - - if (this.externalEntityRecognizer) { - // Run external recognition - const externalResults = await this.externalEntityRecognizer.recognize( - dc, - activity, - telemetryProperties, - telemetryMetrics - ); - recognizerResult.entities = externalResults.entities; - } - // Score with orchestrator - const results = await this._resolver.score(text); + const results = await this._resolver?.score(text); - // Add full recognition result as a 'result' property - recognizerResult[this.resultProperty] = results; + if (results?.length) { + // Add full recognition result as a 'result' property + recognizerResult[this.resultProperty] = results; - if (results.length) { const topScore = results[0].score; // if top scoring intent is less than threshold, return None @@ -219,10 +205,14 @@ export class OrchestratorRecognizer extends AdaptiveRecognizer implements Orches recognizerResult.intents.None = { score: 1.0 }; } else { // add all scores - recognizerResult.intents = results.reduce(function (intents, result) { + recognizerResult.intents = results.reduce(function ( + intents: { [index: string]: { score: number } }, + result + ) { intents[result.label.name] = { score: result.score }; return intents; - }, {}); + }, + {}); // disambiguate if (detectAmbiguity) { @@ -257,7 +247,18 @@ export class OrchestratorRecognizer extends AdaptiveRecognizer implements Orches recognizerResult.intents.None = { score: 1.0 }; } - await this.tryScoreEntities(text, recognizerResult); + if (this.externalEntityRecognizer) { + // Run external recognition + const externalResults = await this.externalEntityRecognizer.recognize( + dc, + activity, + telemetryProperties, + telemetryMetrics + ); + recognizerResult.entities = externalResults.entities; + } + + await this.tryScoreEntitiesAsync(text, recognizerResult); await dc.context.sendTraceActivity( 'OrchestratorRecognizer', @@ -281,7 +282,11 @@ export class OrchestratorRecognizer extends AdaptiveRecognizer implements Orches } const intents = Object.entries(result.intents) .map((intent) => { - return { name: intent[0], score: +intent[1].score }; + let score = 0; + if (intent[1].score) { + score = intent[1].score; + } + return { name: intent[0], score: +score }; }) .sort((a, b) => b.score - a.score); intents.length = 2; @@ -299,7 +304,7 @@ export class OrchestratorRecognizer extends AdaptiveRecognizer implements Orches */ protected fillRecognizerResultTelemetryProperties( recognizerResult: RecognizerResult, - telemetryProperties: Record, + telemetryProperties?: Record, dialogContext?: DialogContext ): Record { const topTwo = this.getTopTwoIntents(recognizerResult); @@ -309,12 +314,12 @@ export class OrchestratorRecognizer extends AdaptiveRecognizer implements Orches // eslint-disable-next-line @typescript-eslint/no-unused-vars const { text, alteredText, intents, entities, ...customRecognizerProps } = recognizerResult; const properties: Record = { - TopIntent: intent.length > 0 ? topTwo[0].name : undefined, - TopIntentScore: intent.length > 0 ? topTwo[0].score.toString() : undefined, - NextIntent: intent.length > 1 ? topTwo[1].name : undefined, - NextIntentScore: intent.length > 1 ? topTwo[1].score.toString() : undefined, - Intents: intent.length > 0 ? JSON.stringify(recognizerResult.intents) : undefined, - Entities: recognizerResult.entities ? JSON.stringify(recognizerResult.entities) : undefined, + TopIntent: intent.length > 0 ? topTwo[0].name : '', + TopIntentScore: intent.length > 0 ? topTwo[0].score.toString() : '', + NextIntent: intent.length > 1 ? topTwo[1].name : '', + NextIntentScore: intent.length > 1 ? topTwo[1].score.toString() : '', + Intents: intent.length > 0 ? JSON.stringify(recognizerResult.intents) : '', + Entities: recognizerResult.entities ? JSON.stringify(recognizerResult.entities) : '', AdditionalProperties: JSON.stringify(customRecognizerProps), }; @@ -337,100 +342,136 @@ export class OrchestratorRecognizer extends AdaptiveRecognizer implements Orches return properties; } - private _initializeModel() { - if (!this._modelFolder) { + private _initializeModel(modelFolder?: string, snapshotFile?: string, resolverExternal?: LabelResolver): void { + if (this._resolver) { + return; + } + if (resolverExternal) { + this._resolver = resolverExternal; + this._isResolverExternal = true; + return; + } + + if (!modelFolder) { throw new Error(`Missing "ModelFolder" information.`); } - if (!this._snapshotFile) { + if (!snapshotFile) { throw new Error(`Missing "ShapshotFile" information.`); } - if (!OrchestratorRecognizer.orchestrator && !this._resolver) { - // Create orchestrator core - const fullModelFolder = resolve(this._modelFolder); - if (!existsSync(fullModelFolder)) { - throw new Error(`Model folder does not exist at ${fullModelFolder}.`); - } + // Create orchestrator core + const fullModelFolder: string = resolve(modelFolder); + + this._orchestrator = OrchestratorRecognizer.orchestratorMap.has(fullModelFolder) + ? OrchestratorRecognizer.orchestratorMap.get(fullModelFolder) + : ((): OrchestratorDictionaryEntry => { + if (!existsSync(fullModelFolder)) { + throw new Error(`Model folder does not exist at ${fullModelFolder}.`); + } + const entityModelFolder: string = resolve(modelFolder, 'entity'); + const isEntityExtractionCapable: boolean = existsSync(entityModelFolder); + const orchestrator = new oc.Orchestrator(); + if (isEntityExtractionCapable) { + if (!orchestrator.load(fullModelFolder, entityModelFolder)) { + throw new Error( + `Model load failed - model folder ${fullModelFolder}, entity model folder ${entityModelFolder}.` + ); + } + } else { + if (!orchestrator.load(fullModelFolder)) { + throw new Error(`Model load failed - model folder ${fullModelFolder}.`); + } + } + const orchestratorDictionaryEntry: OrchestratorDictionaryEntry = { + orchestrator, + isEntityExtractionCapable, + }; + OrchestratorRecognizer.orchestratorMap.set(fullModelFolder, orchestratorDictionaryEntry); + return orchestratorDictionaryEntry; + })(); + + const fullSnapshotPath = resolve(snapshotFile); + if (!existsSync(fullSnapshotPath)) { + throw new Error(`Snapshot file does not exist at ${fullSnapshotPath}.`); + } - const entityModelFolder = resolve(this._modelFolder, 'entity'); - this.scoreEntities = existsSync(entityModelFolder); + // Load the snapshot + const snapshot: Uint8Array = readFileSync(fullSnapshotPath); - const orchestrator = new oc.Orchestrator(); - if (this.scoreEntities) { - if (!orchestrator.load(fullModelFolder, entityModelFolder)) { - throw new Error( - `Model load failed - model folder ${fullModelFolder}, entity model folder ${entityModelFolder}.` - ); - } - } else { - if (!orchestrator.load(fullModelFolder)) { - throw new Error(`Model load failed - model folder ${fullModelFolder}.`); - } - } - OrchestratorRecognizer.orchestrator = orchestrator; - } + // Load snapshot and create resolver + this._resolver = this._orchestrator?.orchestrator.createLabelResolver(snapshot); + } + private async tryScoreEntitiesAsync(text: string, recognizerResult: RecognizerResult) { + // It's impossible to extract entities without a _resolver object. if (!this._resolver) { - const fullSnapshotPath = resolve(this._snapshotFile); - if (!existsSync(fullSnapshotPath)) { - throw new Error(`Snapshot file does not exist at ${fullSnapshotPath}.`); - } - // Load the snapshot - const snapshot: Uint8Array = readFileSync(fullSnapshotPath); - - // Load snapshot and create resolver - this._resolver = OrchestratorRecognizer.orchestrator.createLabelResolver(snapshot); + return; } - } - private async tryScoreEntities(text: string, recognizerResult: RecognizerResult) { + // Entity extraction can be controlled by the ScoreEntities flag. + // NOTE: SHOULD consider removing this flag in the next major SDK release (V5). if (!this.scoreEntities) { return; } - const results = await this._resolver.score(text, LabelType.Entity); - if (!results) { - throw new Error(`Failed scoring entities for: ${text}`); + // The following check is necessary to ensure that the _resolver object + // is capable of entity extraction. However, this check does not apply to + // an external, mock-up _resolver. + if (!this._isResolverExternal) { + if (!this._orchestrator || !this._orchestrator.isEntityExtractionCapable) { + return; + } } - // Add full entity recognition result as a 'entityResult' property - recognizerResult[this.entityProperty] = results; - if (results.length) { - recognizerResult.entities ??= {}; - - results.forEach((result: Result) => { - const entityType = result.label.name; - - // add value - const values = recognizerResult.entities[entityType] ?? []; - recognizerResult.entities[entityType] = values; - - const span = result.label.span; - const entityText = text.substr(span.offset, span.length); - values.push({ - type: entityType, - score: result.score, - text: entityText, - start: span.offset, - end: span.offset + span.length, - }); + // As this method is TryScoreEntities, so it's best effort only, there should + // not be any exception thrown out of this method. + try { + const results = await this._resolver.score(text, LabelType.Entity); + if (!results) { + throw new Error(`Failed scoring entities for: ${text}`); + } - // get/create $instance - recognizerResult.entities['$instance'] ??= {}; - const instanceRoot = recognizerResult.entities['$instance']; - - // add instanceData - instanceRoot[entityType] ??= []; - const instanceData = instanceRoot[entityType]; - instanceData.push({ - startIndex: span.offset, - endIndex: span.offset + span.length, - score: result.score, - text: entityText, - type: entityType, + // Add full entity recognition result as a 'entityResult' property + recognizerResult[this.entityProperty] = results; + if (results.length) { + recognizerResult.entities ??= {}; + + results.forEach((result: Result) => { + const entityType = result.label.name; + + // add value + const values = recognizerResult.entities[entityType] ?? []; + recognizerResult.entities[entityType] = values; + + const span = result.label.span; + const entityText = text.substr(span.offset, span.length); + values.push({ + type: entityType, + score: result.score, + text: entityText, + start: span.offset, + end: span.offset + span.length, + }); + + // get/create $instance + recognizerResult.entities['$instance'] ??= {}; + const instanceRoot = recognizerResult.entities['$instance']; + + // add instanceData + instanceRoot[entityType] ??= []; + const instanceData = instanceRoot[entityType]; + instanceData.push({ + startIndex: span.offset, + endIndex: span.offset + span.length, + score: result.score, + text: entityText, + type: entityType, + }); }); - }); + } + } catch { + return; } } } diff --git a/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js b/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js index 6ce19fb0ff..1ba412fa58 100644 --- a/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js +++ b/libraries/botbuilder-ai-orchestrator/tests/orchestratorAdaptiveRecognizer.test.js @@ -14,7 +14,7 @@ const sinon = require('sinon'); const { orchestratorIntentText, getLogPersonalInformation, validateTelemetry } = require('./recognizerTelemetryUtils'); describe('OrchestratorAdpativeRecognizer tests', function () { - it('Expect initialize is called when orchestrator obj is null', async function () { + it('Expect initialize is called when labelresolver is not null', async function () { const result = [ { score: 0.9, @@ -27,28 +27,22 @@ describe('OrchestratorAdpativeRecognizer tests', function () { const testPaths = 'test'; const rec = new OrchestratorRecognizer(testPaths, testPaths, mockResolver); - OrchestratorRecognizer.orchestrator = null; - rec._initializeModel = sinon.fake(); const { dc, activity } = createTestDcAndActivity('hello'); const res = await rec.recognize(dc, activity); strictEqual(res.text, 'hello'); strictEqual(res.intents.mockLabel.score, 0.9); - ok(rec._initializeModel.calledOnce); }); it('Expect initialize is called when labelresolver is null', async function () { - const testPaths = 'test'; - const rec = new OrchestratorRecognizer(testPaths, testPaths, null); - OrchestratorRecognizer.orchestrator = null; - - rec._initializeModel = sinon.fake(); - - const { dc, activity } = createTestDcAndActivity('hello'); - rejects(async () => await rec.recognize(dc, activity)); + rejects(async () => { + const testPaths = 'test'; + const rec = new OrchestratorRecognizer(testPaths, testPaths, null); - ok(rec._initializeModel.calledOnce); + const { dc, activity } = createTestDcAndActivity('hello'); + await rec.recognize(dc, activity); + }); }); it('Test intent recognition', async function () { @@ -63,7 +57,6 @@ describe('OrchestratorAdpativeRecognizer tests', function () { const mockResolver = new MockResolver(result); const testPaths = 'test'; const rec = new OrchestratorRecognizer(testPaths, testPaths, mockResolver); - OrchestratorRecognizer.orchestrator = 'mock'; rec.modelFolder = new StringExpression(testPaths); rec.snapshotFile = new StringExpression(testPaths); const { dc, activity } = createTestDcAndActivity('hello'); @@ -99,7 +92,6 @@ describe('OrchestratorAdpativeRecognizer tests', function () { const testPaths = 'test'; const rec = new OrchestratorRecognizer(testPaths, testPaths, mockResolver); rec.scoreEntities = true; - OrchestratorRecognizer.orchestrator = 'mock'; rec.modelFolder = new StringExpression(testPaths); rec.snapshotFile = new StringExpression(testPaths); rec.externalEntityRecognizer = new NumberEntityRecognizer(); @@ -177,7 +169,6 @@ describe('OrchestratorAdpativeRecognizer tests', function () { const mockResolver = new MockResolver(result); const testPaths = 'test'; const recognizer = new OrchestratorRecognizer(testPaths, testPaths, mockResolver); - OrchestratorRecognizer.orchestrator = 'mock'; recognizer.modelFolder = new StringExpression(testPaths); recognizer.snapshotFile = new StringExpression(testPaths); @@ -221,7 +212,6 @@ describe('OrchestratorAdpativeRecognizer tests', function () { const mockResolver = new MockResolver(result); const testPaths = 'test'; const recognizer = new OrchestratorRecognizer(testPaths, testPaths, mockResolver); - OrchestratorRecognizer.orchestrator = 'mock'; recognizer.modelFolder = new StringExpression(testPaths); recognizer.snapshotFile = new StringExpression(testPaths); @@ -265,7 +255,6 @@ describe('OrchestratorAdpativeRecognizer tests', function () { const mockResolver = new MockResolver(result); const testPaths = 'test'; const recognizerWithDefaultLogPii = new OrchestratorRecognizer(testPaths, testPaths, mockResolver); - OrchestratorRecognizer.orchestrator = 'mock'; recognizerWithDefaultLogPii.modelFolder = new StringExpression(testPaths); recognizerWithDefaultLogPii.snapshotFile = new StringExpression(testPaths);