Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Telemetry LuisRecognizer & Telemetry Middleware #816

Merged
merged 9 commits into from
Mar 18, 2019
102 changes: 94 additions & 8 deletions libraries/botbuilder-ai/src/luisRecognizer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,13 @@
* Licensed under the MIT License.
*/
import { LUISRuntimeClient as LuisClient, LUISRuntimeModels as LuisModels } from 'azure-cognitiveservices-luis-runtime';
import { RecognizerResult, TurnContext } from 'botbuilder-core';
import * as msRest from 'ms-rest';

import { RecognizerResult, TurnContext, BotTelemetryClient, NullTelemetryClient } from 'botbuilder-core';
import * as msRest from "ms-rest";
import * as os from 'os';
import * as Url from 'url-parse';
import { TelemetryConstants } from 'botbuilder-core/lib/telemetryConstants';
import { LuisTelemetryConstants } from './luisTelemetryConstants';

const pjson = require('../package.json');

Expand Down Expand Up @@ -111,6 +114,9 @@ export interface LuisPredictionOptions {
* This component can be used within your bots logic by calling [recognize()](#recognize).
*/
export class LuisRecognizer {
protected telemetryClient: BotTelemetryClient;
protected logPersonalInformation: boolean;

private application: LuisApplication;
private options: LuisPredictionOptions;
private includeApiResults: boolean;
Expand All @@ -124,9 +130,9 @@ export class LuisRecognizer {
* @param options (Optional) options object used to control predictions. Should conform to the [LuisPrectionOptions](#luispredictionoptions) definition.
* @param includeApiResults (Optional) flag that if set to `true` will force the inclusion of LUIS Api call in results returned by [recognize()](#recognize). Defaults to a value of `false`.
*/
constructor(application: string, options?: LuisPredictionOptions, includeApiResults?: boolean);
constructor(application: LuisApplication, options?: LuisPredictionOptions, includeApiResults?: boolean);
constructor(application: LuisApplication|string, options?: LuisPredictionOptions, includeApiResults?: boolean) {
constructor(application: string, options?: LuisPredictionOptions, includeApiResults?: boolean, telemetryClient?: BotTelemetryClient, logPersonalInformation?: boolean);
constructor(application: LuisApplication, options?: LuisPredictionOptions, includeApiResults?: boolean, telemetryClient?: BotTelemetryClient, logPersonalInformation?: boolean);
constructor(application: LuisApplication|string, options?: LuisPredictionOptions, includeApiResults?: boolean, telemetryClient?: BotTelemetryClient, logPersonalInformation?: boolean) {
if (typeof application === 'string') {
const parsedEndpoint: Url = Url(application);
// Use exposed querystringify to parse the query string for the endpointKey value.
Expand Down Expand Up @@ -160,6 +166,9 @@ export class LuisRecognizer {
const creds: msRest.TokenCredentials = new msRest.TokenCredentials(this.application.endpointKey);
const baseUri: string = this.application.endpoint || 'https://westus.api.cognitive.microsoft.com';
this.luisClient = new LuisClient(creds, baseUri);

this.telemetryClient = telemetryClient || new NullTelemetryClient();
this.logPersonalInformation = logPersonalInformation;
}

/**
Expand Down Expand Up @@ -187,7 +196,6 @@ export class LuisRecognizer {

/**
* Calls LUIS to recognize intents and entities in a users utterance.
*
* @remarks
* Returns a [RecognizerResult](../botbuilder-core/recognizerresult) containing any intents and entities recognized by LUIS.
*
Expand All @@ -213,8 +221,10 @@ export class LuisRecognizer {
* }
* ```
* @param context Context for the current turn of conversation with the use.
* @param telemetryProperties Additional properties to be logged to telemetry with the LuisResult event.
* @param telemetryMetrics Additional metrics to be logged to telemetry with the LuisResult event.
*/
public recognize(context: TurnContext): Promise<RecognizerResult> {
public recognize(context: TurnContext, telemetryProperties?: {[key: string]:string}, telemetryMetrics?: {[key: string]:number} ): Promise<RecognizerResult> {
const cached: any = context.turnState.get(this.cacheKey);
if (!cached) {
const utterance: string = context.activity.text || '';
Expand Down Expand Up @@ -248,6 +258,9 @@ export class LuisRecognizer {
// Write to cache
context.turnState.set(this.cacheKey, recognizerResult);

// Log telemetry
this.onRecognizerResults(recognizerResult, context, telemetryProperties, telemetryMetrics);

return this.emitTraceInfo(context, luisResult, recognizerResult).then(() => {
return recognizerResult;
});
Expand All @@ -261,7 +274,80 @@ export class LuisRecognizer {
return Promise.resolve(cached);
}

private getUserAgent(): string {
/**
* Invoked prior to a LuisResult Event being logged.
* @param recognizerResult The Luis Results for the call.
* @param turnContext Context object containing information for a single turn of conversation with a user.
* @param telemetryProperties Additional properties to be logged to telemetry with the LuisResult event.
* @param telemetryMetrics Additional metrics to be logged to telemetry with the LuisResult event.
*/
protected onRecognizerResults(recognizerResult: RecognizerResult, turnContext: TurnContext, telemetryProperties?: {[key: string]:string}, telemetryMetrics?: {[key: string]:number}): Promise<void> {
this.fillLuisProperties(recognizerResult, turnContext, telemetryProperties).then(props => {
this.telemetryClient.trackEvent(
{
name: LuisTelemetryConstants.luisResultEvent,
properties: props,
metrics: telemetryMetrics
});
});
return;
}

/**
* Fills the event properties for LuisResult event for telemetry.
* These properties are logged when the recognizer is called.
* @param recognizerResult Last activity sent from user.
* @param turnContext Context object containing information for a single turn of conversation with a user.
* @param telemetryProperties Additional properties to be logged to telemetry with the LuisResult event.
* @returns A dictionary that is sent as properties to BotTelemetryClient.trackEvent method for the LuisResult event.
*/
protected async fillLuisProperties(recognizerResult: RecognizerResult, turnContext: TurnContext, telemetryProperties?: {[key: string]:string}): Promise<{[key: string]:string}> {
const topLuisIntent: string = LuisRecognizer.topIntent(recognizerResult);
const intentScore: number = (recognizerResult.intents[topLuisIntent] && 'score' in recognizerResult.intents[topLuisIntent]) ?
recognizerResult.intents[topLuisIntent].score : 0;

// Add the intent score and conversation id properties
const properties: { [key: string]: string } = {};
properties[LuisTelemetryConstants.applicationIdProperty] = this.application.applicationId;
properties[LuisTelemetryConstants.intentProperty] = topLuisIntent;
properties[LuisTelemetryConstants.intentScoreProperty] = intentScore.toString();
if (turnContext.activity.from) {
properties[LuisTelemetryConstants.fromIdProperty] = turnContext.activity.from.id;;
}

if (recognizerResult.sentiment) {
if (recognizerResult.sentiment.label) {
properties[LuisTelemetryConstants.sentimentLabelProperty] = recognizerResult.sentiment.label;
}

if (recognizerResult.sentiment.score) {
properties[LuisTelemetryConstants.sentimentScoreProperty] = recognizerResult.sentiment.score.toString();
}
}

// Log entity names
if (recognizerResult.entities)
{
properties[LuisTelemetryConstants.entitiesProperty] = JSON.stringify(recognizerResult.entities);
}

// Use the LogPersonalInformation flag to toggle logging PII data, text is a common example
if (this.logPersonalInformation && turnContext.activity.text)
{
properties[LuisTelemetryConstants.questionProperty] = turnContext.activity.text;
}

// Additional Properties can override "stock" properties.
if (telemetryProperties != null)
{
return Object.assign({}, properties, telemetryProperties);
}

return properties;
}


private getUserAgent() : string {

// Note when the ms-rest dependency the LuisClient uses has been updated
// this code should be modified to use the client's addUserAgentInfo() function.
Expand Down
18 changes: 18 additions & 0 deletions libraries/botbuilder-ai/src/luisTelemetryConstants.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License

/**
* The BotTelemetryClient event and property names that logged by default.
*/
export class LuisTelemetryConstants {
public static readonly luisResultEvent: string = 'LuisResult'; // Event name
public static readonly applicationIdProperty: string = 'applicationId';
public static readonly intentProperty: string = 'intent';
public static readonly intentScoreProperty: string = 'intentScore';
public static readonly entitiesProperty: string = 'entities';
public static readonly questionProperty: string = 'question';
public static readonly activityIdProperty: string = 'activityId';
public static readonly sentimentLabelProperty: string = 'sentimentLabel';
public static readonly sentimentScoreProperty: string = 'sentimentScore';
public static readonly fromIdProperty: string = 'fromId';
}
Loading