feat: support validate lg custom functions (#3273)

* fix: support validate lg custom functions and separate the dialog validation from parser

* use file name as namespace

* add unit test

* fix lint

Composer/packages/client/src/store/parsers/workers/indexer.worker.ts

@@ -6,11 +6,11 @@ const ctx: Worker = self as any;
 
 ctx.onmessage = function (msg) {
   const { id, payload } = msg.data;
-  const { files, botName, schemas, locale } = payload;
+  const { files, botName, locale } = payload;
   const { index } = indexer;
 
   try {
-    const { dialogs, luFiles, lgFiles } = index(files, botName, schemas, locale);
+    const { dialogs, luFiles, lgFiles } = index(files, botName, locale);
     ctx.postMessage({ id, payload: { dialogs, luFiles, lgFiles } });
   } catch (error) {
     ctx.postMessage({ id, error });

Composer/packages/client/src/store/reducer/index.ts

@@ -5,7 +5,7 @@ import get from 'lodash/get';
 import set from 'lodash/set';
 import has from 'lodash/has';
 import merge from 'lodash/merge';
-import { indexer, dialogIndexer, lgIndexer, luIndexer, autofixReferInDialog } from '@bfc/indexers';
+import { indexer, dialogIndexer, lgIndexer, luIndexer, autofixReferInDialog, validateDialog } from '@bfc/indexers';
 import {
   SensitiveProperties,
   LuFile,
@@ -84,9 +84,12 @@ const initLuFilesStatus = (projectId: string, luFiles: LuFile[], dialogs: Dialog
 const getProjectSuccess: ReducerFunc = (state, { response }) => {
   const { files, botName, botEnvironment, location, schemas, settings, id, locale, diagnostics } = response.data;
   schemas.sdk.content = processSchema(id, schemas.sdk.content);
-  const { dialogs, luFiles, lgFiles, skillManifestFiles } = indexer.index(files, botName, schemas.sdk.content, locale);
+  const { dialogs, luFiles, lgFiles, skillManifestFiles } = indexer.index(files, botName, locale);
   state.projectId = id;
-  state.dialogs = dialogs;
+  state.dialogs = dialogs.map((dialog) => {
+    dialog.diagnostics = validateDialog(dialog, schemas.sdk.content, lgFiles, luFiles);
+    return dialog;
+  });
   state.botEnvironment = botEnvironment || state.botEnvironment;
   state.botName = botName;
   state.botStatus = location === state.location ? state.botStatus : BotStatus.unConnected;
@@ -166,8 +169,7 @@ const createLgFile: ReducerFunc = (state, { id, content }) => {
 
   const { parse } = lgIndexer;
   const lgImportresolver = importResolverGenerator(state.lgFiles, '.lg');
-  const { templates, diagnostics } = parse(content, id, lgImportresolver);
-  const lgFile = { id, templates, diagnostics, content };
+  const lgFile = { id, content, ...parse(content, id, lgImportresolver) };
   state.lgFiles.push(lgFile);
   return state;
 };
@@ -228,7 +230,12 @@ const updateLuTemplate: ReducerFunc = (state, luFile: LuFile) => {
 const updateDialog: ReducerFunc = (state, { id, content }) => {
   state.dialogs = state.dialogs.map((dialog) => {
     if (dialog.id === id) {
-      return { ...dialog, ...dialogIndexer.parse(dialog.id, content, state.schemas.sdk.content) };
+      dialog = {
+        ...dialog,
+        ...dialogIndexer.parse(dialog.id, content),
+      };
+      dialog.diagnostics = validateDialog(dialog, state.schemas.sdk.content, state.lgFiles, state.luFiles);
+      return dialog;
     }
     return dialog;
   });
@@ -261,8 +268,9 @@ const createDialog: ReducerFunc = (state, { id, content }) => {
   const dialog = {
     isRoot: false,
     displayName: id,
-    ...dialogIndexer.parse(id, fixedContent, state.schemas.sdk.content),
+    ...dialogIndexer.parse(id, fixedContent),
   };
+  dialog.diagnostics = validateDialog(dialog, state.schemas.sdk.content, state.lgFiles, state.luFiles);
   state.dialogs.push(dialog);
   state = createLgFile(state, { id, content: '' });
   state = createLuFile(state, { id, content: '' });

Composer/packages/lib/indexers/__tests__/dialogUtils/dialogChecker.test.ts

@@ -1,6 +1,6 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
-import { createPath } from './../../src/dialogUtils/dialogChecker';
+import { createPath } from '../../src/validations/expressionValidation/utils';
 
 describe('create right diagnostic path', () => {
   it('should check if the diagnostics have errors', () => {

Composer/packages/lib/indexers/__tests__/lgUtil.test.ts

@@ -3,7 +3,7 @@
 
 import { Templates } from 'botbuilder-lg';
 
-import { updateTemplate, addTemplate, removeTemplate } from '../src/utils/lgUtil';
+import { updateTemplate, addTemplate, removeTemplate, extractOptionByKey } from '../src/utils/lgUtil';
 
 describe('update lg template', () => {
   it('should update lg template', () => {
@@ -69,3 +69,15 @@ describe('add lg template', () => {
     expect(templates[0].name).toEqual('Exit');
   });
 });
+
+describe('extract option by key', () => {
+  it('should extract optin', () => {
+    const options = ['@strict = false', '@Namespace = foo', '@Exports = bar, cool'];
+    const namespace = extractOptionByKey('@namespace', options);
+    expect(namespace).toBe('foo');
+    const namespace2 = extractOptionByKey('@wrong', options);
+    expect(namespace2).toBe('');
+    const strict = extractOptionByKey('@strict', options);
+    expect(strict).toBe('false');
+  });
+});

Composer/packages/lib/indexers/__tests__/validations/expressionValidation.test.ts

@@ -0,0 +1,61 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+import { LgFile } from '@bfc/shared';
+
+import { validate } from '../../src/validations/expressionValidation/validation';
+
+import { searchLgCustomFunction } from './../../src/validations/expressionValidation/index';
+
+describe('search lg custom function', () => {
+  it('should return custom functions with namespace', () => {
+    const lgFiles = [{ options: ['@strict = false', '@Namespace = foo', '@Exports = bar, cool'], id: 'test.en-us' }];
+    const result = searchLgCustomFunction(lgFiles as LgFile[]);
+    expect(result.length).toEqual(2);
+    expect(result[0]).toEqual('foo.bar');
+    expect(result[1]).toEqual('foo.cool');
+  });
+
+  it('should return custom functions with namespace', () => {
+    const lgFiles = [{ options: ['@strict = false', '@Exports = bar, cool'], id: 'test.en-us' }];
+    const result = searchLgCustomFunction(lgFiles as LgFile[]);
+    expect(result.length).toEqual(2);
+    expect(result[0]).toEqual('test.en-us.bar');
+    expect(result[1]).toEqual('test.en-us.cool');
+  });
+});
+
+describe('validate expression', () => {
+  it('if string expression do nothing', () => {
+    const expression = { value: 'hello', required: false, path: 'test', types: ['string'] };
+    const result = validate(expression, []);
+    expect(result).toBeNull();
+  });
+
+  it('if start with =, but type is not match', () => {
+    const expression = { value: '=13', required: false, path: 'test', types: ['string'] };
+    const result = validate(expression, []);
+    expect(result?.message).toBe('the expression type is not match');
+  });
+
+  it('if start with =, and type is match', () => {
+    const expression = { value: '=13', required: false, path: 'test', types: ['integer'] };
+    const result = validate(expression, []);
+    expect(result).toBeNull();
+    expression.value = '=true';
+    expression.types[0] = 'boolean';
+    const result1 = validate(expression, []);
+    expect(result1).toBeNull();
+  });
+
+  it('use custom functions, but lg file does not export', () => {
+    const expression = { value: '=foo.bar()', required: false, path: 'test', types: ['boolean'] };
+    const result = validate(expression, []);
+    expect(result).not.toBeNull();
+  });
+
+  it('use custom functions, and lg file does export', () => {
+    const expression = { value: '=foo.bar()', required: false, path: 'test', types: ['boolean'] };
+    const result = validate(expression, ['foo.bar']);
+    expect(result).toBeNull();
+  });
+});

Composer/packages/lib/indexers/src/dialogIndexer.ts

@@ -10,15 +10,14 @@ import {
   DialogInfo,
   FileInfo,
   LgTemplateJsonPath,
-  Diagnostic,
   ReferredLuIntents,
+  Diagnostic,
 } from '@bfc/shared';
 
-import { createPath } from './dialogUtils/dialogChecker';
-import { checkerFuncs } from './dialogUtils/dialogChecker';
 import { JsonWalk, VisitorFunc } from './utils/jsonWalk';
 import { getBaseName } from './utils/help';
 import ExtractIntentTriggers from './dialogUtils/extractIntentTriggers';
+import { createPath } from './validations/expressionValidation/utils';
 // find out all lg templates given dialog
 function ExtractLgTemplates(id, dialog): LgTemplateJsonPath[] {
   const templates: LgTemplateJsonPath[] = [];
@@ -155,55 +154,14 @@ function ExtractReferredDialogs(dialog): string[] {
   return uniq(dialogs);
 }
 
-// check all fields
-function CheckFields(dialog, id: string, schema: any): Diagnostic[] {
-  const diagnostics: Diagnostic[] = [];
-  /**
-   *
-   * @param path , jsonPath string
-   * @param value , current node value    *
-   * @return boolean, true to stop walk
-   * */
-  const visitor: VisitorFunc = (path: string, value: any): boolean => {
-    if (has(value, '$kind')) {
-      const allChecks = [...checkerFuncs['.']];
-      const checkerFunc = checkerFuncs[value.$kind];
-      if (checkerFunc) {
-        allChecks.splice(0, 0, ...checkerFunc);
-      }
-
-      allChecks.forEach((func) => {
-        const result = func(path, value, value.$kind, schema.definitions[value.$kind]);
-        if (result) {
-          diagnostics.splice(0, 0, ...result);
-        }
-      });
-    }
-    return false;
-  };
-  JsonWalk(id, dialog, visitor);
-  return diagnostics.map((e) => {
-    e.source = id;
-    return e;
-  });
-}
-
-function validate(id: string, content, schema: any): Diagnostic[] {
-  try {
-    return CheckFields(content, id, schema);
-  } catch (error) {
-    return [new Diagnostic(error.message, id)];
-  }
-}
-
-function parse(id: string, content: any, schema: any) {
+function parse(id: string, content: any) {
   const luFile = typeof content.recognizer === 'string' ? content.recognizer : '';
   const lgFile = typeof content.generator === 'string' ? content.generator : '';
-
+  const diagnostics: Diagnostic[] = [];
   return {
     id,
     content,
-    diagnostics: validate(id, content, schema),
+    diagnostics,
     referredDialogs: ExtractReferredDialogs(content),
     lgTemplates: ExtractLgTemplates(id, content),
     referredLuIntents: ExtractLuIntents(content, id),
@@ -214,7 +172,7 @@ function parse(id: string, content: any, schema: any) {
   };
 }
 
-function index(files: FileInfo[], botName: string, schema: any): DialogInfo[] {
+function index(files: FileInfo[], botName: string): DialogInfo[] {
   const dialogs: DialogInfo[] = [];
   if (files.length !== 0) {
     for (const file of files) {
@@ -226,7 +184,7 @@ function index(files: FileInfo[], botName: string, schema: any): DialogInfo[] {
           const dialog = {
             isRoot,
             displayName: isRoot ? `${botName}` : id,
-            ...parse(id, dialogJson, schema),
+            ...parse(id, dialogJson),
           };
           dialogs.push(dialog);
         }

Composer/packages/lib/indexers/src/index.ts

@@ -34,10 +34,10 @@ class Indexer {
     return importResolverGenerator(lgFiles, '.lg', locale);
   };
 
-  public index(files: FileInfo[], botName: string, schema: any, locale: string) {
+  public index(files: FileInfo[], botName: string, locale: string) {
     const result = this.classifyFile(files);
     return {
-      dialogs: dialogIndexer.index(result[FileExtensions.Dialog], botName, schema),
+      dialogs: dialogIndexer.index(result[FileExtensions.Dialog], botName),
       lgFiles: lgIndexer.index(result[FileExtensions.lg], this.getLgImportResolver(result[FileExtensions.lg], locale)),
       luFiles: luIndexer.index(result[FileExtensions.Lu]),
       skillManifestFiles: skillManifestIndexer.index(result[FileExtensions.Manifest]),
@@ -50,5 +50,5 @@ export const indexer = new Indexer();
 export * from './dialogIndexer';
 export * from './lgIndexer';
 export * from './luIndexer';
-export * from './dialogUtils';
 export * from './utils';
+export * from './validations';