feat: Language-Understanding LSP (#1711)

* Add LSP of LG * change folder name * change folder location * seperate server and client demo * add completion and hover for builtin-functions * change file names * change dependency * refactor the package * update demo readme * make some refine * remove npm lock file * add demos to workspace and run test in 1 command * remove declaration files * change tsc outDir to dist, simplify test command * add syntax highlight in demo and new API * change naming of the project and move the demo * remove package.json in demo * change the content in readme * fix lg-lsp package publish problem * fix build command and redundent d.ts files * integrate LG LSP server to composer server * change api in demo * change the order of commands in build:prod * delete redundent files generated from build * change lg-lsp-server api to attachLSPServer * remove gitignore in lg-lsp-server demo * remove attachLSPServer in server * fix token rules and suggestion context awareness * init of LU LSP * add diagnostic in LSP * add auto suggestions * fix * remove redundent in client * add doc on type config * fix * modify doc type format * fix * change version of bf-lu * fix diagnostic * refactor getRangeAtPosition method * fix tokens * remove black line * fix tokens * add roles suggestions * add regex entity seperated line definition * add suggestions for ml entity * matching only ML entities * fix * update documents * fix * integrate lu-lsp to composer * upgrade botbuilder-expressions * clean up * update sample * add test * add syntax highlight * add token rule for {@ expr * add a case in token * fix import statement token * fix composite entity auto complete * fix entity suggestion and tokens * add labeling experience of add unlabeled entity * fix typos * fix lint * fix redundent edits * fix unnecessary space * fix naming and role suggestions * fix labeling and error postion * fix insert text in wrong line, intent usesFeature * update bf-lu version * change find a valid luisJson and suggest composite * add unit test for LU lsp basic funcionalities Co-authored-by: Hongyang Du (hond) <hond@microsoft.com> Co-authored-by: Dong Lei <donglei@microsoft.com> Co-authored-by: Zhixiang Zhan <zhixzhan@microsoft.com>
microsoft · Jan 20, 2020 · bed2bdc · bed2bdc
1 parent c58666e
commit bed2bdc
Show file tree

Hide file tree

Showing 32 changed files with 2,166 additions and 64 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -9,7 +9,23 @@
       "webRoot": "${workspaceFolder}"
     },
     {
-      "name": "LSP Server",
+      "name": "LU LSP Server",
+      "type": "node",
+      "request": "launch",
+      "args": [
+        "${workspaceFolder}/Composer/packages/tools/language-servers/language-understanding/demo/src/server.ts"
+      ],
+      "runtimeArgs": [
+        "--nolazy",
+        "-r",
+        "${workspaceFolder}/Composer/node_modules/ts-node/register"
+      ],
+      "sourceMaps": true,
+      "cwd": "${workspaceFolder}/Composer/packages/tools/language-servers/language-understanding/demo/src",
+      "protocol": "inspector",
+    },
+    {
+      "name": "LG LSP Server",
       "type": "node",
       "request": "launch",
       "args": [

diff --git a/Composer/packages/client/setupProxy.js b/Composer/packages/client/setupProxy.js
@@ -4,7 +4,7 @@ const paths = require('./config/paths');
 const proxySetting = require(paths.appPackageJson).proxy;
 
 module.exports = function(app) {
-  const wsProxy = proxy('/lg-language-server', {
+  const wsProxy = proxy(['/lg-language-server', '/lu-language-server'], {
     target: proxySetting.replace('/^http/', 'ws'),
     changeOrigin: true,
     ws: true,

diff --git a/Composer/packages/client/src/pages/language-understanding/code-editor.js b/Composer/packages/client/src/pages/language-understanding/code-editor.js
@@ -43,6 +43,14 @@ export default function CodeEditor(props) {
         minimap: 'on',
         lineDecorationsWidth: undefined,
         lineNumbersMinChars: false,
+        glyphMargin: true,
+        autoClosingBrackets: 'always',
+        wordBasedSuggestions: false,
+        autoIndent: true,
+        formatOnType: true,
+        lightbulb: {
+          enabled: true,
+        },
       }}
       errorMsg={errorMsg}
       value={content}

diff --git a/Composer/packages/lib/code-editor/demo/src/index.tsx b/Composer/packages/lib/code-editor/demo/src/index.tsx
@@ -4,9 +4,9 @@
 import React from 'react';
 import ReactDOM from 'react-dom';
 
-import App from './lgEditor';
+// import App from './lgEditor';
 // import App from './jsonEditor';
-// import App from './luEditor';
+import App from './luEditor';
 // import App from './lgJsonEditor';
 // import App from './inlineEditor';
 // import App from './multiEditors';

diff --git a/Composer/packages/lib/code-editor/demo/src/luEditor.tsx b/Composer/packages/lib/code-editor/demo/src/luEditor.tsx
@@ -224,6 +224,10 @@ export default function App() {
   const props = {
     value,
     onChange,
+    languageServer: {
+      port: 5003,
+      path: '/lu-language-server',
+    },
   };
   return <LuEditor {...props} />;
 }
diff --git a/Composer/packages/lib/code-editor/src/LuEditor.tsx b/Composer/packages/lib/code-editor/src/LuEditor.tsx
@@ -2,12 +2,139 @@
 // Licensed under the MIT License.
 
 import React from 'react';
+import { listen, MessageConnection } from 'vscode-ws-jsonrpc';
+import * as monacoEditor from '@bfcomposer/monaco-editor/esm/vs/editor/editor.api';
+import * as monacoCore from 'monaco-editor-core';
+import get from 'lodash/get';
+import { MonacoServices, MonacoLanguageClient } from 'monaco-languageclient';
 
+import { registerLULanguage } from './languages';
+import { createUrl, createWebSocket, createLanguageClient } from './utils/lspUtil';
 import { RichEditor, RichEditorProps } from './RichEditor';
 
-const LU_HELP = 'https://aka.ms/lu-file-format';
-const placeholder = `> See ${LU_HELP} to learn about supported LU concepts.`;
+const LU_HELP = 'https://github.com/microsoft/botframework-cli/blob/master/packages/lu/docs/lu-file-format.md';
+const placeholder = `> To learn more about the LU file format, read the documentation at
+> ${LU_HELP}`;
 
-export function LuEditor(props: RichEditorProps) {
-  return <RichEditor placeholder={placeholder} helpURL={LU_HELP} {...props} />;
+export interface LUOption {
+  inline: boolean;
+  content: string;
+  template?: {
+    name: string;
+    parameters?: string[];
+    body: string;
+  };
+}
+
+export interface LULSPEditorProps extends RichEditorProps {
+  luOption?: LUOption;
+  languageServer?:
+    | {
+        host?: string;
+        hostname?: string;
+        port?: number | string;
+        path: string;
+      }
+    | string;
+}
+
+const defaultLUServer = {
+  path: '/lu-language-server',
+};
+declare global {
+  interface Window {
+    monacoServiceInstance: MonacoServices;
+    monacoLUEditorInstance: MonacoLanguageClient;
+  }
+}
+
+type ServerEdit = {
+  range: { start: { line: number; character: number }; end: { line: number; character: number } };
+  newText: string;
+};
+
+/*
+convert the edits results from the server to an exectable object in manoco editor
+*/
+function convertEdit(serverEdit: ServerEdit) {
+  return {
+    range: {
+      startLineNumber: serverEdit.range.start.line,
+      startColumn: serverEdit.range.start.character,
+      endLineNumber: serverEdit.range.end.line,
+      endColumn: serverEdit.range.end.character,
+    },
+    text: serverEdit.newText,
+    forceMoveMarkers: true,
+  };
+}
+
+export function LuEditor(props: LULSPEditorProps) {
+  const options = {
+    quickSuggestions: true,
+    wordBasedSuggestions: false,
+    formatOnType: true,
+  };
+
+  const { languageServer, ...restProps } = props;
+  const luServer = languageServer || defaultLUServer;
+
+  const editorWillMount = (monaco: typeof monacoEditor) => {
+    registerLULanguage(monaco);
+    if (typeof props.editorWillMount === 'function') {
+      return props.editorWillMount(monaco);
+    }
+  };
+  const editorDidMount = (editor: monacoEditor.editor.IStandaloneCodeEditor, monaco: typeof monacoEditor) => {
+    if (!window.monacoServiceInstance) {
+      window.monacoServiceInstance = MonacoServices.install(editor as monacoCore.editor.IStandaloneCodeEditor | any);
+    }
+
+    if (!window.monacoLUEditorInstance) {
+      const uri = get(editor.getModel(), 'uri._formatted', '');
+      const url = createUrl(luServer);
+      const webSocket: WebSocket = createWebSocket(url);
+      listen({
+        webSocket,
+        onConnection: (connection: MessageConnection) => {
+          const languageClient = createLanguageClient('LU Language Client', ['lu'], connection);
+          if (!window.monacoLUEditorInstance) {
+            window.monacoLUEditorInstance = languageClient;
+          }
+
+          editor.addCommand(monaco.KeyMod.Shift | monaco.KeyCode.Enter, function() {
+            const position = editor.getPosition();
+            languageClient.sendRequest('labelingExperienceRequest', { uri, position });
+          });
+          languageClient.onReady().then(() =>
+            languageClient.onNotification('addUnlabelUtterance', result => {
+              const edits = result.edits.map(e => {
+                return convertEdit(e);
+              });
+              editor.executeEdits(uri, edits);
+            })
+          );
+          const disposable = languageClient.start();
+          connection.onClose(() => disposable.dispose());
+        },
+      });
+    }
+
+    if (typeof props.editorDidMount === 'function') {
+      return props.editorDidMount(editor, monaco);
+    }
+  };
+
+  return (
+    <RichEditor
+      placeholder={placeholder}
+      helpURL={LU_HELP}
+      {...restProps}
+      theme={'lu'}
+      language={'lu'}
+      options={options}
+      editorWillMount={editorWillMount}
+      editorDidMount={editorDidMount}
+    />
+  );
 }
diff --git a/Composer/packages/lib/code-editor/src/languages/index.ts b/Composer/packages/lib/code-editor/src/languages/index.ts
@@ -2,3 +2,4 @@
 // Licensed under the MIT License.
 
 export * from './lg';
+export * from './lu';
diff --git a/Composer/packages/lib/code-editor/src/languages/lu.ts b/Composer/packages/lib/code-editor/src/languages/lu.ts
@@ -0,0 +1,77 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import * as monacoEditor from '@bfcomposer/monaco-editor/esm/vs/editor/editor.api';
+
+export function registerLULanguage(monaco: typeof monacoEditor) {
+  monaco.languages.setMonarchTokensProvider('lu', {
+    tokenizer: {
+      root: [
+        [/^\s*#/, { token: 'intent', next: '@intent' }],
+        [/^\s*@/, { token: 'entity-identifier', goBack: 1, next: '@entityMode' }],
+        [/^\s*>\s*[\s\S]*$/, { token: 'comments' }],
+      ],
+
+      intent: [
+        [/^\s*#/, { token: 'intent', next: '@intent' }],
+        [/^\s*-/, { token: 'utterrance-indentifier', next: '@utterrance' }],
+        [/^\s*>\s*[\s\S]*$/, { token: 'comments' }],
+        [/^\s*@/, { token: 'entity-identifier', goBack: 1, next: '@entityMode' }],
+        [/.*$/, 'intent'],
+      ],
+      utterrance: [
+        [/^\s*#/, { token: 'intent', next: '@intent' }],
+        [/^\s*>\s*[\s\S]*$/, { token: 'comments' }],
+        [/^\s*-/, { token: 'utterrance-indentifier', next: 'utterrance' }],
+        [/^\s*@/, { token: 'entity-identifier', goBack: 1, next: '@entityMode' }],
+        [/({)(\s*[\w.@:\s]*\s*)(=)(\s*[\w.]*\s*)(})/, ['lb', 'pattern', 'equal', 'entity-name', 'rb']],
+        [/({\s*@)(\s*[\w.]*\s*)(})/, ['lb', 'entity-name', 'rb']],
+        // eslint-disable-next-line security/detect-unsafe-regex
+        [/\s*\[[\w\s.]+\]\(.{1,2}\/[\w.*]+(#[\w.?]+)?\)/, 'import-desc'],
+        [/./, 'utterance-other'],
+      ],
+      entityMode: [
+        [/^\s*#/, { token: 'intent', next: '@intent' }],
+        [/^\s*>\s*[\s\S]*$/, { token: 'comments' }],
+        [/^\s*-/, { token: 'utterrance-indentifier', next: 'utterrance' }],
+        [
+          /(@\s*)(ml|prebuilt|regex|list|composite|patternany|phraselist)(\s*\w*)/,
+          ['intent-indentifier', 'entity-type', 'entity-name'],
+        ],
+        [/(@\s*)(\s*\w*)/, ['intent-indentifier', 'entity-name']],
+        [/\s*(hasRoles|useFeature)\s*/, 'keywords'],
+        [/.*$/, 'entity-other', '@pop'],
+      ],
+    },
+  });
+
+  monaco.languages.register({
+    id: 'lu',
+    extensions: ['.lu'],
+    aliases: ['LU', 'language-understanding'],
+    mimetypes: ['application/lu'],
+  });
+
+  monaco.languages.setLanguageConfiguration('lu', {
+    autoClosingPairs: [
+      { open: '{', close: '}' },
+      { open: '[', close: ']' },
+      { open: '(', close: ')' },
+    ],
+  });
+
+  monaco.editor.defineTheme('lu', {
+    base: 'vs',
+    inherit: false,
+    colors: {},
+    rules: [
+      { token: 'intent', foreground: '0000FF' },
+      { token: 'pattern', foreground: '00B7C3' },
+      { token: 'entity-name', foreground: '038387' },
+      { token: 'comments', foreground: '7A7A7A' },
+      { token: 'import-desc', foreground: '00A32B' },
+      { token: 'entity-type', foreground: 'DF2C2C' },
+      { token: 'keywords', foreground: '0078D7' },
+    ],
+  });
+}
diff --git a/Composer/packages/server/package.json b/Composer/packages/server/package.json
@@ -55,6 +55,7 @@
     "@azure/ms-rest-js": "^1.8.7",
     "@bfc/indexers": "*",
     "@bfc/lg-languageserver": "*",
+    "@bfc/lu-languageserver": "*",
     "@bfc/shared": "*",
     "@bfcomposer/lubuild": "1.1.2-preview",
     "archiver": "^3.0.0",

diff --git a/Composer/packages/server/src/server.ts b/Composer/packages/server/src/server.ts
@@ -13,6 +13,7 @@ import * as ws from 'ws';
 import * as rpc from 'vscode-ws-jsonrpc';
 import { IConnection, createConnection } from 'vscode-languageserver';
 import { LGServer } from '@bfc/lg-languageserver';
+import { LUServer } from '@bfc/lu-languageserver';
 
 import { BotProjectService } from './services/project';
 import { getAuthProvider } from './router/auth';
@@ -124,6 +125,14 @@ function launchLanguageServer(socket: rpc.IWebSocket) {
   server.start();
 }
 
+function launchLuLanguageServer(socket: rpc.IWebSocket) {
+  const reader = new rpc.WebSocketMessageReader(socket);
+  const writer = new rpc.WebSocketMessageWriter(socket);
+  const connection: IConnection = createConnection(reader, writer);
+  const server = new LUServer(connection);
+  server.start();
+}
+
 attachLSPServer(wss, server, '/lg-language-server', webSocket => {
   // launch language server when the web socket is opened
   if (webSocket.readyState === webSocket.OPEN) {
@@ -134,3 +143,14 @@ attachLSPServer(wss, server, '/lg-language-server', webSocket => {
     });
   }
 });
+
+attachLSPServer(wss, server, '/lu-language-server', webSocket => {
+  // launch language server when the web socket is opened
+  if (webSocket.readyState === webSocket.OPEN) {
+    launchLuLanguageServer(webSocket);
+  } else {
+    webSocket.on('open', () => {
+      launchLuLanguageServer(webSocket);
+    });
+  }
+});
diff --git a/Composer/packages/tools/language-servers/language-generation/package.json b/Composer/packages/tools/language-servers/language-generation/package.json
@@ -17,6 +17,7 @@
   "dependencies": {
     "@bfc/indexers": "*",
     "botbuilder-lg": "4.7.0-preview.93464",
+    "botframework-expressions": "4.7.0-preview.93464",
     "request-light": "^0.2.2",
     "vscode-languageserver": "^5.3.0-next"
   },

diff --git a/Composer/packages/tools/language-servers/language-understanding/.gitignore b/Composer/packages/tools/language-servers/language-understanding/.gitignore
@@ -0,0 +1,3 @@
+/lib
+/demo/lib
+/demo/dist
diff --git a/Composer/packages/tools/language-servers/language-understanding/README.md b/Composer/packages/tools/language-servers/language-understanding/README.md
@@ -0,0 +1,27 @@
+# Language-understanding language server demo
+
+
+
+## goto language-understanding directory, install packages, run
+``` 
+npm install
+```
+
+## How to start the demo 
+
+#### 1. under language-understanding directory, run
+```
+yarn start
+```
+
+### 2. go to code-editor directory
+
+start luEditor demo, connect :5003/lu-language-server
+
+## Features
+
+### 1.Auto-suggestions for entity types and defined entities
+
+### 2.Syntax and semantic diagonostics
+
+### Auto-copletion for ml entities and list entities
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,3 +2,4 @@
		// Licensed under the MIT License.

		export * from './lg';
		export * from './lu';