feat: Start Botproject runtime in bot folder (#1672)

* start runtime from samplpebot csharp runtime

* fix

* fix

* fix

* use logger instead of console log

* fix comment

* fix comment

* add port usable validation and use log instead console log

* fix comments

* move build script from template to server

* polish output error message

* remove environment

* fix comments and add back the build script into template runtime

* add migrate old bot feature

* change docker file

* remove annotation

* fix docker

* simpler dockerfile

* fix dockerfile bug and port mapping

* add bash script for unix, and remove pwsh from docker

* split dockerfile into two

* use nultistage instead of two dockerfile

* update yarn lock

* add more debug logs

* fix settings file mis-created in server folder

Co-authored-by: Lu Han <32191031+luhan2017@users.noreply.github.com>
Co-authored-by: Chris Whitten <christopher.whitten@microsoft.com>
Co-authored-by: Andy Brown <asbrown002@gmail.com>

BotProject/Templates/CSharp/Program.cs

@@ -2,6 +2,8 @@
 // Licensed under the MIT License.
 
 using System;
+using System.Diagnostics;
+using System.IO;
 using Microsoft.AspNetCore;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.Extensions.Configuration;
@@ -24,8 +26,20 @@ public static IWebHost BuildWebHost(string[] args) =>
                 config
                     .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true)
                     .AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true, reloadOnChange: true)
+                    .AddJsonFile($"ComposerDialogs/settings/appsettings.json", optional: true, reloadOnChange: true)
                     .AddJsonFile($"luis.settings.{env.EnvironmentName}.{luisAuthoringRegion}.json", optional: true, reloadOnChange: true)
                     .AddJsonFile($"luis.settings.{Environment.UserName}.{luisAuthoringRegion}.json", optional: true, reloadOnChange: true);
+                try
+                {
+                    foreach (string filePath in Directory.GetFiles($"ComposerDialogs", "generated/luis.settings.*.json"))
+                    {
+                        config.AddJsonFile(filePath, optional: true, reloadOnChange: true);
+                    }
+                }
+                catch (Exception ex)
+                {
+                    Trace.WriteLine(ex.Message);
+                }
 
                 if (env.IsDevelopment())
                 {

BotProject/Templates/CSharp/Scripts/build_runtime.ps1

@@ -0,0 +1,25 @@
+Param(
+	[object] $config,
+	[string] $customSettingFolder,
+  [string] $luisAuthroingKey,
+  [SecureString] $appPassword,
+  [string] $projFolder = $(Join-Path $(Get-Location) BotProject CSharp)
+)
+
+if ($PSVersionTable.PSVersion.Major -lt 6){
+	Write-Host "! Powershell 6 is required, current version is $($PSVersionTable.PSVersion.Major), please refer following documents for help."
+	Write-Host "For Windows - https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-windows?view=powershell-6"
+	Write-Host "For Mac - https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-macos?view=powershell-6"
+	Break
+}
+
+if ((dotnet --version) -lt 3) {
+	Write-Host "! dotnet core 3.0 is required, please refer following documents for help."
+	Write-Host "https://dotnet.microsoft.com/download/dotnet-core/3.0"
+	Break
+}
+
+# This command need dotnet core more than 3.0
+dotnet user-secrets init
+
+dotnet build

BotProject/Templates/CSharp/Scripts/build_runtime.sh

@@ -0,0 +1,11 @@
+versionString=`dotnet --version`
+versionNum=`echo $versionString | cut -d . -f 1`
+if [[ $versionNum -lt 3 ]]
+then
+    echo "! dotnet core 3.0 is required, please refer following documents for help.
+https://dotnet.microsoft.com/download/dotnet-core/3.0"
+	exit 1
+else
+	dotnet user-secrets init
+	dotnet build
+fi

Composer/Dockerfile

@@ -18,8 +18,8 @@ ENV NODE_ENV "production"
 RUN yarn build:prod
 
 
-# use a multi-stage build to reduce the final image size
-FROM node:12-alpine
+
+FROM node:12-alpine as composerbasic
 
 WORKDIR /app/Composer
 COPY --from=build /src/Composer/yarn.lock .
@@ -31,4 +31,62 @@ COPY --from=build /src/Composer/packages/tools ./packages/tools
 ENV NODE_ENV "production"
 RUN yarn --production --frozen-lockfile --force && yarn cache clean
 WORKDIR /app/Composer
-CMD ["yarn", "start:server"]
+
+
+FROM composerbasic
+
+RUN apk add --no-cache \
+  ca-certificates \
+  \
+  # .NET Core dependencies
+  krb5-libs \
+  libgcc \
+  libintl \
+  libssl1.1 \
+  libstdc++ \
+  zlib
+
+# Install .Net Core SDK
+ENV \
+  # Unset the value from the base image
+  ASPNETCORE_URLS= \
+  # Disable the invariant mode (set in base image)
+  DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=false \
+  # Enable correct mode for dotnet watch (only mode supported in a container)
+  DOTNET_USE_POLLING_FILE_WATCHER=true \
+  LC_ALL=en_US.UTF-8 \
+  LANG=en_US.UTF-8 \
+  # Skip extraction of XML docs - generally not useful within an image/container - helps performance
+  NUGET_XMLDOC_MODE=skip \
+  # PowerShell telemetry for docker image usage
+  POWERSHELL_DISTRIBUTION_CHANNEL=PSDocker-DotnetCoreSDK-Alpine-3.10
+
+# Add dependencies for disabling invariant mode (set in base image)
+RUN apk add --no-cache icu-libs
+
+# Install .NET Core 2.1
+ENV DOTNET_SDK_VERSION 2.1.607
+
+RUN wget -O dotnet.tar.gz https://dotnetcli.azureedge.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-musl-x64.tar.gz \
+  && dotnet_sha512='61caf6602b8a2aa89769b3e91ddaec963d8ab9f802cd7f6c6da4f02426358712bc2bb0930e7ee3a81d75c7607039543b554cb8ed50e45610655f9e91ed0f2f17' \
+  && echo "$dotnet_sha512  dotnet.tar.gz" | sha512sum -c - \
+  && mkdir -p /usr/share/dotnet \
+  && tar -C /usr/share/dotnet -xzf dotnet.tar.gz \
+  && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet \
+  && rm dotnet.tar.gz
+
+# Install .NET Core SDK 3.0
+ENV DOTNET_SDK_VERSION 3.0.101
+
+RUN wget -O dotnet.tar.gz https://dotnetcli.azureedge.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-musl-x64.tar.gz \
+  && dotnet_sha512='98cc98f58187d208bd388f8c71862ea75e50ca25666e265f40a4e7c28082c2784738172e8ae4af7815057f7c57072cbe4fc03301d01738fc1ed5bb5e4d30a363' \
+  && echo "$dotnet_sha512  dotnet.tar.gz" | sha512sum -c - \
+  && tar -C /usr/share/dotnet -xzf dotnet.tar.gz \
+  && rm dotnet.tar.gz
+
+# Enable detection of running in a container
+ENV DOTNET_RUNNING_IN_CONTAINER=true \
+  # Set the invariant mode since icu_libs isn't included (see https://github.com/dotnet/announcements/issues/20)
+  DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=true
+
+CMD ["yarn","start:server"]

Composer/package.json

@@ -29,7 +29,7 @@
     "build:client": "yarn workspace @bfc/client build",
     "build:tools": "yarn workspace @bfc/tools build:all",
     "start": "cross-env NODE_ENV=production PORT=3000 yarn start:server",
-    "startall": "node scripts/update.js && concurrently --kill-others-on-fail \"npm:runtime\" \"npm:start\"",
+    "startall": "node scripts/update.js && yarn start",
     "start:dev": "concurrently  \"npm:start:client\" \"npm:start:server:dev\"",
     "start:client": "yarn workspace @bfc/client start",
     "start:server": "yarn workspace @bfc/server start",

Composer/packages/client/src/constants/index.ts

@@ -67,6 +67,7 @@ export enum ActionTypes {
   SET_CREATION_FLOW_STATUS = 'SET_CREATION_FLOW_STATUS',
   SET_DESIGN_PAGE_LOCATION = 'SET_DESIGN_PAGE_LOCATION',
   CONNECT_BOT_SUCCESS = 'CONNECT_BOT_SUCCESS',
+  CONNECT_BOT_FAILURE = 'CONNECT_BOT_FAILURE',
   RELOAD_BOT_SUCCESS = 'RELOAD_BOT_SUCCESS',
   SYNC_ENV_SETTING = 'SYNC_ENV_SETTING',
   GET_ENV_SETTING = 'GET_ENV_SETTING',

Composer/packages/client/src/store/action/bot.ts

@@ -13,6 +13,7 @@ export const connectBot: ActionCreator = async (store, settings) => {
 
   try {
     const res = await httpClient.get(path);
+    await reloadBot(store, settings);
     store.dispatch({
       type: ActionTypes.CONNECT_BOT_SUCCESS,
       payload: {
@@ -21,10 +22,14 @@ export const connectBot: ActionCreator = async (store, settings) => {
       },
     });
   } catch (err) {
+    store.dispatch({
+      type: ActionTypes.CONNECT_BOT_FAILURE,
+      payload: {
+        status: 'unConnected',
+      },
+    });
     throw new Error(err.response.data.message);
   }
-
-  await reloadBot(store, settings);
 };
 
 // return only the connect URL -- do not reload

Composer/packages/client/src/store/action/setting.ts

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 
 import get from 'lodash/get';
+import has from 'lodash/has';
 import { SensitiveProperties } from '@bfc/shared';
 
 import { ActionCreator, DialogSetting } from '../types';
@@ -27,8 +28,10 @@ export const setSettings: ActionCreator = async (
     });
     // set value in local storage
     for (const property of SensitiveProperties) {
-      const propertyValue = get(settings, property);
-      settingsStorage.setField(botName, property, propertyValue ? propertyValue : '');
+      if (has(settings, property)) {
+        const propertyValue = get(settings, property, '');
+        settingsStorage.setField(botName, property, propertyValue);
+      }
     }
     // set value to server
     const suffix = slot ? `/${slot}` : '';

Composer/packages/client/src/store/reducer/index.ts

@@ -313,6 +313,7 @@ export const reducer = createReducer({
   [ActionTypes.REMOVE_LU_FAILURE]: noOp,
   [ActionTypes.PUBLISH_LU_SUCCCESS]: updateLuTemplate,
   [ActionTypes.CONNECT_BOT_SUCCESS]: setBotStatus,
+  [ActionTypes.CONNECT_BOT_FAILURE]: setBotStatus,
   [ActionTypes.RELOAD_BOT_SUCCESS]: setBotLoadErrorMsg,
   // [ActionTypes.RELOAD_BOT_FAILURE]: setBotLoadErrorMsg,
   [ActionTypes.SET_ERROR]: setError,

Composer/packages/server/package.json

@@ -69,6 +69,7 @@
     "express": "^4.16.4",
     "form-data": "^2.3.3",
     "format-message": "^6.2.1",
+    "get-port": "^5.1.0",
     "globby": "^9.1.0",
     "http-errors": "^1.7.2",
     "immer": "^2.1.4",

Composer/packages/server/src/controllers/connector.ts

@@ -1,27 +1,28 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
+import merge from 'lodash/merge';
 
 import { BotEnvironments } from '../models/connector';
-import { EnvironmentProvider } from '../models/environment';
+import { BotProjectService } from '../services/project';
 
 async function connect(req: any, res: any) {
   try {
     const hostName = req.hostname;
     const env: BotEnvironments = req.query && req.query.botEnvironment ? req.query.botEnvironment : 'production';
-    const environment = EnvironmentProvider.getCurrent();
-    const botEndpoint = await environment.getBotConnector().connect(env || 'production', hostName);
+    const environment = BotProjectService.getCurrentBotProject()?.environment;
+    const botEndpoint = await environment?.getBotConnector().connect(env || 'production', hostName);
     res.send({ botEndpoint });
   } catch (error) {
     res.status(400).json({
-      message: 'cannot connect to a bot runtime, make sure you start the bot runtime',
+      message: error.message || 'cannot connect to a bot runtime, make sure you start the bot runtime',
     });
   }
 }
 
 async function getPublishHistory(req: any, res: any) {
   try {
-    const environment = EnvironmentProvider.getCurrent();
-    const history = await environment.getBotConnector().getPublishHistory();
+    const environment = BotProjectService.getCurrentBotProject()?.environment;
+    const history = await environment?.getBotConnector().getPublishHistory();
     res.send(history);
   } catch (error) {
     res.status(400).json({
@@ -32,8 +33,9 @@ async function getPublishHistory(req: any, res: any) {
 
 async function sync(req: any, res: any) {
   try {
-    const environment = EnvironmentProvider.getCurrent();
-    await environment.getBotConnector().sync({ ...req.body, user: req.user });
+    const environment = BotProjectService.getCurrentBotProject()?.environment;
+    const settingsInDisk = await environment?.getSettingsManager().get();
+    await environment?.getBotConnector().sync(merge({}, settingsInDisk, req.body, { user: req.user }));
     res.send('OK');
   } catch (error) {
     res.status(400).json({
@@ -45,8 +47,8 @@ async function sync(req: any, res: any) {
 async function publish(req: any, res: any) {
   try {
     const label = req.params ? req.params.label : undefined;
-    const environment = EnvironmentProvider.getCurrent();
-    await environment.getBotConnector().publish({ ...req.body, user: req.user }, label);
+    const environment = BotProjectService.getCurrentBotProject()?.environment;
+    await environment?.getBotConnector().publish({ ...req.body, user: req.user }, label);
     res.send('OK');
   } catch (error) {
     res.status(400).json({
@@ -56,8 +58,8 @@ async function publish(req: any, res: any) {
 }
 
 function status(req: any, res: any) {
-  const environment = EnvironmentProvider.getCurrent();
-  res.send(environment.getBotConnector().status);
+  const environment = BotProjectService.getCurrentBotProject()?.environment;
+  res.send(environment?.getBotConnector().status);
 }
 
 export const BotConnectorController = {

Composer/packages/server/src/models/asset/assetManager.ts

@@ -10,6 +10,7 @@ import { LocationRef } from '../bot/interface';
 import { Path } from '../../utility/path';
 import { copyDir } from '../../utility/storage';
 import StorageService from '../../services/storage';
+import { IFileStorage } from '../storage/interface';
 
 interface TemplateData {
   [key: string]: {
@@ -156,36 +157,34 @@ export class AssetManager {
     return output;
   }
 
-  public async copyProjectTemplateTo(templateId: string, ref: LocationRef): Promise<LocationRef> {
-    if (this.projectTemplates.length === 0) {
-      await this.getProjectTemplates();
-    }
-    if (this.runtimeTemplates.length === 0) {
-      await this.getProjectRuntime();
-    }
+  public async copyDataFilesTo(templateId: string, dstDir: string, dstStorage: IFileStorage) {
     const template = find(this.projectTemplates, { id: templateId });
     if (template === undefined || template.path === undefined) {
       throw new Error(`no such template with id ${templateId}`);
     }
+    // copy Composer data files
+    await copyDir(template.path, this.templateStorage, dstDir, dstStorage);
+  }
 
+  public async copyRuntimeTo(dstDir: string, dstStorage: IFileStorage) {
     const runtime = find(this.runtimeTemplates, { id: DEFAULT_RUNTIME });
     if (runtime === undefined || runtime.path === undefined) {
       throw new Error(`no such runtime with id ${DEFAULT_RUNTIME}`);
     }
+    // copy runtime code files
+    await copyDir(runtime.path, this.templateStorage, dstDir, dstStorage);
+  }
 
+  public async copyProjectTemplateTo(templateId: string, ref: LocationRef): Promise<LocationRef> {
     // user storage maybe diff from template storage
     const dstStorage = StorageService.getStorageClient(ref.storageId);
     const dstDir = Path.resolve(ref.path);
     if (await dstStorage.exists(dstDir)) {
       log('Failed copying template to %s', dstDir);
       throw new Error('already have this folder, please give another name');
     }
-
-    // copy Composer data files
-    await copyDir(template.path, this.templateStorage, dstDir, dstStorage);
-
-    // copy runtime code files
-    await copyDir(runtime.path, this.templateStorage, dstDir, dstStorage);
+    await this.copyDataFilesTo(templateId, dstDir, dstStorage);
+    await this.copyRuntimeTo(dstDir, dstStorage);
     return ref;
   }
 }

Composer/packages/server/src/models/bot/botProject.ts

@@ -20,12 +20,14 @@ import { copyDir } from '../../utility/storage';
 import StorageService from '../../services/storage';
 import { IEnvironment, EnvironmentProvider } from '../environment';
 import { ISettingManager, OBFUSCATED_VALUE } from '../settings';
+import log from '../../logger';
 
 import { IFileStorage } from './../storage/interface';
 import { LocationRef, LuisStatus, FileUpdateType } from './interface';
 import { LuPublisher } from './luPublisher';
 import { DialogSetting } from './interface';
 
+const debug = log.extend('bot-project');
 const DIALOGFOLDER = 'ComposerDialogs';
 
 const oauthInput = () => ({
@@ -385,6 +387,7 @@ export class BotProject {
   private _createFile = async (relativePath: string, content: string) => {
     const absolutePath = Path.resolve(this.dir, relativePath);
     await this.ensureDirExists(Path.dirname(absolutePath));
+    debug('Creating file: %s', absolutePath);
     await this.fileStorage.writeFile(absolutePath, content);
 
     // update this.files which is memory cache of all files
@@ -457,6 +460,7 @@ export class BotProject {
       return;
     }
     if (!(await this.fileStorage.exists(dir))) {
+      debug('Creating directory: %s', dir);
       await this.fileStorage.mkDir(dir, { recursive: true });
     }
   };