From 89faea0b8c95281e0bb6574c1c53126a084acf56 Mon Sep 17 00:00:00 2001 From: VanyLaw Date: Fri, 10 Jan 2020 01:55:27 +0800 Subject: [PATCH] feat: Start Botproject runtime in bot folder (#1672) * start runtime from samplpebot csharp runtime * fix * fix * fix * use logger instead of console log * fix comment * fix comment * add port usable validation and use log instead console log * fix comments * move build script from template to server * polish output error message * remove environment * fix comments and add back the build script into template runtime * add migrate old bot feature * change docker file * remove annotation * fix docker * simpler dockerfile * fix dockerfile bug and port mapping * add bash script for unix, and remove pwsh from docker * split dockerfile into two * use nultistage instead of two dockerfile * update yarn lock * add more debug logs * fix settings file mis-created in server folder Co-authored-by: Lu Han <32191031+luhan2017@users.noreply.github.com> Co-authored-by: Chris Whitten Co-authored-by: Andy Brown --- BotProject/CSharp/Dockerfile | 20 -- BotProject/Templates/CSharp/Program.cs | 14 ++ .../CSharp/Scripts/build_runtime.ps1 | 25 +++ .../Templates/CSharp/Scripts/build_runtime.sh | 11 + Composer/Dockerfile | 64 +++++- Composer/package.json | 2 +- .../packages/client/src/constants/index.ts | 1 + .../packages/client/src/store/action/bot.ts | 9 +- .../client/src/store/action/setting.ts | 7 +- .../client/src/store/reducer/index.ts | 1 + Composer/packages/server/package.json | 1 + .../server/src/controllers/connector.ts | 26 ++- .../server/src/models/asset/assetManager.ts | 25 +-- .../server/src/models/bot/botProject.ts | 4 + .../models/connector/csharpBotConnector.ts | 207 +++++++++++++----- .../models/environment/defaultEnvironment.ts | 2 +- .../models/settings/defaultSettingManager.ts | 4 + .../src/models/settings/fileSettingManager.ts | 6 +- .../server/src/models/settings/interface.ts | 2 +- .../packages/server/src/settings/index.ts | 2 +- Composer/yarn.lock | 5 + docker-compose.yml | 9 +- 22 files changed, 330 insertions(+), 117 deletions(-) delete mode 100644 BotProject/CSharp/Dockerfile create mode 100644 BotProject/Templates/CSharp/Scripts/build_runtime.ps1 create mode 100644 BotProject/Templates/CSharp/Scripts/build_runtime.sh diff --git a/BotProject/CSharp/Dockerfile b/BotProject/CSharp/Dockerfile deleted file mode 100644 index 9893fa62bf..0000000000 --- a/BotProject/CSharp/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM mcr.microsoft.com/dotnet/core/sdk:2.1-alpine AS build - -WORKDIR /src/botproject/csharp - -COPY *.sln . -COPY *.csproj . -COPY Tests/*.csproj Tests/ -COPY NuGet.Config . - -# run restore to create a distinct layer -RUN dotnet restore - -COPY . . -RUN dotnet publish -o out - -FROM mcr.microsoft.com/dotnet/core/aspnet:2.1-alpine AS runtime -WORKDIR /app/botproject/csharp -COPY --from=build /src/botproject/csharp/ComposerDialogs ./ComposerDialogs -COPY --from=build /src/botproject/csharp/out . -CMD ["dotnet", "BotProject.dll"] diff --git a/BotProject/Templates/CSharp/Program.cs b/BotProject/Templates/CSharp/Program.cs index d1a3176b0f..fd4a5e14f0 100644 --- a/BotProject/Templates/CSharp/Program.cs +++ b/BotProject/Templates/CSharp/Program.cs @@ -2,6 +2,8 @@ // Licensed under the MIT License. using System; +using System.Diagnostics; +using System.IO; using Microsoft.AspNetCore; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Configuration; @@ -24,8 +26,20 @@ public static IWebHost BuildWebHost(string[] args) => config .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) .AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true, reloadOnChange: true) + .AddJsonFile($"ComposerDialogs/settings/appsettings.json", optional: true, reloadOnChange: true) .AddJsonFile($"luis.settings.{env.EnvironmentName}.{luisAuthoringRegion}.json", optional: true, reloadOnChange: true) .AddJsonFile($"luis.settings.{Environment.UserName}.{luisAuthoringRegion}.json", optional: true, reloadOnChange: true); + try + { + foreach (string filePath in Directory.GetFiles($"ComposerDialogs", "generated/luis.settings.*.json")) + { + config.AddJsonFile(filePath, optional: true, reloadOnChange: true); + } + } + catch (Exception ex) + { + Trace.WriteLine(ex.Message); + } if (env.IsDevelopment()) { diff --git a/BotProject/Templates/CSharp/Scripts/build_runtime.ps1 b/BotProject/Templates/CSharp/Scripts/build_runtime.ps1 new file mode 100644 index 0000000000..dc6cd87d9a --- /dev/null +++ b/BotProject/Templates/CSharp/Scripts/build_runtime.ps1 @@ -0,0 +1,25 @@ +Param( + [object] $config, + [string] $customSettingFolder, + [string] $luisAuthroingKey, + [SecureString] $appPassword, + [string] $projFolder = $(Join-Path $(Get-Location) BotProject CSharp) +) + +if ($PSVersionTable.PSVersion.Major -lt 6){ + Write-Host "! Powershell 6 is required, current version is $($PSVersionTable.PSVersion.Major), please refer following documents for help." + Write-Host "For Windows - https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-windows?view=powershell-6" + Write-Host "For Mac - https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-macos?view=powershell-6" + Break +} + +if ((dotnet --version) -lt 3) { + Write-Host "! dotnet core 3.0 is required, please refer following documents for help." + Write-Host "https://dotnet.microsoft.com/download/dotnet-core/3.0" + Break +} + +# This command need dotnet core more than 3.0 +dotnet user-secrets init + +dotnet build \ No newline at end of file diff --git a/BotProject/Templates/CSharp/Scripts/build_runtime.sh b/BotProject/Templates/CSharp/Scripts/build_runtime.sh new file mode 100644 index 0000000000..3a87dea815 --- /dev/null +++ b/BotProject/Templates/CSharp/Scripts/build_runtime.sh @@ -0,0 +1,11 @@ +versionString=`dotnet --version` +versionNum=`echo $versionString | cut -d . -f 1` +if [[ $versionNum -lt 3 ]] +then + echo "! dotnet core 3.0 is required, please refer following documents for help. +https://dotnet.microsoft.com/download/dotnet-core/3.0" + exit 1 +else + dotnet user-secrets init + dotnet build +fi diff --git a/Composer/Dockerfile b/Composer/Dockerfile index 351572a63c..b8c9fdfda3 100644 --- a/Composer/Dockerfile +++ b/Composer/Dockerfile @@ -18,8 +18,8 @@ ENV NODE_ENV "production" RUN yarn build:prod -# use a multi-stage build to reduce the final image size -FROM node:12-alpine + +FROM node:12-alpine as composerbasic WORKDIR /app/Composer COPY --from=build /src/Composer/yarn.lock . @@ -31,4 +31,62 @@ COPY --from=build /src/Composer/packages/tools ./packages/tools ENV NODE_ENV "production" RUN yarn --production --frozen-lockfile --force && yarn cache clean WORKDIR /app/Composer -CMD ["yarn", "start:server"] + + +FROM composerbasic + +RUN apk add --no-cache \ + ca-certificates \ + \ + # .NET Core dependencies + krb5-libs \ + libgcc \ + libintl \ + libssl1.1 \ + libstdc++ \ + zlib + +# Install .Net Core SDK +ENV \ + # Unset the value from the base image + ASPNETCORE_URLS= \ + # Disable the invariant mode (set in base image) + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=false \ + # Enable correct mode for dotnet watch (only mode supported in a container) + DOTNET_USE_POLLING_FILE_WATCHER=true \ + LC_ALL=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + # Skip extraction of XML docs - generally not useful within an image/container - helps performance + NUGET_XMLDOC_MODE=skip \ + # PowerShell telemetry for docker image usage + POWERSHELL_DISTRIBUTION_CHANNEL=PSDocker-DotnetCoreSDK-Alpine-3.10 + +# Add dependencies for disabling invariant mode (set in base image) +RUN apk add --no-cache icu-libs + +# Install .NET Core 2.1 +ENV DOTNET_SDK_VERSION 2.1.607 + +RUN wget -O dotnet.tar.gz https://dotnetcli.azureedge.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-musl-x64.tar.gz \ + && dotnet_sha512='61caf6602b8a2aa89769b3e91ddaec963d8ab9f802cd7f6c6da4f02426358712bc2bb0930e7ee3a81d75c7607039543b554cb8ed50e45610655f9e91ed0f2f17' \ + && echo "$dotnet_sha512 dotnet.tar.gz" | sha512sum -c - \ + && mkdir -p /usr/share/dotnet \ + && tar -C /usr/share/dotnet -xzf dotnet.tar.gz \ + && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet \ + && rm dotnet.tar.gz + +# Install .NET Core SDK 3.0 +ENV DOTNET_SDK_VERSION 3.0.101 + +RUN wget -O dotnet.tar.gz https://dotnetcli.azureedge.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-musl-x64.tar.gz \ + && dotnet_sha512='98cc98f58187d208bd388f8c71862ea75e50ca25666e265f40a4e7c28082c2784738172e8ae4af7815057f7c57072cbe4fc03301d01738fc1ed5bb5e4d30a363' \ + && echo "$dotnet_sha512 dotnet.tar.gz" | sha512sum -c - \ + && tar -C /usr/share/dotnet -xzf dotnet.tar.gz \ + && rm dotnet.tar.gz + +# Enable detection of running in a container +ENV DOTNET_RUNNING_IN_CONTAINER=true \ + # Set the invariant mode since icu_libs isn't included (see https://github.com/dotnet/announcements/issues/20) + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=true + +CMD ["yarn","start:server"] diff --git a/Composer/package.json b/Composer/package.json index dce8d3039a..7f337e5a3e 100644 --- a/Composer/package.json +++ b/Composer/package.json @@ -29,7 +29,7 @@ "build:client": "yarn workspace @bfc/client build", "build:tools": "yarn workspace @bfc/tools build:all", "start": "cross-env NODE_ENV=production PORT=3000 yarn start:server", - "startall": "node scripts/update.js && concurrently --kill-others-on-fail \"npm:runtime\" \"npm:start\"", + "startall": "node scripts/update.js && yarn start", "start:dev": "concurrently \"npm:start:client\" \"npm:start:server:dev\"", "start:client": "yarn workspace @bfc/client start", "start:server": "yarn workspace @bfc/server start", diff --git a/Composer/packages/client/src/constants/index.ts b/Composer/packages/client/src/constants/index.ts index d3d416d0ce..1f7d922ba1 100644 --- a/Composer/packages/client/src/constants/index.ts +++ b/Composer/packages/client/src/constants/index.ts @@ -67,6 +67,7 @@ export enum ActionTypes { SET_CREATION_FLOW_STATUS = 'SET_CREATION_FLOW_STATUS', SET_DESIGN_PAGE_LOCATION = 'SET_DESIGN_PAGE_LOCATION', CONNECT_BOT_SUCCESS = 'CONNECT_BOT_SUCCESS', + CONNECT_BOT_FAILURE = 'CONNECT_BOT_FAILURE', RELOAD_BOT_SUCCESS = 'RELOAD_BOT_SUCCESS', SYNC_ENV_SETTING = 'SYNC_ENV_SETTING', GET_ENV_SETTING = 'GET_ENV_SETTING', diff --git a/Composer/packages/client/src/store/action/bot.ts b/Composer/packages/client/src/store/action/bot.ts index 976d9cd4d9..e35dd7f6e5 100644 --- a/Composer/packages/client/src/store/action/bot.ts +++ b/Composer/packages/client/src/store/action/bot.ts @@ -13,6 +13,7 @@ export const connectBot: ActionCreator = async (store, settings) => { try { const res = await httpClient.get(path); + await reloadBot(store, settings); store.dispatch({ type: ActionTypes.CONNECT_BOT_SUCCESS, payload: { @@ -21,10 +22,14 @@ export const connectBot: ActionCreator = async (store, settings) => { }, }); } catch (err) { + store.dispatch({ + type: ActionTypes.CONNECT_BOT_FAILURE, + payload: { + status: 'unConnected', + }, + }); throw new Error(err.response.data.message); } - - await reloadBot(store, settings); }; // return only the connect URL -- do not reload diff --git a/Composer/packages/client/src/store/action/setting.ts b/Composer/packages/client/src/store/action/setting.ts index 8dd79ac515..dda0bbef98 100644 --- a/Composer/packages/client/src/store/action/setting.ts +++ b/Composer/packages/client/src/store/action/setting.ts @@ -2,6 +2,7 @@ // Licensed under the MIT License. import get from 'lodash/get'; +import has from 'lodash/has'; import { SensitiveProperties } from '@bfc/shared'; import { ActionCreator, DialogSetting } from '../types'; @@ -27,8 +28,10 @@ export const setSettings: ActionCreator = async ( }); // set value in local storage for (const property of SensitiveProperties) { - const propertyValue = get(settings, property); - settingsStorage.setField(botName, property, propertyValue ? propertyValue : ''); + if (has(settings, property)) { + const propertyValue = get(settings, property, ''); + settingsStorage.setField(botName, property, propertyValue); + } } // set value to server const suffix = slot ? `/${slot}` : ''; diff --git a/Composer/packages/client/src/store/reducer/index.ts b/Composer/packages/client/src/store/reducer/index.ts index 8ddd5dbe20..5f75345bfe 100644 --- a/Composer/packages/client/src/store/reducer/index.ts +++ b/Composer/packages/client/src/store/reducer/index.ts @@ -313,6 +313,7 @@ export const reducer = createReducer({ [ActionTypes.REMOVE_LU_FAILURE]: noOp, [ActionTypes.PUBLISH_LU_SUCCCESS]: updateLuTemplate, [ActionTypes.CONNECT_BOT_SUCCESS]: setBotStatus, + [ActionTypes.CONNECT_BOT_FAILURE]: setBotStatus, [ActionTypes.RELOAD_BOT_SUCCESS]: setBotLoadErrorMsg, // [ActionTypes.RELOAD_BOT_FAILURE]: setBotLoadErrorMsg, [ActionTypes.SET_ERROR]: setError, diff --git a/Composer/packages/server/package.json b/Composer/packages/server/package.json index 7333faf1ca..01d4c84bcb 100644 --- a/Composer/packages/server/package.json +++ b/Composer/packages/server/package.json @@ -69,6 +69,7 @@ "express": "^4.16.4", "form-data": "^2.3.3", "format-message": "^6.2.1", + "get-port": "^5.1.0", "globby": "^9.1.0", "http-errors": "^1.7.2", "immer": "^2.1.4", diff --git a/Composer/packages/server/src/controllers/connector.ts b/Composer/packages/server/src/controllers/connector.ts index ee721da2b4..14ea9cc88c 100644 --- a/Composer/packages/server/src/controllers/connector.ts +++ b/Composer/packages/server/src/controllers/connector.ts @@ -1,27 +1,28 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +import merge from 'lodash/merge'; import { BotEnvironments } from '../models/connector'; -import { EnvironmentProvider } from '../models/environment'; +import { BotProjectService } from '../services/project'; async function connect(req: any, res: any) { try { const hostName = req.hostname; const env: BotEnvironments = req.query && req.query.botEnvironment ? req.query.botEnvironment : 'production'; - const environment = EnvironmentProvider.getCurrent(); - const botEndpoint = await environment.getBotConnector().connect(env || 'production', hostName); + const environment = BotProjectService.getCurrentBotProject()?.environment; + const botEndpoint = await environment?.getBotConnector().connect(env || 'production', hostName); res.send({ botEndpoint }); } catch (error) { res.status(400).json({ - message: 'cannot connect to a bot runtime, make sure you start the bot runtime', + message: error.message || 'cannot connect to a bot runtime, make sure you start the bot runtime', }); } } async function getPublishHistory(req: any, res: any) { try { - const environment = EnvironmentProvider.getCurrent(); - const history = await environment.getBotConnector().getPublishHistory(); + const environment = BotProjectService.getCurrentBotProject()?.environment; + const history = await environment?.getBotConnector().getPublishHistory(); res.send(history); } catch (error) { res.status(400).json({ @@ -32,8 +33,9 @@ async function getPublishHistory(req: any, res: any) { async function sync(req: any, res: any) { try { - const environment = EnvironmentProvider.getCurrent(); - await environment.getBotConnector().sync({ ...req.body, user: req.user }); + const environment = BotProjectService.getCurrentBotProject()?.environment; + const settingsInDisk = await environment?.getSettingsManager().get(); + await environment?.getBotConnector().sync(merge({}, settingsInDisk, req.body, { user: req.user })); res.send('OK'); } catch (error) { res.status(400).json({ @@ -45,8 +47,8 @@ async function sync(req: any, res: any) { async function publish(req: any, res: any) { try { const label = req.params ? req.params.label : undefined; - const environment = EnvironmentProvider.getCurrent(); - await environment.getBotConnector().publish({ ...req.body, user: req.user }, label); + const environment = BotProjectService.getCurrentBotProject()?.environment; + await environment?.getBotConnector().publish({ ...req.body, user: req.user }, label); res.send('OK'); } catch (error) { res.status(400).json({ @@ -56,8 +58,8 @@ async function publish(req: any, res: any) { } function status(req: any, res: any) { - const environment = EnvironmentProvider.getCurrent(); - res.send(environment.getBotConnector().status); + const environment = BotProjectService.getCurrentBotProject()?.environment; + res.send(environment?.getBotConnector().status); } export const BotConnectorController = { diff --git a/Composer/packages/server/src/models/asset/assetManager.ts b/Composer/packages/server/src/models/asset/assetManager.ts index 3714dc281d..f827ffb458 100644 --- a/Composer/packages/server/src/models/asset/assetManager.ts +++ b/Composer/packages/server/src/models/asset/assetManager.ts @@ -10,6 +10,7 @@ import { LocationRef } from '../bot/interface'; import { Path } from '../../utility/path'; import { copyDir } from '../../utility/storage'; import StorageService from '../../services/storage'; +import { IFileStorage } from '../storage/interface'; interface TemplateData { [key: string]: { @@ -156,23 +157,25 @@ export class AssetManager { return output; } - public async copyProjectTemplateTo(templateId: string, ref: LocationRef): Promise { - if (this.projectTemplates.length === 0) { - await this.getProjectTemplates(); - } - if (this.runtimeTemplates.length === 0) { - await this.getProjectRuntime(); - } + public async copyDataFilesTo(templateId: string, dstDir: string, dstStorage: IFileStorage) { const template = find(this.projectTemplates, { id: templateId }); if (template === undefined || template.path === undefined) { throw new Error(`no such template with id ${templateId}`); } + // copy Composer data files + await copyDir(template.path, this.templateStorage, dstDir, dstStorage); + } + public async copyRuntimeTo(dstDir: string, dstStorage: IFileStorage) { const runtime = find(this.runtimeTemplates, { id: DEFAULT_RUNTIME }); if (runtime === undefined || runtime.path === undefined) { throw new Error(`no such runtime with id ${DEFAULT_RUNTIME}`); } + // copy runtime code files + await copyDir(runtime.path, this.templateStorage, dstDir, dstStorage); + } + public async copyProjectTemplateTo(templateId: string, ref: LocationRef): Promise { // user storage maybe diff from template storage const dstStorage = StorageService.getStorageClient(ref.storageId); const dstDir = Path.resolve(ref.path); @@ -180,12 +183,8 @@ export class AssetManager { log('Failed copying template to %s', dstDir); throw new Error('already have this folder, please give another name'); } - - // copy Composer data files - await copyDir(template.path, this.templateStorage, dstDir, dstStorage); - - // copy runtime code files - await copyDir(runtime.path, this.templateStorage, dstDir, dstStorage); + await this.copyDataFilesTo(templateId, dstDir, dstStorage); + await this.copyRuntimeTo(dstDir, dstStorage); return ref; } } diff --git a/Composer/packages/server/src/models/bot/botProject.ts b/Composer/packages/server/src/models/bot/botProject.ts index f4cf30299c..876b234db5 100644 --- a/Composer/packages/server/src/models/bot/botProject.ts +++ b/Composer/packages/server/src/models/bot/botProject.ts @@ -20,12 +20,14 @@ import { copyDir } from '../../utility/storage'; import StorageService from '../../services/storage'; import { IEnvironment, EnvironmentProvider } from '../environment'; import { ISettingManager, OBFUSCATED_VALUE } from '../settings'; +import log from '../../logger'; import { IFileStorage } from './../storage/interface'; import { LocationRef, LuisStatus, FileUpdateType } from './interface'; import { LuPublisher } from './luPublisher'; import { DialogSetting } from './interface'; +const debug = log.extend('bot-project'); const DIALOGFOLDER = 'ComposerDialogs'; const oauthInput = () => ({ @@ -385,6 +387,7 @@ export class BotProject { private _createFile = async (relativePath: string, content: string) => { const absolutePath = Path.resolve(this.dir, relativePath); await this.ensureDirExists(Path.dirname(absolutePath)); + debug('Creating file: %s', absolutePath); await this.fileStorage.writeFile(absolutePath, content); // update this.files which is memory cache of all files @@ -457,6 +460,7 @@ export class BotProject { return; } if (!(await this.fileStorage.exists(dir))) { + debug('Creating directory: %s', dir); await this.fileStorage.mkDir(dir, { recursive: true }); } }; diff --git a/Composer/packages/server/src/models/connector/csharpBotConnector.ts b/Composer/packages/server/src/models/connector/csharpBotConnector.ts index 8a17c3eeee..d6d825e605 100644 --- a/Composer/packages/server/src/models/connector/csharpBotConnector.ts +++ b/Composer/packages/server/src/models/connector/csharpBotConnector.ts @@ -1,92 +1,186 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +import { parse as urlParse } from 'url'; +import { ChildProcess, spawn } from 'child_process'; import fs from 'fs'; -import Path from 'path'; -import axios from 'axios'; -import archiver from 'archiver'; -import FormData from 'form-data'; +import getPort from 'get-port'; import { BotProjectService } from '../../services/project'; import { DialogSetting } from '../bot/interface'; +import { Path } from '../../utility/path'; +import log from '../../logger'; +import AssetService from '../../services/asset'; +import { IFileStorage } from '../storage/interface'; import { BotConfig, BotEnvironments, BotStatus, IBotConnector, IPublishHistory } from './interface'; +const debug = log.extend('bot-runtime'); +const buildDebug = debug.extend('build'); +const runtimeDebugs: { [key: string]: debug.Debugger } = {}; export class CSharpBotConnector implements IBotConnector { - private adminEndpoint: string; + public status: BotStatus = BotStatus.NotConnected; private endpoint: string; - constructor(adminEndpoint: string, endpoint: string) { - this.adminEndpoint = adminEndpoint; + static botRuntimes: { [key: string]: ChildProcess } = {}; + constructor(endpoint: string) { this.endpoint = endpoint; } - public status: BotStatus = BotStatus.NotConnected; - - connect = async (_: BotEnvironments, __: string) => { - // confirm bot runtime is listening here - try { - await axios.get(this.adminEndpoint + '/api/admin'); - } catch (err) { - throw new Error(err); + static stopAll = (signal: string) => { + for (const pid in CSharpBotConnector.botRuntimes) { + const runtime = CSharpBotConnector.botRuntimes[pid]; + runtime.kill(signal); + runtimeDebugs[pid]('successfully stopped bot runtime'); + delete CSharpBotConnector.botRuntimes[pid]; } + }; + private stop = () => { + CSharpBotConnector.stopAll('SIGINT'); this.status = BotStatus.NotConnected; - - return `${this.endpoint}/api/messages`; }; - sync = async (config: DialogSetting) => { - // archive the project - // send to bot runtime service - const currentProject = BotProjectService.getCurrentBotProject(); - if (currentProject === undefined) { - throw new Error('no project is opened, nothing to sync'); + private isOldBot = (dir: string): boolean => { + // check bot the bot version through build script existence. + if (process.platform === 'win32') { + return !fs.existsSync(Path.resolve(dir, './Scripts/build_runtime.ps1')); + } else { + return !fs.existsSync(Path.resolve(dir, './Scripts/build_runtime.sh')); } - const dir = Path.join(currentProject.dataDir); - const luisConfig = currentProject.luPublisher.getLuisConfig(); - await this.archiveDirectory(dir, './tmp.zip'); - const content = fs.readFileSync('./tmp.zip'); - - const form = new FormData(); - form.append('file', content, 'bot.zip'); + }; - if (luisConfig && luisConfig.authoringKey !== null && !currentProject.checkLuisPublished()) { - throw new Error('Please publish your Luis models'); + private migrateBot = async (dir: string, storage: IFileStorage) => { + if (this.isOldBot(dir)) { + // cover the old bot runtime with new runtime template + await AssetService.manager.copyRuntimeTo(dir, storage); } + }; - if (luisConfig) { - form.append('endpointKey', luisConfig.endpointKey || luisConfig.authoringKey || ''); - } + private buildProcess = async (dir: string): Promise => { + // build bot runtime + return new Promise((resolve, reject) => { + let shell = 'sh'; + let script = './Scripts/build_runtime.sh'; + if (process.platform === 'win32') { + shell = 'pwsh'; + script = './Scripts/build_runtime.ps1'; + } + const build = spawn(`${shell}`, [`${script}`], { + cwd: dir, + stdio: ['pipe', 'pipe', 'pipe'], + }); + buildDebug('building bot runtime: %d', build.pid); + build.stdout && + build.stdout.on('data', function(str) { + buildDebug('%s', str); + }); + build.stderr && + build.stderr.on('data', function(err) { + reject(err.toString()); + }); + build.on('exit', function(code) { + resolve(code); + }); + }); + }; - config = { - ...(await currentProject.settingManager.get(currentProject.environment.getDefaultSlot(), false)), - ...config, - }; + private getConnectorConfig = (config: DialogSetting) => { + const configList: string[] = []; if (config.MicrosoftAppPassword) { - form.append('microsoftAppPassword', config.MicrosoftAppPassword); + configList.push('--MicrosoftAppPassword'); + configList.push(config.MicrosoftAppPassword); } - try { - await axios.post(this.adminEndpoint + '/api/admin', form, { headers: form.getHeaders() }); - } catch (err) { - throw new Error('Unable to sync content to bot runtime'); + if (config.luis) { + if (config.luis.authoringKey) { + configList.push('--luis:endpointKey'); + configList.push(config.luis.authoringKey); + } + if (config.luis.authoringRegion) { + configList.push('--luis:endpoint'); + configList.push(`https://${config.luis.authoringRegion}.api.cognitive.microsoft.com`); + } } + + return configList; }; - archiveDirectory = (src: string, dest: string) => { - return new Promise((resolve, reject) => { - const archive = archiver('zip'); - const output = fs.createWriteStream(dest); + private addListeners = (child: ChildProcess, handler: Function, resolve: Function, reject: Function) => { + const currentDebugger = runtimeDebugs[child.pid]; + let erroutput = ''; + child.stdout && + child.stdout.on('data', (data: any) => { + currentDebugger('%s', data); + resolve(child.pid); + }); + + child.stderr && + child.stderr.on('data', (err: any) => { + erroutput += err.toString(); + }); + + child.on('exit', code => { + currentDebugger('exit %d', code); + handler(); + if (code !== 0) { + currentDebugger('exit %d: %s', code, erroutput); + reject(erroutput); + } + }); + + child.on('message', msg => { + currentDebugger('%s', msg); + }); + }; - archive.pipe(output); - archive.directory(src, false); - archive.finalize(); + private getBotPathAndStorage = () => { + const currentProject = BotProjectService.getCurrentBotProject(); + if (currentProject === undefined) { + throw new Error('no project is opened, nothing to sync'); + } + return { dir: Path.join(currentProject.dir), storage: currentProject.fileStorage }; + }; - output.on('close', () => resolve(archive)); - archive.on('error', err => reject(err)); + private start = async (dir: string, config: DialogSetting): Promise => { + return new Promise((resolve, reject) => { + const runtime = spawn( + 'dotnet', + ['bin/Debug/netcoreapp2.1/BotProject.dll', `--urls`, this.endpoint, ...this.getConnectorConfig(config)], + { + cwd: dir, + stdio: ['ignore', 'pipe', 'pipe'], + } + ); + // extend runtime debugger + runtimeDebugs[runtime.pid] = debug.extend(`process (${runtime.pid})`); + runtimeDebugs[runtime.pid]('bot runtime started'); + CSharpBotConnector.botRuntimes[runtime.pid] = runtime; + this.addListeners(runtime, this.stop, resolve, reject); }); }; + connect = async (_: BotEnvironments, __: string) => { + const originPort = urlParse(this.endpoint).port; + const port = await getPort({ host: 'localhost', port: parseInt(originPort || '3979') }); + return `http://localhost:${port}/api/messages`; + }; + + sync = async (config: DialogSetting) => { + try { + this.stop(); + const { dir, storage } = this.getBotPathAndStorage(); + await this.migrateBot(dir, storage); + + await this.buildProcess(dir); + await this.start(dir, config); + this.status = BotStatus.Connected; + } catch (err) { + this.stop(); + this.status = BotStatus.NotConnected; + throw new Error('Error while syncing bot runtime.'); + } + }; + getEditingStatus = (): Promise => { return new Promise(resolve => { resolve(true); @@ -109,3 +203,10 @@ export class CSharpBotConnector implements IBotConnector { }); }; } +const cleanup = (signal: NodeJS.Signals) => { + CSharpBotConnector.stopAll(signal); + process.exit(0); +}; +(['SIGINT', 'SIGTERM', 'SIGQUIT'] as NodeJS.Signals[]).forEach((signal: NodeJS.Signals) => { + process.on(signal, cleanup.bind(null, signal)); +}); diff --git a/Composer/packages/server/src/models/environment/defaultEnvironment.ts b/Composer/packages/server/src/models/environment/defaultEnvironment.ts index d7b2a8e7ba..c18d8a9290 100644 --- a/Composer/packages/server/src/models/environment/defaultEnvironment.ts +++ b/Composer/packages/server/src/models/environment/defaultEnvironment.ts @@ -18,7 +18,7 @@ export class DefaultEnvironment implements IEnvironment { public constructor(config: IEnvironmentConfig) { this.config = config; this.settingManager = new DefaultSettingManager(this.config.basePath); - this.botConnector = new CSharpBotConnector(this.config.adminEndpoint, this.config.endpoint); + this.botConnector = new CSharpBotConnector(this.config.endpoint); } public getEnvironmentName(_: string): string | undefined { diff --git a/Composer/packages/server/src/models/settings/defaultSettingManager.ts b/Composer/packages/server/src/models/settings/defaultSettingManager.ts index 3df934368e..a985f80b7d 100644 --- a/Composer/packages/server/src/models/settings/defaultSettingManager.ts +++ b/Composer/packages/server/src/models/settings/defaultSettingManager.ts @@ -5,9 +5,12 @@ import omit from 'lodash/omit'; import { SensitiveProperties } from '@bfc/shared'; import { Path } from '../../utility/path'; +import log from '../../logger'; import { FileSettingManager } from './fileSettingManager'; +const debug = log.extend('default-settings-manager'); + export class DefaultSettingManager extends FileSettingManager { constructor(basePath: string) { super(basePath); @@ -45,6 +48,7 @@ export class DefaultSettingManager extends FileSettingManager { const path = this.getPath(slot); const dir = Path.dirname(path); if (!(await this.storage.exists(dir))) { + debug('Storage path does not exist. Creating directory now: %s', dir); await this.storage.mkDir(dir, { recursive: true }); } // remove sensitive values before saving to disk diff --git a/Composer/packages/server/src/models/settings/fileSettingManager.ts b/Composer/packages/server/src/models/settings/fileSettingManager.ts index c8ea14c67e..c4923f985c 100644 --- a/Composer/packages/server/src/models/settings/fileSettingManager.ts +++ b/Composer/packages/server/src/models/settings/fileSettingManager.ts @@ -3,9 +3,12 @@ import { Path } from '../../utility/path'; import { LocalDiskStorage } from '../storage/localDiskStorage'; +import log from '../../logger'; import { ISettingManager, OBFUSCATED_VALUE } from '.'; +const debug = log.extend('file-settings-manager'); + // TODO: this causes tests to fail const subPath = 'ComposerDialogs/settings/appsettings.json'; @@ -18,7 +21,7 @@ export class FileSettingManager implements ISettingManager { this.storage = new LocalDiskStorage(); } - public get = async (slot: string, obfuscate: boolean): Promise => { + public get = async (slot = '', obfuscate = false): Promise => { this.validateSlot(slot); const path = this.getPath(slot); @@ -50,6 +53,7 @@ export class FileSettingManager implements ISettingManager { const dir = Path.dirname(path); if (!(await this.storage.exists(dir))) { + debug('Storage path does not exist. Creating directory now: %s', dir); await this.storage.mkDir(dir, { recursive: true }); } await this.storage.writeFile(path, JSON.stringify(settings, null, 2)); diff --git a/Composer/packages/server/src/models/settings/interface.ts b/Composer/packages/server/src/models/settings/interface.ts index 3514aae953..2a839af129 100644 --- a/Composer/packages/server/src/models/settings/interface.ts +++ b/Composer/packages/server/src/models/settings/interface.ts @@ -4,6 +4,6 @@ export const OBFUSCATED_VALUE = '*****'; export interface ISettingManager { - get(slot: string, obfuscate: boolean): Promise; + get(slot?: string, obfuscate?: boolean): Promise; set(slot: string, settings: any): Promise; } diff --git a/Composer/packages/server/src/settings/index.ts b/Composer/packages/server/src/settings/index.ts index 9f0f16e34d..3c2b166d82 100644 --- a/Composer/packages/server/src/settings/index.ts +++ b/Composer/packages/server/src/settings/index.ts @@ -22,7 +22,7 @@ interface Settings { const envSettings: { [env: string]: Settings } = { development: { botAdminEndpoint: botEndpoint, - botEndpoint: 'http://localhost:3979', //botEndpoint, + botEndpoint: botEndpoint, assetsLibray: Path.resolve(__dirname, '../../assets'), botsFolder: botsFolder || Path.join(os.homedir(), 'Documents', 'Composer'), runtimeFolder, diff --git a/Composer/yarn.lock b/Composer/yarn.lock index fa75a114bf..4dbe07aed3 100644 --- a/Composer/yarn.lock +++ b/Composer/yarn.lock @@ -8718,6 +8718,11 @@ get-own-enumerable-property-symbols@^3.0.0: resolved "https://registry.yarnpkg.com/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.0.tgz#b877b49a5c16aefac3655f2ed2ea5b684df8d203" integrity sha512-CIJYJC4GGF06TakLg8z4GQKvDsx9EMspVxOYih7LerEL/WosUnFIww45CGfxfeKHqlg3twgUrYRT1O3WQqjGCg== +get-port@^5.1.0: + version "5.1.0" + resolved "https://registry.npmjs.org/get-port/-/get-port-5.1.0.tgz#a8f6510d0000f07d5c65653a4b0ae648fe832683" + integrity sha512-bjioH1E9bTQUvgaB6VycVy1QVbTZI41yTnF9qkZz6ixgy/uhCH6D63bKeZ6Code/07JYA61MeI94jSdHss8PNA== + get-stdin@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-6.0.0.tgz#9e09bf712b360ab9225e812048f71fde9c89657b" diff --git a/docker-compose.yml b/docker-compose.yml index d5f6c88df3..380431c1af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,7 @@ services: build: Composer ports: - "3000:3000" + - "3979:3979" volumes: - ~/Documents/Composer:/Bots - ./BotProject:/BotProject @@ -12,11 +13,5 @@ services: COMPOSER_BOTS_FOLDER: /Bots COMPOSER_RUNTIME_FOLDER: /BotProject/Templates COMPOSER_APP_DATA: /appdata/data.json - BOT_ENDPOINT: http://botruntime:80 PORT: 3000 - botruntime: - build: BotProject/CSharp - ports: - - "3979:80" - volumes: - - ~/Documents/Composer:/Bots + BOT_ENDPOINT: http://0.0.0.0:3979