build: make docker great again (#1709)

* wip docker

* fix docker build issues

* add ComposerDialogs folder to bot runtime image

* don't log error when git is missing

* make app data.json path configurable

allows us to save docker state across builds

* add compose override example

* no need to be in composer directory

* finish comment and remove dead code

* use correct syntax for run directive

.github/workflows/main.yml

@@ -67,3 +67,14 @@ jobs:
       - name: dotnet test
         run: dotnet test
         working-directory: BotProject/CSharp
+
+  docker-build:
+    name: Docker Build
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+      - name: docker-compose build
+        run: docker-compose build

.gitignore

@@ -397,3 +397,7 @@ MyBots/*
 
 # VsCode
 Composer/.vscode/
+
+# Docker App Data
+.appdata
+docker-compose.override.yml

BotProject/CSharp/Dockerfile

@@ -1,6 +1,6 @@
 FROM mcr.microsoft.com/dotnet/core/sdk:2.1-alpine AS build
 
-WORKDIR /app/botproject/csharp
+WORKDIR /src/botproject/csharp
 
 COPY *.sln .
 COPY *.csproj .
@@ -15,5 +15,6 @@ RUN dotnet publish -o out
 
 FROM mcr.microsoft.com/dotnet/core/aspnet:2.1-alpine AS runtime
 WORKDIR /app/botproject/csharp
-COPY --from=build /app/botproject/csharp/out .
-CMD ["dotnet", "BotProject.dll"]
+COPY --from=build /src/botproject/csharp/ComposerDialogs ./ComposerDialogs
+COPY --from=build /src/botproject/csharp/out .
+CMD ["dotnet", "BotProject.dll"]

Composer/.dockerignore

@@ -7,4 +7,7 @@
 **/server/tmp.zip
 # not ignore all lib folder because packages/lib, so probably we should rename that to libs
 packages/lib/*/lib
-packages/extensions/*/lib
+packages/extensions/*/lib
+
+Dockerfile
+.dockerignore

Composer/Dockerfile

@@ -1,39 +1,34 @@
-FROM node:12-alpine as build
-WORKDIR /src/Composer
-
-# yes, docker copy is really this stupid, https://github.com/moby/moby/issues/15858
-COPY yarn.lock .
-COPY .npmrc .
-COPY package.json .
-COPY packages/client/package.json ./packages/client/
-COPY yarn.lock ./packages/server/
-COPY packages/server/package.json ./packages/server/
-COPY packages/lib/package.json ./packages/lib/
-COPY packages/lib/code-editor/package.json ./packages/lib/code-editor/
-COPY packages/lib/shared/package.json ./packages/lib/shared/
-COPY packages/lib/indexers/package.json ./packages/lib/indexers/
-COPY packages/extensions/package.json ./packages/extensions/
-COPY packages/lib/eslint-plugin-bfcomposer/package.json ./packages/lib/eslint-plugin-bfcomposer/
-COPY packages/extensions/obiformeditor/package.json ./packages/extensions/obiformeditor/
-COPY packages/extensions/visual-designer/package.json ./packages/extensions/visual-designer/
+#################
+#
+# Because Composer is organized as a monorepo with multiple packages
+# managed by yarn workspaces, our Dockerfile may not look like other
+# node / react projects. Specifically, we have to add all source files
+# before doing yarn install due to yarn workspace symlinking.
+#
+################
 
-# run yarn install as a distinct layer
-RUN yarn install
+FROM node:12-alpine as build
 
+WORKDIR /src/Composer
 COPY . .
+# run yarn install as a distinct layer
+RUN yarn install --frozen-lock-file
 ENV NODE_OPTIONS "--max-old-space-size=4096"
+ENV NODE_ENV "production"
 RUN yarn build:prod
 
+
 # use a multi-stage build to reduce the final image size
 FROM node:12-alpine
 
-WORKDIR /app/Composer/server
-COPY --from=build /src/Composer/.npmrc .
-COPY --from=build /src/Composer/packages/lib ./lib
-COPY --from=build /src/Composer/packages/server .
-
-# update server package json to point to local packages
-RUN node ./prepare-prod.js
+WORKDIR /app/Composer
+COPY --from=build /src/Composer/yarn.lock .
+COPY --from=build /src/Composer/package.json .
+COPY --from=build /src/Composer/packages/server ./packages/server
+COPY --from=build /src/Composer/packages/lib ./packages/lib
+COPY --from=build /src/Composer/packages/tools ./packages/tools
 
-RUN yarn --production && yarn cache clean
-CMD ["node", "build/server.js"]
+ENV NODE_ENV "production"
+RUN yarn --production --frozen-lockfile --force && yarn cache clean
+WORKDIR /app/Composer
+CMD ["yarn", "start:server"]

Composer/packages/client/config/env.js

@@ -42,7 +42,7 @@ dotenvFiles.forEach(dotenvFile => {
 
 function getGitSha() {
   try {
-    const sha = execSync('git rev-parse --short master');
+    const sha = execSync('git rev-parse --short', { stdio: ['ignore', 'ignore', 'ignore'] });
     return sha;
   } catch (e) {
     return 'test';

Composer/packages/server/src/settings/env.ts

@@ -1,6 +1,8 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+import path from 'path';
+
 export const absHosted = process.env.COMPOSER_AUTH_PROVIDER === 'abs-h';
 export const absHostRoot = process.env.WEBSITE_HOSTNAME
   ? `https://${process.env.WEBSITE_HOSTNAME}`
@@ -11,4 +13,7 @@ if (folder && folder.endsWith(':')) {
   folder = folder + '/';
 }
 
+export const environment = process.env.NODE_ENV || 'development';
 export const botsFolder = folder;
+export const botEndpoint = process.env.BOT_ENDPOINT || 'http://localhost:3979';
+export const appDataPath = process.env.COMPOSER_APP_DATA || path.resolve(__dirname, '../../data.json');

Composer/packages/server/src/settings/index.ts

@@ -1,27 +1,37 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+import os from 'os';
+
 import merge from 'lodash/merge';
 
 import log from '../logger';
+import { Path } from '../utility/path';
 
-import settings from './settings';
-
-// overall the guidance in settings.json is to list every item in "development"
-// section with a default value, and override the value for different environment
-// in later sections
+import { botsFolder, botEndpoint, appDataPath, environment } from './env';
 
 interface Settings {
   botAdminEndpoint: string;
   botEndpoint: string;
   assetsLibray: string;
   runtimeFolder: string;
   botsFolder: string;
+  appDataPath: string;
 }
 
-const defaultSettings = settings.development;
-const environment = process.env.NODE_ENV || 'development';
-const environmentSettings = settings[environment];
+const envSettings: { [env: string]: Settings } = {
+  development: {
+    botAdminEndpoint: botEndpoint,
+    botEndpoint: 'http://localhost:3979', //botEndpoint,
+    assetsLibray: Path.resolve('./assets'),
+    runtimeFolder: Path.resolve('../../../BotProject/Templates'),
+    botsFolder: botsFolder || Path.join(os.homedir(), 'Documents', 'Composer'),
+    appDataPath,
+  },
+};
+
+const defaultSettings = envSettings.development;
+const environmentSettings = envSettings[environment];
 
 const finalSettings = merge<Settings, Settings>(defaultSettings, environmentSettings);
 

Composer/packages/server/src/store/store.ts

@@ -5,15 +5,15 @@ import fs from 'fs';
 import path from 'path';
 
 import log from '../logger';
+import settings from '../settings';
 
 import localInitData from './data.template';
 import abhInitData from './abh-template.json';
 import { runMigrations } from './migrations';
+
 const isHostedInAzure = !!process.env.WEBSITE_NODE_DEFAULT_VERSION;
 const dataStorePath =
-  isHostedInAzure && process.env.HOME
-    ? path.resolve(process.env.HOME, './site/data.json')
-    : path.resolve(__dirname, '../../data.json');
+  isHostedInAzure && process.env.HOME ? path.resolve(process.env.HOME, './site/data.json') : settings.appDataPath;
 
 let initData = isHostedInAzure ? abhInitData : localInitData;
 

Composer/packages/tools/language-servers/language-generation/package.json

@@ -15,12 +15,12 @@
     "lint:typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@types/node": "^12.0.4",
     "botbuilder-lg": "4.7.0-preview.93464",
     "request-light": "^0.2.2",
     "vscode-languageserver": "^5.3.0-next"
   },
   "devDependencies": {
+    "@types/node": "^12.0.4",
     "express": "^4.17.1",
     "jest": "24.0.0",
     "rimraf": "^2.6.3",

docker-compose.override.yml.example

@@ -0,0 +1,12 @@
+# Use this to override Composer's default docker configuration
+
+version: "3"
+services:
+  composer:
+    volumes:
+      - /Some/path/to/my/bots:/Bots
+    environment:
+      DEBUG: composer
+  botruntime:
+    volumes:
+      - /Some/path/to/my/bots:/Bots

docker-compose.yml

@@ -3,22 +3,19 @@ services:
   composer:
     build: Composer
     ports:
-      - "3000:5000"
-      - "5000:5000"
-      # expose 9229 for debugging
-      # - "9229:9229"
+      - "3000:3000"
     volumes:
-      - ./MyBots:/MyBots
+      - ~/Documents/Composer:/Bots
       - ./BotProject:/BotProject
+      - ./.appdata:/appdata
     environment:
-      - NODE_ENV=container
-    # command:
-    #   - node
-    #   - "--inspect-brk=0.0.0.0"
-    #   - "build/server.js"
+      COMPOSER_BOTS_FOLDER: /Bots
+      COMPOSER_APP_DATA: /appdata/data.json
+      BOT_ENDPOINT: http://botruntime:80
+      PORT: 3000
   botruntime:
     build: BotProject/CSharp
     ports:
       - "3979:80"
     volumes:
-      - ./MyBots:/app/MyBots
+      - ~/Documents/Composer:/Bots