From ddb03f0b052a1bb9e4f44c15f53320702b1aee4b Mon Sep 17 00:00:00 2001 From: maslow Date: Fri, 29 Apr 2022 21:31:27 +0800 Subject: [PATCH] feat: impl instance controller sevice; --- docker-compose.yml | 34 ++ packages/instance-controller/src/config.ts | 29 +- packages/instance-controller/src/index.ts | 3 + packages/instance-controller/src/scheduler.ts | 98 ++++-- .../src/support/application.ts | 4 +- .../src/support/instance-docker-driver.ts | 40 ++- .../src/support/instance-kubernetes-driver.ts | 120 ++----- .../src/support/instance-operator.ts | 60 ++-- .../src/handler/application/create.ts | 4 +- .../src/handler/application/index.ts | 2 +- .../application/{service.ts => instance.ts} | 44 ++- .../src/handler/application/remove.ts | 3 +- packages/system-server/src/handler/router.ts | 4 +- packages/system-server/src/index.ts | 52 +-- .../system-server/src/support/application.ts | 29 +- .../system-server/src/support/initializer.ts | 7 +- .../src/support/service-docker-driver.ts | 178 ---------- .../src/support/service-kubernetes-driver.ts | 312 ------------------ .../src/support/service-operator.ts | 101 ------ 19 files changed, 278 insertions(+), 846 deletions(-) rename packages/system-server/src/handler/application/{service.ts => instance.ts} (55%) delete mode 100644 packages/system-server/src/support/service-docker-driver.ts delete mode 100644 packages/system-server/src/support/service-kubernetes-driver.ts delete mode 100644 packages/system-server/src/support/service-operator.ts diff --git a/docker-compose.yml b/docker-compose.yml index 308b627b57..6591083460 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -114,6 +114,40 @@ services: networks: - laf_shared_network + instance-controller: + # image: node:16-alpine + build: ./packages/instance-controller + user: root + working_dir: /app + environment: + SCHEDULER_INTERVAL: 1000 + SYS_DB_URI: mongodb://my_user:password123@mongo:27017/?authSource=laf-sys&replicaSet=laf&writeConcern=majority + APP_DB_URI: mongodb://root:password123@mongo:27017/?authSource=admin&replicaSet=laf&writeConcern=majority + SHARED_NETWORK: laf_shared_network + LOG_LEVEL: debug + SERVICE_DRIVER: docker + APP_SERVICE_DEPLOY_HOST: local-dev.host:8080 # `*.local-dev.host` always resolved to 127.0.0.1, used to local development + APP_SERVICE_DEPLOY_URL_SCHEMA: 'http' + APP_SERVICE_ENV_NPM_INSTALL_FLAGS: '--registry=https://registry.npm.taobao.org --no-audit --no-fund' + MINIO_ACCESS_KEY: minio-root-user + MINIO_ACCESS_SECRET: minio-root-password + MINIO_INTERNAL_ENDPOINT: http://oss:9000 + MINIO_EXTERNAL_ENDPOINT: http://oss.local-dev.host:8080 + MINIO_REGION_NAME: cn-default + DEBUG_BIND_HOST_APP_PATH: '${PWD}/packages/app-service' + SYSTEM_EXTENSION_APPID: '0000000000000000' + command: node ./dist/index.js + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./packages/instance-controller:/app + ports: + - "9000" + depends_on: + - mongo + restart: always + networks: + - laf_shared_network + volumes: db-data: oss-data: diff --git a/packages/instance-controller/src/config.ts b/packages/instance-controller/src/config.ts index 14244c4aa5..1dbaa3e08e 100644 --- a/packages/instance-controller/src/config.ts +++ b/packages/instance-controller/src/config.ts @@ -7,7 +7,7 @@ dotenv.config() export default class Config { /** - * scheduler loop interval + * scheduler loop interval, in ms */ static get SCHEDULER_INTERVAL(): number { const value = process.env.SCHEDULER_INTERVAL || '1000' @@ -84,10 +84,6 @@ export default class Config { return process.env.KUBE_NAMESPACE_OF_APP_SERVICES || 'laf' } - static get KUBE_NAMESPACE_OF_SYS_SERVICES() { - return process.env.KUBE_NAMESPACE_OF_SYS_SERVICES || 'laf' - } - static get APP_SERVICE_ENV_NPM_INSTALL_FLAGS(): string { return process.env.APP_SERVICE_ENV_NPM_INSTALL_FLAGS || '' } @@ -96,29 +92,6 @@ export default class Config { return process.env.SYSTEM_EXTENSION_APPID || '0000000000000000' } - /** - * The host to access the app service - * For example, if set this to `lafyun.com`, then you can access app service by format `[appid].lafyun.com`: - * - 7b0b318c-b96c-4cc5-b521-33d11bd16cde.lafyun.com - * - http://7b0b318c-b96c-4cc5-b521-33d11bd16cde.lafyun.com/file/public/33d11bd16cde.png - * - http://7b0b318c-b96c-4cc5-b521-33d11bd16cde.lafyun.com/FUNC_NAME - * - * You should resolve `*.lafyun.com` to your laf server ip, to support `[appid].lafyun.com` url. - * You can also provide the PORT, like `lafyun.com:8080`. - */ - static get APP_SERVICE_DEPLOY_HOST(): string { - return process.env.APP_SERVICE_DEPLOY_HOST ?? '' - } - - /** - * The schema of app deployed url: `http` | `https`. - * Default value is `http`. - */ - static get APP_SERVICE_DEPLOY_URL_SCHEMA(): string { - return process.env.APP_SERVICE_DEPLOY_URL_SCHEMA ?? 'http' - } - - /** * Minio configuration */ diff --git a/packages/instance-controller/src/index.ts b/packages/instance-controller/src/index.ts index 28d42e5414..3805edb016 100644 --- a/packages/instance-controller/src/index.ts +++ b/packages/instance-controller/src/index.ts @@ -4,6 +4,7 @@ import * as express from 'express' import Config from './config' import { logger } from './support/logger' import { DatabaseAgent } from './support/db' +import { start_schedular } from './scheduler' DatabaseAgent.init(Config.SYS_DB_URI) @@ -16,6 +17,8 @@ app.get('/healthz', (_req, res) => { return res.status(200).send('ok') }) +start_schedular() + const server = app.listen(Config.PORT, () => logger.info(`listened on ${Config.PORT}`)) process.on('unhandledRejection', (reason, promise) => { diff --git a/packages/instance-controller/src/scheduler.ts b/packages/instance-controller/src/scheduler.ts index 1e24bdb15c..688d3aa9da 100644 --- a/packages/instance-controller/src/scheduler.ts +++ b/packages/instance-controller/src/scheduler.ts @@ -1,66 +1,118 @@ import Config from "./config" import { getApplicationsInStatus, InstanceStatus, updateApplicationStatus } from "./support/application" -import { ApplicationInstanceOperator } from "./support/instance-operator" +import { InstanceOperator } from "./support/instance-operator" import { logger } from "./support/logger" -export async function start_schedular() { +export function start_schedular() { + logger.info('start schedular loop') setInterval(loop, Config.SCHEDULER_INTERVAL) } + function loop() { const tick = new Date() - logger.info('enter loop ' + tick) - handle_prepared_start(tick) handle_starting(tick) handle_prepared_stop(tick) handle_prepared_stop(tick) handle_stopping(tick) handle_prepared_restart(tick) - handle_restarting_stopping(tick) + handle_restarting(tick) } -async function handle_prepared_start(tick: Date) { - logger.info('processing `prepared_start` status with tick: ', tick) +async function handle_prepared_start(tick: any) { const apps = await getApplicationsInStatus(InstanceStatus.PREPARED_START) for (let app of apps) { try { - const res = await ApplicationInstanceOperator.start(app) - if (res) await updateApplicationStatus(app.appid, app.status, InstanceStatus.STARTING) + const res = await InstanceOperator.start(app) + if (res) { + logger.info(tick, `update ${app.appid} status from 'PREPARED_START' to 'STARTING'`) + await updateApplicationStatus(app.appid, app.status, InstanceStatus.STARTING) + } } catch (error) { - logger.error(`handle_prepared_start got error for app ${app.appid} with tick: ${tick}`) + logger.error(tick, `handle_prepared_start(${app.appid}) error: `, error) } } } -async function handle_starting(tick: Date) { - logger.info('processing `starting` status with tick: ', tick) +async function handle_starting(tick: any) { const apps = await getApplicationsInStatus(InstanceStatus.STARTING) for (let app of apps) { try { - // const res = await ApplicationInstanceOperator.start(app) - // if (res) await updateApplicationStatus(app.appid, app.status, InstanceStatus.STARTING) + const status = await InstanceOperator.status(app) + if (status === InstanceStatus.RUNNING) { + logger.info(tick, `update ${app.appid} status from 'STARTING' to 'RUNNING'`) + await updateApplicationStatus(app.appid, app.status, InstanceStatus.RUNNING) + } } catch (error) { - logger.error(`handle_starting got error for app ${app.appid} with tick: ${tick}`) + logger.error(tick, `handle_starting(${app.appid}) error: `, error) } } } -async function handle_prepared_stop(tick: Date) { - logger.info('processing `prepared_stop` status with tick: ', tick) +async function handle_prepared_stop(tick: any) { + const apps = await getApplicationsInStatus(InstanceStatus.PREPARED_STOP) + for (let app of apps) { + try { + const res = await InstanceOperator.stop(app) + if (res) { + logger.info(tick, `update ${app.appid} status from 'PREPARED_STOP' to 'STOPPING'`) + await updateApplicationStatus(app.appid, app.status, InstanceStatus.STOPPING) + } + } catch (error) { + logger.error(tick, `handle_prepared_stop(${app.appid}) error: `, error) + } + } } -async function handle_stopping(tick: Date) { - logger.info('processing `stopping` status with tick: ', tick) +async function handle_stopping(tick: any) { + const apps = await getApplicationsInStatus(InstanceStatus.STOPPING) + for (let app of apps) { + try { + const status = await InstanceOperator.status(app) + if (status === InstanceStatus.STOPPED) { + logger.info(tick, `update ${app.appid} status from 'STOPPING' to 'STOPPED'`) + await updateApplicationStatus(app.appid, app.status, InstanceStatus.STOPPED) + } + } catch (error) { + logger.error(tick, `handle_stopping(${app.appid}) error: `, error) + } + } } -async function handle_prepared_restart(tick: Date) { - logger.info('processing `prepared_restart` status with tick: ', tick) +async function handle_prepared_restart(tick: any) { + const apps = await getApplicationsInStatus(InstanceStatus.PREPARED_RESTART) + for (let app of apps) { + try { + const res = await InstanceOperator.stop(app) + if (res) { + logger.info(tick, `update ${app.appid} status from 'PREPARED_RESTART' to 'RESTARTING'`) + await updateApplicationStatus(app.appid, app.status, InstanceStatus.RESTARTING) + } + } catch (error) { + logger.error(tick, `handle_prepared_restart(${app.appid}) error: `, error) + } + } } -async function handle_restarting_stopping(tick: Date) { - logger.info('processing `restarting:stopping` status with tick: ', tick) +async function handle_restarting(tick: any) { + const apps = await getApplicationsInStatus(InstanceStatus.RESTARTING) + for (let app of apps) { + try { + const status = await InstanceOperator.status(app) + if (status !== InstanceStatus.STOPPED) continue + + logger.info(tick, `start stopped ${app.appid} for restarting`) + const res = await InstanceOperator.start(app) + if (res) { + logger.info(tick, `update ${app.appid} status from 'RESTARTING' to 'STARTING'`) + await updateApplicationStatus(app.appid, app.status, InstanceStatus.STARTING) + } + } catch (error) { + logger.error(tick, `handle_stopping(${app.appid}) error: `, error) + } + } } \ No newline at end of file diff --git a/packages/instance-controller/src/support/application.ts b/packages/instance-controller/src/support/application.ts index 397bdd6069..3c8515eff1 100644 --- a/packages/instance-controller/src/support/application.ts +++ b/packages/instance-controller/src/support/application.ts @@ -141,7 +141,9 @@ export async function updateApplicationStatus(appid: string, from: InstanceStatu appid: appid, status: from, }, { - status: to + $set: { + status: to + } }) return r.modifiedCount diff --git a/packages/instance-controller/src/support/instance-docker-driver.ts b/packages/instance-controller/src/support/instance-docker-driver.ts index f028cb730b..f50c288d98 100644 --- a/packages/instance-controller/src/support/instance-docker-driver.ts +++ b/packages/instance-controller/src/support/instance-docker-driver.ts @@ -1,5 +1,5 @@ import * as Docker from 'dockerode' -import { IApplicationData, getApplicationDbUri } from './application' +import { IApplicationData, getApplicationDbUri, InstanceStatus } from './application' import Config from '../config' import { logger } from './logger' import { InstanceDriverInterface } from './instance-operator' @@ -19,7 +19,7 @@ export class DockerContainerDriver implements InstanceDriverInterface { * Get name of service * @param app */ - getName(app: IApplicationData): string { + public getName(app: IApplicationData): string { return `app-${app.appid}` } @@ -28,31 +28,31 @@ export class DockerContainerDriver implements InstanceDriverInterface { * @param app * @returns the container id */ - async startService(app: IApplicationData) { + public async create(app: IApplicationData) { let container = this.getContainer(app) - const info = await this.info(app) + const info = await this.inspect(app) if (!info) { container = await this.createService(app) } - if (info?.State?.Running || info?.State?.Restarting) { - return container.id + if (info?.State?.Running) { + return true } await container.start() logger.debug(`start container ${container.id} of app ${app.appid}`) - return container.id + return true } /** * Remove application service * @param app */ - async removeService(app: IApplicationData) { - const info = await this.info(app) + public async remove(app: IApplicationData) { + const info = await this.inspect(app) if (!info) { - return + return true } const container = this.getContainer(app) @@ -67,15 +67,14 @@ export class DockerContainerDriver implements InstanceDriverInterface { await container.remove() logger.debug(`stop & remove container ${container.id} of app ${app.appid}`) - return container.id + return true } /** * Get container info - * @param container * @returns return null if container not exists */ - async info(app: IApplicationData): Promise { + public async inspect(app: IApplicationData): Promise { try { const container = this.getContainer(app) const info = await container.inspect() @@ -88,6 +87,21 @@ export class DockerContainerDriver implements InstanceDriverInterface { } } + /** + * Get instance status + * @param app + * @returns + */ + public async status(app: IApplicationData): Promise { + const res = await this.inspect(app) + if (!res) return InstanceStatus.STOPPED + const state = res?.State + if (state.Running) + return InstanceStatus.RUNNING + + return InstanceStatus.STOPPING + } + /** * Create application service * @param app diff --git a/packages/instance-controller/src/support/instance-kubernetes-driver.ts b/packages/instance-controller/src/support/instance-kubernetes-driver.ts index 4c0ecd3225..e4b59db85e 100644 --- a/packages/instance-controller/src/support/instance-kubernetes-driver.ts +++ b/packages/instance-controller/src/support/instance-kubernetes-driver.ts @@ -1,5 +1,5 @@ import * as k8s from '@kubernetes/client-node' -import { IApplicationData, getApplicationDbUri } from './application' +import { IApplicationData, getApplicationDbUri, InstanceStatus } from './application' import Config from '../config' import { MB } from './constants' import { logger } from './logger' @@ -25,7 +25,7 @@ export class KubernetesDriver implements InstanceDriverInterface { * Get name of service * @param app */ - getName(app: IApplicationData): string { + public getName(app: IApplicationData): string { return `app-${app.appid}` } @@ -34,10 +34,10 @@ export class KubernetesDriver implements InstanceDriverInterface { * @param app * @returns the container id */ - async startService(app: IApplicationData) { + public async create(app: IApplicationData) { const labels = { appid: app.appid, type: 'laf-app' } - const info = await this.info(app) + const info = await this.inspect(app) let deployment = info?.deployment if (!deployment) { deployment = await this.createK8sDeployment(app, labels) @@ -47,22 +47,15 @@ export class KubernetesDriver implements InstanceDriverInterface { await this.createK8sService(app, labels) } - if (!info?.ingress) { - try { - await this.createK8sIngress(app, labels) - } catch (error) { - console.log(error) - } - } - return deployment + return true } /** * Remove application service * @param app */ - async removeService(app: IApplicationData) { - const info = await this.info(app) + public async remove(app: IApplicationData) { + const info = await this.inspect(app) if (!info) return if (info?.deployment) { @@ -77,30 +70,35 @@ export class KubernetesDriver implements InstanceDriverInterface { logger.debug(`removed k8s service of app ${app.appid}:`, res_svc.body) } - if (info?.ingress) { - const res_ingr = await this.net_api.deleteNamespacedIngress(this.getName(app), this.namespace) - logger.info(`remove k8s ingress of app ${app.appid}`) - logger.debug(`removed k8s ingress of app ${app.appid}:`, res_ingr.body) - } - - return app.appid + return true } /** - * Get container info - * @param container + * Get instance info * @returns return null if container not exists */ - async info(app: IApplicationData) { - try { - const name = this.getName(app) - const deployment = await this.getK8sDeployment(name) - const service = await this.getK8sService(name) - const ingress = await this.getK8sIngress(name) - return { deployment, service, ingress } - } catch (error) { - throw error - } + public async inspect(app: IApplicationData) { + const name = this.getName(app) + const deployment = await this.getK8sDeployment(name) + const service = await this.getK8sService(name) + return { deployment, service } + } + + /** + * Get instance status + * @param app + * @returns + */ + public async status(app: IApplicationData) { + const res = await this.inspect(app) + const deployment = res?.deployment + if (!deployment) return InstanceStatus.STOPPED + + const state = deployment.status + if (state.readyReplicas > 0) + return InstanceStatus.RUNNING + + return InstanceStatus.STOPPING } /** @@ -236,53 +234,6 @@ export class KubernetesDriver implements InstanceDriverInterface { return service } - /** - * create k8s ingress for app - * @param app - * @param labels - * @returns - */ - private async createK8sIngress(app: IApplicationData, labels: any) { - const { body: ingress } = await this.net_api.createNamespacedIngress(this.namespace, { - metadata: { - name: this.getName(app), - labels: labels, - annotations: { - 'nginx.ingress.kubernetes.io/enable-cors': "true", - 'nginx.ingress.kubernetes.io/cors-expose-headers': "*", - 'nginx.ingress.kubernetes.io/configuration-snippet': `more_set_headers "request-id: $req_id";` - } - }, - spec: { - ingressClassName: "nginx", - rules: [ - { - host: `${app.appid}.${Config.APP_SERVICE_DEPLOY_HOST}`, - http: { - paths: [ - { - path: '/', - pathType: 'Prefix', - backend: { - service: { - name: this.getName(app), - port: { number: 8000 } - } - } - } - ] - } - } - ] - } - }) - - logger.info(`create k8s ingress ${ingress.metadata.name} of app ${app.appid}`) - logger.debug(`created k8s ingress of app ${app.appid}:`, ingress) - - return ingress - } - private async getK8sDeployment(name: string) { try { const res = await this.apps_api.readNamespacedDeployment(name, this.namespace) @@ -300,13 +251,4 @@ export class KubernetesDriver implements InstanceDriverInterface { return null } } - - private async getK8sIngress(name: string) { - try { - const res = await this.net_api.readNamespacedIngress(name, this.namespace) - return res.body - } catch (error) { - return null - } - } } \ No newline at end of file diff --git a/packages/instance-controller/src/support/instance-operator.ts b/packages/instance-controller/src/support/instance-operator.ts index 0351c7b917..5bbd185d69 100644 --- a/packages/instance-controller/src/support/instance-operator.ts +++ b/packages/instance-controller/src/support/instance-operator.ts @@ -1,22 +1,20 @@ -import { IApplicationData } from "./application" -import { logger } from "./logger" +import { IApplicationData, InstanceStatus } from "./application" import Config from "../config" import { KubernetesDriver } from "./instance-kubernetes-driver" import { DockerContainerDriver } from "./instance-docker-driver" - -export class ApplicationInstanceOperator { +/** + * Application instance operator + */ +export class InstanceOperator { /** * start app service * @param app * @returns */ - static async start(app: IApplicationData) { - const driver = this.create() - const res = await driver.startService(app) - - return res + public static async start(app: IApplicationData) { + return await this.driver().create(app) } /** @@ -24,63 +22,51 @@ export class ApplicationInstanceOperator { * @param app * @returns */ - static async stop(app: IApplicationData) { - const driver = this.create() - const res = await driver.removeService(app) - return res + public static async stop(app: IApplicationData) { + return await this.driver().remove(app) } - /** - * restart app service - * @param app - * @returns - */ - static async restart(app: IApplicationData) { - await this.stop(app) - return await this.start(app) + public static async status(app: IApplicationData) { + return await this.driver().status(app) } - static create(): InstanceDriverInterface { + private static driver(): InstanceDriverInterface { const driver = Config.SERVICE_DRIVER || 'docker' - logger.info("creating ServiceDriver with driver: " + driver) if (driver === 'kubernetes') { return new KubernetesDriver() } else { return new DockerContainerDriver() } } - - /** - * @todo - */ - static status() { - // to be done - } } export interface InstanceDriverInterface { /** - * Start application service + * Start application instance * @param app * @returns */ - startService(app: IApplicationData): Promise + create(app: IApplicationData): Promise /** - * Remove application service + * Remove application instance * @param app */ - removeService(app: IApplicationData): Promise + remove(app: IApplicationData): Promise /** - * Get service info - * @param container + * Get instance information * @returns return null if container not exists */ - info(app: IApplicationData): Promise + inspect(app: IApplicationData): Promise + /** + * Get instance status + * @param app + */ + status(app: IApplicationData): Promise /** * Get name of service diff --git a/packages/system-server/src/handler/application/create.ts b/packages/system-server/src/handler/application/create.ts index 7553442ef9..8d625af48c 100644 --- a/packages/system-server/src/handler/application/create.ts +++ b/packages/system-server/src/handler/application/create.ts @@ -9,7 +9,7 @@ import * as assert from 'assert' import { Request, Response } from 'express' import { ObjectId } from 'mongodb' import { getAccountById } from '../../support/account' -import { IApplicationData, createApplicationDb, generateAppid, getApplicationByAppid, getMyApplications } from '../../support/application' +import { IApplicationData, createApplicationDb, generateAppid, getApplicationByAppid, getMyApplications, InstanceStatus } from '../../support/application' import { MinioAgent } from '../../support/minio' import Config from '../../config' import { CN_APPLICATIONS, DATE_NEVER } from '../../constants' @@ -63,7 +63,7 @@ export async function handleCreateApplication(req: Request, res: Response) { name: app_name, created_by: new ObjectId(uid), appid: appid, - status: 'created', + status: InstanceStatus.CREATED, collaborators: [], config: { db_name: db_name, diff --git a/packages/system-server/src/handler/application/index.ts b/packages/system-server/src/handler/application/index.ts index 38e5d0148f..8517a8ed22 100644 --- a/packages/system-server/src/handler/application/index.ts +++ b/packages/system-server/src/handler/application/index.ts @@ -17,7 +17,7 @@ import { handleGetSpecs } from './get-specs' import { handleImportApplication, handleInitApplicationWithTemplate } from './importer' import { handleAddPackage, handleGetPackages, handleRemovePackage, handleUpdatePackage } from './packages' import { handleRemoveApplication } from './remove' -import { handleStopApplicationService, handleStartApplicationService } from './service' +import { handleStopApplicationService, handleStartApplicationService } from './instance' import { handleUpdateApplication } from './update' /** diff --git a/packages/system-server/src/handler/application/service.ts b/packages/system-server/src/handler/application/instance.ts similarity index 55% rename from packages/system-server/src/handler/application/service.ts rename to packages/system-server/src/handler/application/instance.ts index 40f093b019..16ae6fcdc2 100644 --- a/packages/system-server/src/handler/application/service.ts +++ b/packages/system-server/src/handler/application/instance.ts @@ -6,12 +6,12 @@ */ import { Request, Response } from 'express' -import { getApplicationByAppid } from '../../support/application' +import { getApplicationByAppid, InstanceStatus, updateApplicationStatus } from '../../support/application' import { checkPermission } from '../../support/permission' import { permissions } from '../../permissions' -import { ApplicationServiceOperator } from '../../support/service-operator' const { APPLICATION_UPDATE } = permissions + /** * The handler of starting application */ @@ -25,15 +25,13 @@ export async function handleStartApplicationService(req: Request, res: Response) // check permission const code = await checkPermission(uid, APPLICATION_UPDATE.name, app) - if (code) { + if (code) return res.status(code).send() - } - - const container_id = await ApplicationServiceOperator.start(app) + const ret = await updateApplicationStatus(app.appid, app.status, InstanceStatus.PREPARED_START) return res.send({ data: { - service_id: container_id, + result: ret > 0 ? true : false, appid: app.appid } }) @@ -47,21 +45,43 @@ export async function handleStopApplicationService(req: Request, res: Response) const uid = req['auth']?.uid const appid = req.params.appid const app = await getApplicationByAppid(appid) - if (!app) return res.status(422).send('app not found') // check permission const code = await checkPermission(uid, APPLICATION_UPDATE.name, app) - if (code) { + if (code) return res.status(code).send() - } - const container_id = await ApplicationServiceOperator.stop(app) + const ret = await updateApplicationStatus(app.appid, app.status, InstanceStatus.PREPARED_STOP) + return res.send({ + data: { + result: ret > 0 ? true : false, + appid: app.appid + } + }) +} + +/** + * The handler of restarting application + */ +export async function handleRestartApplicationService(req: Request, res: Response) { + const uid = req['auth']?.uid + const appid = req.params.appid + const app = await getApplicationByAppid(appid) + + if (!app) + return res.status(422).send('app not found') + + // check permission + const code = await checkPermission(uid, APPLICATION_UPDATE.name, app) + if (code) + return res.status(code).send() + const ret = await updateApplicationStatus(app.appid, app.status, InstanceStatus.PREPARED_RESTART) return res.send({ data: { - service_id: container_id, + result: ret > 0 ? true : false, appid: app.appid } }) diff --git a/packages/system-server/src/handler/application/remove.ts b/packages/system-server/src/handler/application/remove.ts index 26cf4567c2..608f04e381 100644 --- a/packages/system-server/src/handler/application/remove.ts +++ b/packages/system-server/src/handler/application/remove.ts @@ -12,7 +12,6 @@ import { getApplicationByAppid } from '../../support/application' import { checkPermission } from '../../support/permission' import { CN_APPLICATIONS, CONST_DICTS } from '../../constants' import { DatabaseAgent } from '../../db' -import { ApplicationServiceOperator } from '../../support/service-operator' const { APPLICATION_REMOVE } = CONST_DICTS.permissions @@ -46,7 +45,7 @@ export async function handleRemoveApplication(req: Request, res: Response) { } if (app.status !== 'stopped') { - await ApplicationServiceOperator.stop(app) + return res.status(400).send('you should stopped application instance before removing') } // save app to recycle collection diff --git a/packages/system-server/src/handler/router.ts b/packages/system-server/src/handler/router.ts index 835ee28cb4..474899aa43 100644 --- a/packages/system-server/src/handler/router.ts +++ b/packages/system-server/src/handler/router.ts @@ -28,9 +28,9 @@ router.use('/apps/:appid/deploy', checkAppid, DeployRouter) router.use('/apps/:appid/oss', checkAppid, OSSRouter) router.use('/healthz', (_req, res) => { - if (!DatabaseAgent.sys_accessor.db) { + if (!DatabaseAgent.sys_accessor.db) return res.status(400).send('no db connection') - } + return res.status(200).send('ok') }) diff --git a/packages/system-server/src/index.ts b/packages/system-server/src/index.ts index e44a0b807e..8d346a9473 100644 --- a/packages/system-server/src/index.ts +++ b/packages/system-server/src/index.ts @@ -7,67 +7,45 @@ import * as express from 'express' import { parseToken, splitBearerToken } from './support/token' -import { v4 as uuidv4 } from 'uuid' import Config from './config' import { router } from './handler/router' import { logger } from './support/logger' import { DatabaseAgent } from './db' -import { ApplicationServiceOperator } from './support/service-operator' - -const app = express() -app.use(express.json({ - limit: '10000kb' -}) as any) +import { generateUUID } from './support/util-passwd' +process.on('unhandledRejection', (reason, promise) => { logger.error(`Caught unhandledRejection:`, reason, promise) }) +process.on('uncaughtException', err => { logger.error(`Caught uncaughtException:`, err) }) +process.on('SIGTERM', gracefullyExit) +process.on('SIGINT', gracefullyExit) -app.all('*', function (_req, res, next) { - res.header('X-Powered-By', 'LaF Server') - next() -}) +const app = express() +app.use(express.json({ limit: '10000kb' }) as any) /** * Parsing bearer token */ -app.use(function (req, _res, next) { +app.use(function (req, res, next) { const token = splitBearerToken(req.headers['authorization'] ?? '') const auth = parseToken(token) || null req['auth'] = auth - const requestId = req['requestId'] = uuidv4() - logger.info(requestId, `${req.method} "${req.url}" - referer: ${req.get('referer') || '-'} ${req.get('user-agent')}`) - logger.trace(requestId, `${req.method} ${req.url}`, { body: req.body, headers: req.headers, auth }) + const requestId = req['requestId'] = req.headers['x-request-id'] || generateUUID() + if (req.url !== '/healthz') { + logger.info(requestId, `${req.method} "${req.url}" - referer: ${req.get('referer') || '-'} ${req.get('user-agent')}`) + logger.trace(requestId, `${req.method} ${req.url}`, { body: req.body, headers: req.headers, auth }) + } + + res.set('request-id', requestId) next() }) app.use(router) - const server = app.listen(Config.PORT, () => logger.info(`listened on ${Config.PORT}`)) -process.on('unhandledRejection', (reason, promise) => { - logger.error(`Caught unhandledRejection:`, reason, promise) -}) - -process.on('uncaughtException', err => { - logger.error(`Caught uncaughtException:`, err) -}) - - -process.on('SIGTERM', gracefullyExit) -process.on('SIGINT', gracefullyExit) - async function gracefullyExit() { - - // NOT remove system extension app service if service driver is 'kubernetes', - if (Config.SERVICE_DRIVER === 'docker') { - logger.info('exiting: removing system extension service') - await ApplicationServiceOperator.create().removeService({ appid: Config.SYSTEM_EXTENSION_APPID } as any) - logger.info('exiting: system extension service has been removed') - } - logger.info('exiting: closing db connection') await DatabaseAgent.sys_accessor.close() - logger.info('exiting: db connection has been closed') server.close(async () => { logger.info('process gracefully exited!') diff --git a/packages/system-server/src/support/application.ts b/packages/system-server/src/support/application.ts index 6b62b88d66..abf44656aa 100644 --- a/packages/system-server/src/support/application.ts +++ b/packages/system-server/src/support/application.ts @@ -16,11 +16,10 @@ import { logger } from "./logger" import { BUCKET_ACL } from "./minio" import { customAlphabet } from 'nanoid' - /** * Status of application instance */ -export enum ApplicationInstanceStatus { +export enum InstanceStatus { CREATED = 'created', PREPARED_START = 'prepared_start', STARTING = 'starting', @@ -36,7 +35,7 @@ export interface IApplicationBucket { name: string, mode: BUCKET_ACL, quota: number, - options: { + options?: { index: string | null domain: string } @@ -50,7 +49,7 @@ export interface IApplicationData { name: string created_by: ObjectId appid: string - status: ApplicationInstanceStatus + status: InstanceStatus config: { db_server_name?: string db_name: string @@ -94,6 +93,28 @@ export async function getApplicationByAppid(appid: string) { return doc } +/** + * Update application status + * @param appid + * @param from original status + * @param to target status to update + * @returns + */ + export async function updateApplicationStatus(appid: string, from: InstanceStatus, to: InstanceStatus) { + const db = DatabaseAgent.db + const r = await db.collection(CN_APPLICATIONS) + .updateOne({ + appid: appid, + status: from, + }, { + $set: { + status: to + } + }) + + return r.modifiedCount +} + /** * Get application created by account_id * @param account_id diff --git a/packages/system-server/src/support/initializer.ts b/packages/system-server/src/support/initializer.ts index 954d50a184..be641d0707 100644 --- a/packages/system-server/src/support/initializer.ts +++ b/packages/system-server/src/support/initializer.ts @@ -3,8 +3,7 @@ import Config from "../config" import { CN_ACCOUNTS, CN_APPLICATIONS, CN_FUNCTIONS, CN_POLICIES, CN_SPECS, DATE_NEVER, GB, MB } from "../constants" import { DatabaseAgent } from "../db" import { hashPassword, generatePassword } from "./util-passwd" -import { IApplicationData, getApplicationByAppid } from "./application" -import { ApplicationServiceOperator } from "./service-operator" +import { IApplicationData, getApplicationByAppid, InstanceStatus, updateApplicationStatus } from "./application" import * as path from 'path' import { MinioAgent } from "./minio" import { ApplicationSpecSupport } from "./application-spec" @@ -99,7 +98,7 @@ export class Initializer { name: SYSTEM_APP_NAME, created_by: account_id, appid: appid, - status: 'created', + status: InstanceStatus.CREATED, collaborators: [], config: { db_name: db_config.database, @@ -148,7 +147,7 @@ export class Initializer { */ static async startSystemExtensionApp(appid: string) { const app = await getApplicationByAppid(appid) - await ApplicationServiceOperator.start(app) + await updateApplicationStatus(appid, app.status, InstanceStatus.PREPARED_START) } /** diff --git a/packages/system-server/src/support/service-docker-driver.ts b/packages/system-server/src/support/service-docker-driver.ts deleted file mode 100644 index 799c88f697..0000000000 --- a/packages/system-server/src/support/service-docker-driver.ts +++ /dev/null @@ -1,178 +0,0 @@ -import * as Docker from 'dockerode' -import { IApplicationData, getApplicationDbUri } from './application' -import Config from '../config' -import { logger } from './logger' -import { ServiceDriverInterface } from './service-operator' -import { ApplicationSpecSupport } from './application-spec' -import * as assert from 'assert' -import { MB } from '../constants' - - -export class DockerContainerServiceDriver implements ServiceDriverInterface { - docker: Docker - - constructor(options?: Docker.DockerOptions) { - this.docker = new Docker(options) - } - - /** - * Get name of service - * @param app - */ - getName(app: IApplicationData): string { - return `app-${app.appid}` - } - - /** - * Start application service - * @param app - * @returns the container id - */ - async startService(app: IApplicationData) { - let container = this.getContainer(app) - const info = await this.info(app) - if (!info) { - container = await this.createService(app) - } - - if (info?.State?.Running || info?.State?.Restarting) { - return container.id - } - - await container.start() - logger.debug(`start container ${container.id} of app ${app.appid}`) - - return container.id - } - - /** - * Remove application service - * @param app - */ - async removeService(app: IApplicationData) { - const info = await this.info(app) - if (!info) { - return - } - - const container = this.getContainer(app) - if (info.State.Running) { - await container.stop() - } - - if (info.State.Restarting) { - await container.stop() - } - - await container.remove() - logger.debug(`stop & remove container ${container.id} of app ${app.appid}`) - - return container.id - } - - /** - * Get container info - * @param container - * @returns return null if container not exists - */ - async info(app: IApplicationData): Promise { - try { - const container = this.getContainer(app) - const info = await container.inspect() - return info - } catch (error) { - if (error.reason === 'no such container') { - return null - } - throw error - } - } - - /** - * Create application service - * @param app - * @returns - */ - private async createService(app: IApplicationData) { - const uri = getApplicationDbUri(app) - const app_spec = await ApplicationSpecSupport.getValidAppSpec(app.appid) - assert.ok(app_spec, `no spec avaliable with app: ${app.appid}`) - - const limit_memory = app_spec.spec.limit_memory - const max_old_space_size = ~~(limit_memory / MB * 0.8) - const limit_cpu = app_spec.spec.limit_cpu - const image_name = app.runtime?.image ?? Config.APP_SERVICE_IMAGE - const log_level = Config.LOG_LEVEL - const npm_install_flags = Config.APP_SERVICE_ENV_NPM_INSTALL_FLAGS - - let binds = [] - // just for debug purpose in local development mode - if (Config.DEBUG_BIND_HOST_APP_PATH) { - binds = [`${Config.DEBUG_BIND_HOST_APP_PATH}:/app`] - } - - if (app.appid === Config.SYSTEM_EXTENSION_APPID) { - binds.push('/var/run/docker.sock:/var/run/docker.sock:ro') - } - - const container = await this.docker.createContainer({ - Image: image_name, - Cmd: ['sh', '/app/start.sh'], - name: this.getName(app), - Env: [ - `DB=${app.config.db_name}`, - `DB_URI=${uri}`, - `LOG_LEVEL=${log_level}`, - `ENABLE_CLOUD_FUNCTION_LOG=always`, - `SERVER_SECRET_SALT=${app.config.server_secret_salt}`, - `OSS_ACCESS_SECRET=${app.config.oss_access_secret}`, - `OSS_INTERNAL_ENDPOINT=${Config.MINIO_CONFIG.endpoint.internal}`, - `OSS_EXTERNAL_ENDPOINT=${Config.MINIO_CONFIG.endpoint.external}`, - `OSS_REGION=${Config.MINIO_CONFIG.region}`, - `FLAGS=--max_old_space_size=${max_old_space_size}`, - `APP_ID=${app.appid}`, - `RUNTIME_IMAGE=${image_name}`, - `NPM_INSTALL_FLAGS=${npm_install_flags}` - ], - ExposedPorts: { - "8000/tcp": {} - }, - HostConfig: { - Memory: limit_memory, - CpuShares: limit_cpu, - Binds: binds - }, - }) - - logger.debug(`create container ${container.id} of app ${app.appid}`) - - // add the the app network - const net = await this.getSharedNetwork() - await net.connect({ Container: container.id }) - - return container - } - - private getContainer(app: IApplicationData) { - const name = this.getName(app) - return this.docker.getContainer(name) - } - - /** - * Get or create the shared network - * @returns - */ - private async getSharedNetwork() { - const name = Config.DOCKER_SHARED_NETWORK - let net = this.docker.getNetwork(name) - const info = await net.inspect() - if (!info) { - net = await this.docker.createNetwork({ - Name: name, - Driver: 'bridge' - }) - } - - return net - } -} \ No newline at end of file diff --git a/packages/system-server/src/support/service-kubernetes-driver.ts b/packages/system-server/src/support/service-kubernetes-driver.ts deleted file mode 100644 index 74d77d3154..0000000000 --- a/packages/system-server/src/support/service-kubernetes-driver.ts +++ /dev/null @@ -1,312 +0,0 @@ -import * as k8s from '@kubernetes/client-node' -import { IApplicationData, getApplicationDbUri } from './application' -import Config from '../config' -import { MB } from '../constants' -import { logger } from './logger' -import { ServiceDriverInterface } from './service-operator' -import { ApplicationSpecSupport } from './application-spec' -import * as assert from 'assert' - -export class KubernetesServiceDriver implements ServiceDriverInterface { - namespace = Config.KUBE_NAMESPACE_OF_APP_SERVICES - apps_api: k8s.AppsV1Api - core_api: k8s.CoreV1Api - net_api: k8s.NetworkingV1Api - - constructor(options?: any) { - const kc = new k8s.KubeConfig() - kc.loadFromDefault(options) - this.apps_api = kc.makeApiClient(k8s.AppsV1Api) - this.core_api = kc.makeApiClient(k8s.CoreV1Api) - this.net_api = kc.makeApiClient(k8s.NetworkingV1Api) - } - - /** - * Get name of service - * @param app - */ - getName(app: IApplicationData): string { - return `app-${app.appid}` - } - - /** - * Start application service - * @param app - * @returns the container id - */ - async startService(app: IApplicationData) { - const labels = { appid: app.appid, type: 'laf-app' } - - const info = await this.info(app) - let deployment = info?.deployment - if (!deployment) { - deployment = await this.createK8sDeployment(app, labels) - } - - if (!info?.service) { - await this.createK8sService(app, labels) - } - - if (!info?.ingress) { - try { - await this.createK8sIngress(app, labels) - } catch (error) { - console.log(error) - } - } - - return deployment - } - - /** - * Remove application service - * @param app - */ - async removeService(app: IApplicationData) { - const info = await this.info(app) - if (!info) return - - if (info?.deployment) { - const res_deploy = await this.apps_api.deleteNamespacedDeployment(this.getName(app), this.namespace) - logger.info(`remove k8s deployment of app ${app.appid}`) - logger.debug(`removed k8s deployment of app ${app.appid}:`, res_deploy.body) - } - - if (info?.service) { - const res_svc = await this.core_api.deleteNamespacedService(this.getName(app), this.namespace) - logger.info(`remove k8s service of app ${app.appid}`) - logger.debug(`removed k8s service of app ${app.appid}:`, res_svc.body) - } - - if (info?.ingress) { - const res_ingr = await this.net_api.deleteNamespacedIngress(this.getName(app), this.namespace) - logger.info(`remove k8s ingress of app ${app.appid}`) - logger.debug(`removed k8s ingress of app ${app.appid}:`, res_ingr.body) - } - - return app.appid - } - - /** - * Get container info - * @param container - * @returns return null if container not exists - */ - async info(app: IApplicationData) { - try { - const name = this.getName(app) - const deployment = await this.getK8sDeployment(name) - const service = await this.getK8sService(name) - const ingress = await this.getK8sIngress(name) - return { deployment, service, ingress } - } catch (error) { - throw error - } - } - - /** - * Create k8s deployment for app - * @param app - * @returns - */ - private async createK8sDeployment(app: IApplicationData, labels: any) { - const uri = getApplicationDbUri(app) - - const app_spec = await ApplicationSpecSupport.getValidAppSpec(app.appid) - assert.ok(app_spec, `no spec avaliable with app: ${app.appid}`) - - const limit_memory = ~~(app_spec.spec.limit_memory / MB) - const limit_cpu = app_spec.spec.limit_cpu - - const req_cpu = app_spec.spec.request_cpu - const req_memory = ~~(app_spec.spec.limit_memory / MB) - - const image_name = app.runtime?.image ?? Config.APP_SERVICE_IMAGE - const max_old_space_size = ~~(limit_memory * 0.8) - const log_level = Config.LOG_LEVEL - const npm_install_flags = Config.APP_SERVICE_ENV_NPM_INSTALL_FLAGS - - - // create k8s deployment - const { body: deployment } = await this.apps_api.createNamespacedDeployment(this.namespace, { - metadata: { - name: this.getName(app), - labels: labels - }, - spec: { - replicas: 1, - selector: { - matchLabels: labels - }, - template: { - metadata: { - labels: labels - }, - spec: { - terminationGracePeriodSeconds: 15, - automountServiceAccountToken: app.appid === Config.SYSTEM_EXTENSION_APPID, - containers: [ - { - image: image_name, - command: ['sh', '/app/start.sh'], - name: this.getName(app), - env: [ - { name: 'DB', value: app.config.db_name }, - { name: 'DB_URI', value: uri }, - { name: 'LOG_LEVEL', value: log_level }, - { name: 'ENABLE_CLOUD_FUNCTION_LOG', value: 'always' }, - { name: 'SERVER_SECRET_SALT', value: app.config.server_secret_salt }, - { name: 'APP_ID', value: app.appid }, - { name: 'RUNTIME_IMAGE', value: app.runtime?.image }, - { name: 'FLAGS', value: `--max_old_space_size=${max_old_space_size}` }, - { name: 'NPM_INSTALL_FLAGS', value: npm_install_flags }, - { name: 'OSS_ACCESS_SECRET', value: app.config.oss_access_secret }, - { name: 'OSS_INTERNAL_ENDPOINT', value: Config.MINIO_CONFIG.endpoint.internal }, - { name: 'OSS_EXTERNAL_ENDPOINT', value: Config.MINIO_CONFIG.endpoint.external }, - { name: 'OSS_REGION', value: Config.MINIO_CONFIG.region }, - ], - ports: [{ containerPort: 8000, name: 'http' }], - resources: { - requests: { - memory: `${req_memory}Mi`, - cpu: `${req_cpu}m` - }, - limits: { - memory: `${limit_memory}Mi`, - cpu: `${limit_cpu}m` - } - }, - startupProbe: { - httpGet: { - path: '/healthz', - port: 'http', - httpHeaders: [{ name: 'Referer', value: 'startupProbe' }] - }, - initialDelaySeconds: 0, - periodSeconds: 3, - timeoutSeconds: 3, - failureThreshold: 40 - }, - readinessProbe: { - httpGet: { - path: '/healthz', - port: 'http', - httpHeaders: [{ name: 'Referer', value: 'readinessProbe' }] - }, - initialDelaySeconds: 0, - periodSeconds: 60, - timeoutSeconds: 5, - failureThreshold: 3 - }, - } - ], - } - } - } - }) - - logger.info(`create k8s deployment ${deployment.metadata.name} of app ${app.appid}`) - logger.debug(`created k8s deployment of app ${app.appid}:`, deployment) - - return deployment - } - - /** - * Create k8s service for app - * @param app - * @param labels - * @returns - */ - private async createK8sService(app: IApplicationData, labels: any) { - const { body: service } = await this.core_api.createNamespacedService(this.namespace, { - metadata: { name: this.getName(app) }, - spec: { - selector: labels, - type: 'ClusterIP', - ports: [{ - targetPort: 8000, - port: 8000 - }] - } - }) - - logger.info(`create k8s service ${service.metadata.name} of app ${app.appid}`) - logger.debug(`created k8s service of app ${app.appid}:`, service) - - return service - } - - /** - * create k8s ingress for app - * @param app - * @param labels - * @returns - */ - private async createK8sIngress(app: IApplicationData, labels: any) { - const { body: ingress } = await this.net_api.createNamespacedIngress(this.namespace, { - metadata: { - name: this.getName(app), - labels: labels, - annotations: { - 'nginx.ingress.kubernetes.io/enable-cors': "true", - 'nginx.ingress.kubernetes.io/cors-expose-headers': "*", - 'nginx.ingress.kubernetes.io/configuration-snippet': `more_set_headers "request-id: $req_id";` - } - }, - spec: { - ingressClassName: "nginx", - rules: [ - { - host: `${app.appid}.${Config.APP_SERVICE_DEPLOY_HOST}`, - http: { - paths: [ - { - path: '/', - pathType: 'Prefix', - backend: { - service: { - name: this.getName(app), - port: { number: 8000 } - } - } - } - ] - } - } - ] - } - }) - - logger.info(`create k8s ingress ${ingress.metadata.name} of app ${app.appid}`) - logger.debug(`created k8s ingress of app ${app.appid}:`, ingress) - - return ingress - } - - private async getK8sDeployment(name: string) { - try { - const res = await this.apps_api.readNamespacedDeployment(name, this.namespace) - return res.body - } catch (error) { - return null - } - } - - private async getK8sService(name: string) { - try { - const res = await this.core_api.readNamespacedService(name, this.namespace) - return res.body - } catch (error) { - return null - } - } - - private async getK8sIngress(name: string) { - try { - const res = await this.net_api.readNamespacedIngress(name, this.namespace) - return res.body - } catch (error) { - return null - } - } -} \ No newline at end of file diff --git a/packages/system-server/src/support/service-operator.ts b/packages/system-server/src/support/service-operator.ts deleted file mode 100644 index 6d441fd20b..0000000000 --- a/packages/system-server/src/support/service-operator.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { CN_APPLICATIONS } from "../constants" -import { DatabaseAgent } from "../db" -import { IApplicationData } from "./application" -import { logger } from "./logger" -import Config from "../config" -import { KubernetesServiceDriver } from "./service-kubernetes-driver" -import { DockerContainerServiceDriver } from "./service-docker-driver" - - -export class ApplicationServiceOperator { - /** - * start app service - * @param app - * @returns - */ - static async start(app: IApplicationData) { - const db = DatabaseAgent.db - const driver = this.create() - const res = await driver.startService(app) - - await db.collection(CN_APPLICATIONS) - .updateOne( - { appid: app.appid }, - { - $set: { status: 'running' } - }) - - return res - } - - /** - * stop app service - * @param app - * @returns - */ - static async stop(app: IApplicationData) { - const db = DatabaseAgent.db - const driver = this.create() - const res = await driver.removeService(app) - - await db.collection(CN_APPLICATIONS) - .updateOne( - { appid: app.appid }, - { - $set: { status: 'stopped' } - }) - - return res - } - - /** - * restart app service - * @param app - * @returns - */ - static async restart(app: IApplicationData) { - await this.stop(app) - return await this.start(app) - } - - static create(): ServiceDriverInterface { - const driver = Config.SERVICE_DRIVER || 'docker' - logger.info("creating ServiceDriver with driver: " + driver) - if (driver === 'kubernetes') { - return new KubernetesServiceDriver() - } else { - return new DockerContainerServiceDriver() - } - } -} - - -export interface ServiceDriverInterface { - - /** - * Start application service - * @param app - * @returns - */ - startService(app: IApplicationData): Promise - - /** - * Remove application service - * @param app - */ - removeService(app: IApplicationData): Promise - - /** - * Get service info - * @param container - * @returns return null if container not exists - */ - info(app: IApplicationData): Promise - - - /** - * Get name of service - * @param app - */ - getName(app: IApplicationData): string -} \ No newline at end of file