diff --git a/server/.env.template b/server/.env.template deleted file mode 100644 index df7e170ead..0000000000 --- a/server/.env.template +++ /dev/null @@ -1,31 +0,0 @@ - -# database -DATABASE_URL=mongodb://admin:passw0rd@127.0.0.1:27017/sys_db?authSource=admin&replicaSet=rs0&w=majority&directConnection=true - -# jwt settings -JWT_SECRET=laf_server_abc123 -JWT_EXPIRES_IN=7d - -# server hosts -SERVER=http://localhost:3000 - -# minio settings -MINIO_CLIENT_PATH=mc -MINIO_DOMAIN=localhost -MINIO_EXTERNAL_ENDPOINT=http://localhost:9000 -MINIO_INTERNAL_ENDPOINT=http://localhost:9000 -MINIO_ROOT_ACCESS_KEY=minio-root -MINIO_ROOT_SECRET_KEY=passw0rd - -# gateway -DOMAIN=localhost -APISIX_API_URL=http://localhost:9080 -APISIX_API_KEY=abc123 - -# casdoor settings -CASDOOR_ENDPOINT=http://localhost:30070 -CASDOOR_ORG_NAME=laf -CASDOOR_APP_NAME=laf -CASDOOR_CLIENT_ID=a71f65e93723c436027e -CASDOOR_CLIENT_SECRET=0d7e157be08055867b81456df3c222ea7c68a097 -CASDOOR_REDIRECT_URI=http://localhost:3001/login_callback diff --git a/server/env.sh b/server/env.sh index 78217017e7..b4d2f2572f 100644 --- a/server/env.sh +++ b/server/env.sh @@ -8,6 +8,12 @@ kubectl exec -it ${POD_NAME} -n laf-system -- env > .env # remove MINIO_CLIENT_PATH line sed -i '' '/MINIO_CLIENT_PATH/d' .env +# remove API_SERVER_URL line +sed -i '' '/API_SERVER_URL/d' .env + +# add 'API_SERVER_URL=http://localhost:3000' line +echo 'API_SERVER_URL=http://localhost:3000' >> .env + # replace 'mongo.laf-system.svc.cluster.local' with '127.0.0.1' sed -i '' 's/mongo.laf-system.svc.cluster.local/127.0.0.1/g' .env diff --git a/server/src/application/application-task.service.ts b/server/src/application/application-task.service.ts index 1f8bdaa973..d2950bef19 100644 --- a/server/src/application/application-task.service.ts +++ b/server/src/application/application-task.service.ts @@ -5,16 +5,18 @@ import { isConditionTrue } from '../utils/getter' import { GatewayCoreService } from '../core/gateway.cr.service' import { PrismaService } from '../prisma.service' import * as assert from 'node:assert' -import { ApplicationCoreService } from '../core/application.cr.service' -import { StorageService } from 'src/storage/storage.service' -import { DatabaseService } from 'src/database/database.service' +import { StorageService } from '../storage/storage.service' +import { DatabaseService } from '../database/database.service' +import { ClusterService } from 'src/region/cluster/cluster.service' +import { RegionService } from 'src/region/region.service' @Injectable() export class ApplicationTaskService { private readonly logger = new Logger(ApplicationTaskService.name) constructor( - private readonly appCore: ApplicationCoreService, + private readonly regionService: RegionService, + private readonly clusterService: ClusterService, private readonly gatewayCore: GatewayCoreService, private readonly storageService: StorageService, private readonly databaseService: DatabaseService, @@ -66,6 +68,10 @@ export class ApplicationTaskService { */ private async reconcileCreatingPhase(app: Application) { const appid = app.appid + // get app region + const region = await this.regionService.findByAppId(appid) + assert(region, `Region ${app.regionName} not found`) + // get app bundle const bundle = await this.prisma.bundle.findUnique({ where: { @@ -77,10 +83,10 @@ export class ApplicationTaskService { }) assert(bundle, `Bundle ${app.bundleName} not found`) - const namespace = await this.appCore.getAppNamespace(appid) + const namespace = await this.clusterService.getAppNamespace(region, appid) if (!namespace) { this.logger.debug(`Creating namespace for application ${appid}`) - await this.appCore.createAppNamespace(appid, app.createdBy) + await this.clusterService.createAppNamespace(region, appid, app.createdBy) return } @@ -132,11 +138,14 @@ export class ApplicationTaskService { */ private async reconcileDeletingPhase(app: Application) { const appid = app.appid + // get app region + const region = await this.regionService.findByAppId(appid) + assert(region, `Region ${app.regionName} not found`) // delete namespace - const namespace = await this.appCore.getAppNamespace(appid) + const namespace = await this.clusterService.getAppNamespace(region, appid) if (namespace) { - await this.appCore.removeAppNamespace(appid) + await this.clusterService.removeAppNamespace(region, appid) return } diff --git a/server/src/core/application.cr.service.ts b/server/src/core/application.cr.service.ts deleted file mode 100644 index 3f6715af86..0000000000 --- a/server/src/core/application.cr.service.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common' -import { KubernetesService } from './kubernetes.service' -import * as k8s from '@kubernetes/client-node' -import { ResourceLabels } from '../constants' -import { GetApplicationNamespaceById } from '../utils/getter' - -@Injectable() -export class ApplicationCoreService { - private readonly logger = new Logger(ApplicationCoreService.name) - constructor(public k8sClient: KubernetesService) {} - - // create app namespace - async createAppNamespace(appid: string, userid: string) { - try { - const namespace = new k8s.V1Namespace() - namespace.metadata = new k8s.V1ObjectMeta() - namespace.metadata.name = GetApplicationNamespaceById(appid) - namespace.metadata.labels = { - [ResourceLabels.APP_ID]: appid, - [ResourceLabels.NAMESPACE_TYPE]: 'app', - [ResourceLabels.USER_ID]: userid, - } - const res = await this.k8sClient.coreV1Api.createNamespace(namespace) - return res.body - } catch (err) { - this.logger.error(err) - this.logger.debug(err?.response?.body) - return null - } - } - - // get app namespace - async getAppNamespace(appid: string) { - try { - const namespace = GetApplicationNamespaceById(appid) - const res = await this.k8sClient.coreV1Api.readNamespace(namespace) - return res.body - } catch (err) { - if (err?.response?.body?.reason === 'NotFound') return null - this.logger.error(err) - this.logger.debug(err?.response?.body) - return null - } - } - - // remove app namespace - async removeAppNamespace(appid: string) { - try { - const namespace = GetApplicationNamespaceById(appid) - const res = await this.k8sClient.coreV1Api.deleteNamespace(namespace) - return res - } catch (err) { - this.logger.error(err) - this.logger.debug(err?.response?.body) - return null - } - } -} diff --git a/server/src/core/core.module.ts b/server/src/core/core.module.ts index b8173f6c25..5b0502df59 100644 --- a/server/src/core/core.module.ts +++ b/server/src/core/core.module.ts @@ -1,11 +1,11 @@ import { Global, Module } from '@nestjs/common' -import { ApplicationCoreService } from './application.cr.service' +import { RegionModule } from 'src/region/region.module' import { GatewayCoreService } from './gateway.cr.service' -import { KubernetesService } from './kubernetes.service' @Global() @Module({ - providers: [KubernetesService, ApplicationCoreService, GatewayCoreService], - exports: [KubernetesService, ApplicationCoreService, GatewayCoreService], + imports: [RegionModule], + providers: [GatewayCoreService], + exports: [GatewayCoreService], }) export class CoreModule {} diff --git a/server/src/core/gateway.cr.service.ts b/server/src/core/gateway.cr.service.ts index 6f13df5844..aaaa2f8875 100644 --- a/server/src/core/gateway.cr.service.ts +++ b/server/src/core/gateway.cr.service.ts @@ -1,15 +1,19 @@ import { Injectable, Logger } from '@nestjs/common' import { GetApplicationNamespaceById } from '../utils/getter' -import { KubernetesService } from './kubernetes.service' import * as assert from 'node:assert' import { ResourceLabels } from '../constants' import { Gateway } from './api/gateway.cr' +import { ClusterService } from '../region/cluster/cluster.service' +import { RegionService } from '../region/region.service' @Injectable() export class GatewayCoreService { private readonly logger = new Logger(GatewayCoreService.name) - constructor(private readonly k8sService: KubernetesService) {} + constructor( + private readonly clusterService: ClusterService, + private readonly regionService: RegionService, + ) {} async create(appid: string) { const namespace = GetApplicationNamespaceById(appid) @@ -22,8 +26,11 @@ export class GatewayCoreService { gw.spec.appid = appid gw.spec.buckets = [] + const region = await this.regionService.findByAppId(appid) + const objectApi = this.clusterService.makeObjectApi(region) + try { - const res = await this.k8sService.objectApi.create(gw) + const res = await objectApi.create(gw) return Gateway.fromObject(res.body) } catch (error) { this.logger.error(error) @@ -35,15 +42,16 @@ export class GatewayCoreService { assert(appid, 'appid is required') const namespace = GetApplicationNamespaceById(appid) const name = appid + const region = await this.regionService.findByAppId(appid) + const customObjectApi = this.clusterService.makeCustomObjectApi(region) try { - const res = - await this.k8sService.customObjectApi.getNamespacedCustomObject( - Gateway.GVK.group, - Gateway.GVK.version, - namespace, - Gateway.GVK.plural, - name, - ) + const res = await customObjectApi.getNamespacedCustomObject( + Gateway.GVK.group, + Gateway.GVK.version, + namespace, + Gateway.GVK.plural, + name, + ) return Gateway.fromObject(res.body) } catch (err) { if (err?.response?.body?.reason === 'NotFound') return null @@ -52,24 +60,4 @@ export class GatewayCoreService { return null } } - - async update(gw: Gateway) { - try { - const res = await this.k8sService.patchCustomObject(gw) - return Gateway.fromObject(res) - } catch (error) { - this.logger.error(error, error.response?.body) - return null - } - } - - async remove(user: Gateway) { - try { - const res = await this.k8sService.deleteCustomObject(user) - return res - } catch (error) { - this.logger.error(error, error.response?.body) - return null - } - } } diff --git a/server/src/core/kubernetes.service.spec.ts b/server/src/core/kubernetes.service.spec.ts deleted file mode 100644 index 5bae5b44c7..0000000000 --- a/server/src/core/kubernetes.service.spec.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing' -import { KubernetesService } from './kubernetes.service' - -let service: KubernetesService - -beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [KubernetesService], - }).compile() - - service = module.get(KubernetesService) -}) - -describe('KubernetesService::apply() & delete()', () => { - const name = 'test-for-apply' - const spec = ` -apiVersion: v1 -kind: Namespace -metadata: - name: ${name} - ` - - it('should be defined', () => { - expect(service).toBeDefined() - }) - - it('should be able to apply a spec', async () => { - await service.applyYamlString(spec) - const exists = await service.existsNamespace(name) - expect(exists).toBeTruthy() - }) - - jest.setTimeout(20000) - - it('should be able to delete a spec', async () => { - await service.deleteYamlString(spec) - // wait for 10s - await new Promise((resolve) => setTimeout(resolve, 10000)) - const exists = await service.existsNamespace(name) - expect(exists).toBeFalsy() - }) - - afterAll(async () => { - await service.deleteYamlString(spec) - }) -}) - -describe('KubernetesService::createNamespace()', () => { - const name = 'test-for-create-namespace' - - beforeAll(async () => { - if (await service.existsNamespace(name)) { - await service.deleteNamespace(name) - } - }) - - it('should be able to create namespace', async () => { - await service.createNamespace(name) - - expect(await service.existsNamespace(name)).toBeTruthy() - - await service.deleteNamespace(name) - }) - - afterAll(async () => { - if (await service.existsNamespace(name)) { - await service.deleteNamespace(name) - } - }) -}) - -// describe.skip('list custom objects with label', () => { -// it('should be able to list custom objects with label', async () => { -// const userid = 'test-user-id' -// const res = await service.customObjectApi.listClusterCustomObject( -// Bucket.GVK.group, -// Bucket.GVK.version, -// Bucket.GVK.plural, -// undefined, -// undefined, -// undefined, -// undefined, -// `${ResourceLabels.USER_ID}=${userid}`, -// ) -// console.log(res.body) -// }) -// }) - -// describe.skip('patch custom objects', () => { -// it('should be able to patch custom objects', async () => { -// const name = '1i43zq' -// const namespace = name -// const res = await service.customObjectApi.getNamespacedCustomObject( -// Bucket.GVK.group, -// Bucket.GVK.version, -// namespace, -// Bucket.GVK.plural, -// name, -// ) - -// const data = res.body as Bucket -// data.spec = { -// ...data.spec, -// policy: BucketPolicy.Public, -// } - -// const res2 = await service.patchCustomObject(data).catch((err) => { -// console.log(err) -// }) -// console.log('patched', res2) -// }) -// }) - -// describe.skip('delete custom objects', () => { -// it('should be able to delete custom objects', async () => { -// const name = 'efme9x' -// const namespace = name -// const res = await service.customObjectApi.getNamespacedCustomObject( -// Bucket.GVK.group, -// Bucket.GVK.version, -// namespace, -// Bucket.GVK.plural, -// name, -// ) - -// const data = res.body as Bucket - -// const res2 = await service.deleteCustomObject(data) -// console.log('deleted', res2) -// }) -// }) diff --git a/server/src/core/kubernetes.service.ts b/server/src/core/kubernetes.service.ts deleted file mode 100644 index 62842a00c4..0000000000 --- a/server/src/core/kubernetes.service.ts +++ /dev/null @@ -1,173 +0,0 @@ -import { Injectable } from '@nestjs/common' -import * as k8s from '@kubernetes/client-node' -import { KubernetesObject } from '@kubernetes/client-node' -import { compare } from 'fast-json-patch' -import { GroupVersionKind } from './api/types' - -/** - * Single instance of the Kubernetes API client. - */ -let config: k8s.KubeConfig = null - -@Injectable() -export class KubernetesService { - get config() { - if (!config) { - config = new k8s.KubeConfig() - config.loadFromDefault() - } - return config - } - - get coreV1Api() { - return this.config.makeApiClient(k8s.CoreV1Api) - } - - get appsV1Api() { - return this.config.makeApiClient(k8s.AppsV1Api) - } - - get objectApi() { - return this.config.makeApiClient(k8s.KubernetesObjectApi) - } - - get customObjectApi() { - return this.config.makeApiClient(k8s.CustomObjectsApi) - } - - async createNamespace(name: string) { - const namespace = new k8s.V1Namespace() - namespace.metadata = new k8s.V1ObjectMeta() - namespace.metadata.name = name - const res = await this.coreV1Api.createNamespace(namespace) - return res.body - } - - async deleteNamespace(name: string) { - const res = await this.coreV1Api.deleteNamespace(name) - return res.body - } - - async existsNamespace(name: string) { - try { - const res = await this.coreV1Api.readNamespace(name) - return res.body - } catch (error) { - return false - } - } - - async applyYamlString(specString: string) { - const client = this.objectApi - const specs: k8s.KubernetesObject[] = k8s.loadAllYaml(specString) - const validSpecs = specs.filter((s) => s && s.kind && s.metadata) - const created: k8s.KubernetesObject[] = [] - - for (const spec of validSpecs) { - // this is to convince the old version of TypeScript that metadata exists even though we already filtered specs - // without metadata out - spec.metadata = spec.metadata || {} - spec.metadata.annotations = spec.metadata.annotations || {} - delete spec.metadata.annotations[ - 'kubectl.kubernetes.io/last-applied-configuration' - ] - spec.metadata.annotations[ - 'kubectl.kubernetes.io/last-applied-configuration' - ] = JSON.stringify(spec) - - try { - // try to get the resource, if it does not exist an error will be thrown and we will end up in the catch - // block. - await client.read(spec as any) - // we got the resource, so it exists, so patch it - // - // Note that this could fail if the spec refers to a custom resource. For custom resources you may need - // to specify a different patch merge strategy in the content-type header. - // - // See: https://github.com/kubernetes/kubernetes/issues/97423 - const response = await client.patch(spec) - created.push(response.body) - } catch (e) { - // we did not get the resource, so it does not exist, so create it - const response = await client.create(spec) - created.push(response.body) - } - } - return created - } - - async deleteYamlString(specString: string) { - const client = this.objectApi - const specs: k8s.KubernetesObject[] = k8s.loadAllYaml(specString) - const validSpecs = specs.filter((s) => s && s.kind && s.metadata) - const deleted: k8s.KubernetesObject[] = [] - - for (const spec of validSpecs) { - try { - // try to get the resource, if it does not exist an error will be thrown and we will end up in the catch - // block. - await client.read(spec as any) - // we got the resource, so it exists, so delete it - const response = await client.delete(spec) - deleted.push(response.body) - } catch (e) { - // we did not get the resource, so it does not exist, so do nothing - } - } - return deleted - } - - async patchCustomObject(spec: KubernetesObject) { - const client = this.customObjectApi - const gvk = GroupVersionKind.fromKubernetesObject(spec) - - // get the current spec - const res = await client.getNamespacedCustomObject( - gvk.group, - gvk.version, - spec.metadata.namespace, - gvk.plural, - spec.metadata.name, - ) - const currentSpec = res.body as KubernetesObject - - // calculate the patch - const patch = compare(currentSpec, spec) - const options = { - headers: { - 'Content-Type': k8s.PatchUtils.PATCH_FORMAT_JSON_PATCH, - }, - } - - // apply the patch - const response = await client.patchNamespacedCustomObject( - gvk.group, - gvk.version, - spec.metadata.namespace, - gvk.plural, - spec.metadata.name, - patch, - undefined, - undefined, - undefined, - options, - ) - - return response.body - } - - async deleteCustomObject(spec: KubernetesObject) { - const client = this.customObjectApi - const gvk = GroupVersionKind.fromKubernetesObject(spec) - - const response = await client.deleteNamespacedCustomObject( - gvk.group, - gvk.version, - spec.metadata.namespace, - gvk.plural, - spec.metadata.name, - ) - - return response.body - } -} diff --git a/server/src/database/database.module.ts b/server/src/database/database.module.ts index 354b384235..8492519e1e 100644 --- a/server/src/database/database.module.ts +++ b/server/src/database/database.module.ts @@ -1,8 +1,6 @@ import { Module } from '@nestjs/common' import { CollectionService } from './collection/collection.service' import { CollectionController } from './collection/collection.controller' -import { CoreModule } from '../core/core.module' -import { ApplicationModule } from '../application/application.module' import { PolicyController } from './policy/policy.controller' import { PolicyService } from './policy/policy.service' import { DatabaseService } from './database.service' @@ -15,7 +13,7 @@ import { RegionModule } from 'src/region/region.module' import { ApplicationService } from 'src/application/application.service' @Module({ - imports: [CoreModule, RegionModule], + imports: [RegionModule], controllers: [ CollectionController, PolicyController, diff --git a/server/src/database/database.service.ts b/server/src/database/database.service.ts index 5bcb72df63..0a3a590f7a 100644 --- a/server/src/database/database.service.ts +++ b/server/src/database/database.service.ts @@ -34,6 +34,7 @@ export class DatabaseService { ) this.logger.debug('Create database result: ', res) + assert.equal(res.ok, 1, 'Create app database failed: ' + appid) // create app database in database const database = await this.prisma.database.create({ diff --git a/server/src/function/function.module.ts b/server/src/function/function.module.ts index 52a0110691..0fc7960b1d 100644 --- a/server/src/function/function.module.ts +++ b/server/src/function/function.module.ts @@ -2,13 +2,12 @@ import { Module } from '@nestjs/common' import { JwtService } from '@nestjs/jwt' import { ApplicationModule } from '../application/application.module' import { PrismaService } from '../prisma.service' -import { CoreModule } from '../core/core.module' import { FunctionController } from './function.controller' import { FunctionService } from './function.service' import { DatabaseModule } from 'src/database/database.module' @Module({ - imports: [CoreModule, ApplicationModule, DatabaseModule], + imports: [ApplicationModule, DatabaseModule], controllers: [FunctionController], providers: [FunctionService, PrismaService, JwtService], exports: [FunctionService], diff --git a/server/src/instance/instance-task.service.ts b/server/src/instance/instance-task.service.ts index f5dabbf8ab..c9d31c4459 100644 --- a/server/src/instance/instance-task.service.ts +++ b/server/src/instance/instance-task.service.ts @@ -33,8 +33,7 @@ export class InstanceTaskService { for (const app of apps) { try { - const appid = app.appid - await this.instanceService.create(appid) + await this.instanceService.create(app) await this.prisma.application.updateMany({ where: { @@ -73,7 +72,7 @@ export class InstanceTaskService { for (const app of apps) { try { const appid = app.appid - const instance = await this.instanceService.get(appid) + const instance = await this.instanceService.get(app) const available = isConditionTrue( 'Available', instance.deployment.status?.conditions, @@ -123,8 +122,7 @@ export class InstanceTaskService { for (const app of apps) { try { - const appid = app.appid - await this.instanceService.remove(appid) + await this.instanceService.remove(app) await this.prisma.application.updateMany({ where: { @@ -161,7 +159,7 @@ export class InstanceTaskService { for (const app of apps) { try { const appid = app.appid - const instance = await this.instanceService.get(appid) + const instance = await this.instanceService.get(app) if (instance.deployment) continue // update application state @@ -198,8 +196,7 @@ export class InstanceTaskService { for (const app of apps) { try { - const appid = app.appid - await this.instanceService.remove(appid) + await this.instanceService.remove(app) await this.prisma.application.updateMany({ where: { @@ -238,8 +235,7 @@ export class InstanceTaskService { for (const app of apps) { try { - const appid = app.appid - await this.instanceService.create(appid) + await this.instanceService.create(app) await this.prisma.application.updateMany({ where: { diff --git a/server/src/instance/instance.module.ts b/server/src/instance/instance.module.ts index 39da094b10..240ac8dc37 100644 --- a/server/src/instance/instance.module.ts +++ b/server/src/instance/instance.module.ts @@ -1,13 +1,13 @@ import { Module } from '@nestjs/common' import { InstanceService } from './instance.service' import { InstanceTaskService } from './instance-task.service' -import { CoreModule } from '../core/core.module' import { PrismaService } from '../prisma.service' -import { StorageModule } from 'src/storage/storage.module' -import { DatabaseModule } from 'src/database/database.module' +import { StorageModule } from '../storage/storage.module' +import { DatabaseModule } from '../database/database.module' +import { RegionModule } from '../region/region.module' @Module({ - imports: [CoreModule, StorageModule, DatabaseModule], + imports: [StorageModule, DatabaseModule, RegionModule], providers: [InstanceService, InstanceTaskService, PrismaService], }) export class InstanceModule {} diff --git a/server/src/instance/instance.service.ts b/server/src/instance/instance.service.ts index 2299cc46fe..16a7112c78 100644 --- a/server/src/instance/instance.service.ts +++ b/server/src/instance/instance.service.ts @@ -2,31 +2,39 @@ import { V1Deployment } from '@kubernetes/client-node' import { Injectable, Logger } from '@nestjs/common' import { GetApplicationNamespaceById } from '../utils/getter' import { ResourceLabels } from '../constants' -import { KubernetesService } from '../core/kubernetes.service' import { PrismaService } from '../prisma.service' import { StorageService } from '../storage/storage.service' import { DatabaseService } from 'src/database/database.service' +import { ClusterService } from 'src/region/cluster/cluster.service' +import { Application, Region } from '@prisma/client' +import { RegionService } from 'src/region/region.service' + +type ApplicationWithRegion = Application & { region: Region } @Injectable() export class InstanceService { private logger = new Logger('InstanceService') constructor( - private readonly k8sService: KubernetesService, + private readonly clusterService: ClusterService, + private readonly regionService: RegionService, private readonly storageService: StorageService, private readonly databaseService: DatabaseService, private readonly prisma: PrismaService, ) {} - async create(appid: string) { + async create(app: Application) { + const appid = app.appid const labels = { [ResourceLabels.APP_ID]: appid } + const region = await this.regionService.findByAppId(appid) + const appWithRegion = { ...app, region } as ApplicationWithRegion - const res = await this.get(appid) + const res = await this.get(appWithRegion) if (!res.deployment) { await this.createDeployment(appid, labels) } if (!res.service) { - await this.createService(appid, labels) + await this.createService(appWithRegion, labels) } } @@ -173,62 +181,61 @@ export class InstanceService { }, } - const res = await this.k8sService.appsV1Api.createNamespacedDeployment( - namespace, - data, - ) + const appsV1Api = this.clusterService.makeAppsV1Api(app.region) + const res = await appsV1Api.createNamespacedDeployment(namespace, data) this.logger.log(`create k8s deployment ${res.body?.metadata?.name}`) return res.body } - async createService(appid: string, labels: any) { - const namespace = GetApplicationNamespaceById(appid) - const serviceName = appid - const res = await this.k8sService.coreV1Api.createNamespacedService( - namespace, - { - metadata: { name: serviceName, labels }, - spec: { - selector: labels, - type: 'ClusterIP', - ports: [{ port: 8000, targetPort: 8000, protocol: 'TCP' }], - }, + async createService(app: ApplicationWithRegion, labels: any) { + const namespace = GetApplicationNamespaceById(app.appid) + const serviceName = app.appid + const coreV1Api = this.clusterService.makeCoreV1Api(app.region) + const res = await coreV1Api.createNamespacedService(namespace, { + metadata: { name: serviceName, labels }, + spec: { + selector: labels, + type: 'ClusterIP', + ports: [{ port: 8000, targetPort: 8000, protocol: 'TCP' }], }, - ) + }) this.logger.log(`create k8s service ${res.body?.metadata?.name}`) return res.body } - async remove(appid: string) { - const { deployment, service } = await this.get(appid) + async remove(app: Application) { + const appid = app.appid + const region = await this.regionService.findByAppId(appid) + const { deployment, service } = await this.get(app) + const appsV1Api = this.clusterService.makeAppsV1Api(region) + const coreV1Api = this.clusterService.makeCoreV1Api(region) + const namespace = GetApplicationNamespaceById(appid) if (deployment) { - await this.k8sService.appsV1Api.deleteNamespacedDeployment( - appid, - namespace, - ) + await appsV1Api.deleteNamespacedDeployment(appid, namespace) } if (service) { const name = appid - await this.k8sService.coreV1Api.deleteNamespacedService(name, namespace) + await coreV1Api.deleteNamespacedService(name, namespace) } } - async get(appid: string) { - const deployment = await this.getDeployment(appid) - const service = await this.getService(appid) + async get(app: Application) { + const region = await this.regionService.findByAppId(app.appid) + const appWithRegion = { ...app, region } + const deployment = await this.getDeployment(appWithRegion) + const service = await this.getService(appWithRegion) return { deployment, service } } - async getDeployment(appid: string) { + async getDeployment(app: ApplicationWithRegion) { + const appid = app.appid + const appsV1Api = this.clusterService.makeAppsV1Api(app.region) try { const namespace = GetApplicationNamespaceById(appid) - const res = await this.k8sService.appsV1Api.readNamespacedDeployment( - appid, - namespace, - ) + const res = await appsV1Api.readNamespacedDeployment(appid, namespace) return res.body } catch (error) { if (error?.response?.body?.reason === 'NotFound') return null @@ -236,14 +243,14 @@ export class InstanceService { } } - async getService(appid: string) { + async getService(app: ApplicationWithRegion) { + const appid = app.appid + const coreV1Api = this.clusterService.makeCoreV1Api(app.region) + try { const serviceName = appid const namespace = GetApplicationNamespaceById(appid) - const res = await this.k8sService.coreV1Api.readNamespacedService( - serviceName, - namespace, - ) + const res = await coreV1Api.readNamespacedService(serviceName, namespace) return res.body } catch (error) { return null diff --git a/server/src/region/cluster/cluster.service.spec.ts b/server/src/region/cluster/cluster.service.spec.ts new file mode 100644 index 0000000000..9fb7eab184 --- /dev/null +++ b/server/src/region/cluster/cluster.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing' +import { ClusterService } from './cluster.service' + +describe('ClusterService', () => { + let service: ClusterService + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ClusterService], + }).compile() + + service = module.get(ClusterService) + }) + + it('should be defined', () => { + expect(service).toBeDefined() + }) +}) diff --git a/server/src/region/cluster/cluster.service.ts b/server/src/region/cluster/cluster.service.ts new file mode 100644 index 0000000000..c04e2f9973 --- /dev/null +++ b/server/src/region/cluster/cluster.service.ts @@ -0,0 +1,158 @@ +import { Injectable, Logger } from '@nestjs/common' +import { KubernetesObject } from '@kubernetes/client-node' +import * as k8s from '@kubernetes/client-node' +import { Region } from '@prisma/client' +import { GetApplicationNamespaceById } from 'src/utils/getter' +import { ResourceLabels } from 'src/constants' +import { compare } from 'fast-json-patch' +import { GroupVersionKind } from 'src/core/api/types' + +@Injectable() +export class ClusterService { + private readonly logger = new Logger(ClusterService.name) + + /** + * Load kubeconfig of region: + * - if region kubeconfig is empty, load from default config (in-cluster service account or ~/.kube/config) + * - if region kubeconfig is not empty, load from string + */ + loadKubeConfig(region: Region) { + const conf = region.clusterConf.kubeconfig + const kc = new k8s.KubeConfig() + + // if conf is empty load from default config (in-cluster service account or ~/.kube/config) + if (!conf) { + kc.loadFromDefault() + return kc + } + + // if conf is not empty load from string + kc.loadFromString(conf) + return kc + } + + // create app namespace + async createAppNamespace(region: Region, appid: string, userid: string) { + try { + const namespace = new k8s.V1Namespace() + namespace.metadata = new k8s.V1ObjectMeta() + namespace.metadata.name = GetApplicationNamespaceById(appid) + namespace.metadata.labels = { + [ResourceLabels.APP_ID]: appid, + [ResourceLabels.NAMESPACE_TYPE]: 'app', + [ResourceLabels.USER_ID]: userid, + } + const coreV1Api = this.makeCoreV1Api(region) + + const res = await coreV1Api.createNamespace(namespace) + return res.body + } catch (err) { + this.logger.error(err) + this.logger.debug(err?.response?.body) + return null + } + } + + // get app namespace + async getAppNamespace(region: Region, appid: string) { + try { + const coreV1Api = this.makeCoreV1Api(region) + const namespace = GetApplicationNamespaceById(appid) + const res = await coreV1Api.readNamespace(namespace) + return res.body + } catch (err) { + if (err?.response?.body?.reason === 'NotFound') return null + this.logger.error(err) + this.logger.debug(err?.response?.body) + return null + } + } + + // remove app namespace + async removeAppNamespace(region: Region, appid: string) { + try { + const coreV1Api = this.makeCoreV1Api(region) + const namespace = GetApplicationNamespaceById(appid) + const res = await coreV1Api.deleteNamespace(namespace) + return res + } catch (err) { + this.logger.error(err) + this.logger.debug(err?.response?.body) + return null + } + } + + async patchCustomObject(region: Region, spec: KubernetesObject) { + const client = this.makeCustomObjectApi(region) + const gvk = GroupVersionKind.fromKubernetesObject(spec) + + // get the current spec + const res = await client.getNamespacedCustomObject( + gvk.group, + gvk.version, + spec.metadata.namespace, + gvk.plural, + spec.metadata.name, + ) + const currentSpec = res.body as KubernetesObject + + // calculate the patch + const patch = compare(currentSpec, spec) + const options = { + headers: { + 'Content-Type': k8s.PatchUtils.PATCH_FORMAT_JSON_PATCH, + }, + } + + // apply the patch + const response = await client.patchNamespacedCustomObject( + gvk.group, + gvk.version, + spec.metadata.namespace, + gvk.plural, + spec.metadata.name, + patch, + undefined, + undefined, + undefined, + options, + ) + + return response.body + } + + async deleteCustomObject(region: Region, spec: KubernetesObject) { + const client = this.makeCustomObjectApi(region) + const gvk = GroupVersionKind.fromKubernetesObject(spec) + + const response = await client.deleteNamespacedCustomObject( + gvk.group, + gvk.version, + spec.metadata.namespace, + gvk.plural, + spec.metadata.name, + ) + + return response.body + } + + makeCoreV1Api(region: Region) { + const kc = this.loadKubeConfig(region) + return kc.makeApiClient(k8s.CoreV1Api) + } + + makeAppsV1Api(region: Region) { + const kc = this.loadKubeConfig(region) + return kc.makeApiClient(k8s.AppsV1Api) + } + + makeObjectApi(region: Region) { + const kc = this.loadKubeConfig(region) + return kc.makeApiClient(k8s.KubernetesObjectApi) + } + + makeCustomObjectApi(region: Region) { + const kc = this.loadKubeConfig(region) + return kc.makeApiClient(k8s.CustomObjectsApi) + } +} diff --git a/server/src/region/region.module.ts b/server/src/region/region.module.ts index 3d6a43090b..da4fb49bb4 100644 --- a/server/src/region/region.module.ts +++ b/server/src/region/region.module.ts @@ -2,10 +2,11 @@ import { Module } from '@nestjs/common' import { RegionService } from './region.service' import { RegionController } from './region.controller' import { PrismaService } from '../prisma.service' +import { ClusterService } from './cluster/cluster.service' @Module({ - providers: [RegionService, PrismaService], + providers: [RegionService, PrismaService, ClusterService], controllers: [RegionController], - exports: [RegionService], + exports: [RegionService, ClusterService], }) export class RegionModule {} diff --git a/server/src/storage/storage.module.ts b/server/src/storage/storage.module.ts index 7d8b1b701e..ce2646b7a7 100644 --- a/server/src/storage/storage.module.ts +++ b/server/src/storage/storage.module.ts @@ -1,6 +1,5 @@ import { Module } from '@nestjs/common' import { BucketController } from './bucket.controller' -import { CoreModule } from '../core/core.module' import { MinioService } from './minio/minio.service' import { StorageService } from './storage.service' import { PrismaService } from 'src/prisma.service' @@ -9,7 +8,7 @@ import { BucketService } from './bucket.service' import { RegionModule } from 'src/region/region.module' @Module({ - imports: [CoreModule, RegionModule], + imports: [RegionModule], controllers: [BucketController], providers: [ MinioService,