feat(server): support multi-region cluster (#648)

server/env.sh

@@ -8,6 +8,12 @@ kubectl exec -it ${POD_NAME} -n laf-system -- env > .env
 # remove MINIO_CLIENT_PATH line
 sed -i '' '/MINIO_CLIENT_PATH/d' .env
 
+# remove API_SERVER_URL line
+sed -i '' '/API_SERVER_URL/d' .env
+
+# add 'API_SERVER_URL=http://localhost:3000' line
+echo 'API_SERVER_URL=http://localhost:3000' >> .env
+
 # replace 'mongo.laf-system.svc.cluster.local' with '127.0.0.1'
 sed -i '' 's/mongo.laf-system.svc.cluster.local/127.0.0.1/g' .env
 

server/src/application/application-task.service.ts

@@ -5,16 +5,18 @@ import { isConditionTrue } from '../utils/getter'
 import { GatewayCoreService } from '../core/gateway.cr.service'
 import { PrismaService } from '../prisma.service'
 import * as assert from 'node:assert'
-import { ApplicationCoreService } from '../core/application.cr.service'
-import { StorageService } from 'src/storage/storage.service'
-import { DatabaseService } from 'src/database/database.service'
+import { StorageService } from '../storage/storage.service'
+import { DatabaseService } from '../database/database.service'
+import { ClusterService } from 'src/region/cluster/cluster.service'
+import { RegionService } from 'src/region/region.service'
 
 @Injectable()
 export class ApplicationTaskService {
   private readonly logger = new Logger(ApplicationTaskService.name)
 
   constructor(
-    private readonly appCore: ApplicationCoreService,
+    private readonly regionService: RegionService,
+    private readonly clusterService: ClusterService,
     private readonly gatewayCore: GatewayCoreService,
     private readonly storageService: StorageService,
     private readonly databaseService: DatabaseService,
@@ -66,6 +68,10 @@ export class ApplicationTaskService {
    */
   private async reconcileCreatingPhase(app: Application) {
     const appid = app.appid
+    // get app region
+    const region = await this.regionService.findByAppId(appid)
+    assert(region, `Region ${app.regionName} not found`)
+
     // get app bundle
     const bundle = await this.prisma.bundle.findUnique({
       where: {
@@ -77,10 +83,10 @@ export class ApplicationTaskService {
     })
     assert(bundle, `Bundle ${app.bundleName} not found`)
 
-    const namespace = await this.appCore.getAppNamespace(appid)
+    const namespace = await this.clusterService.getAppNamespace(region, appid)
     if (!namespace) {
       this.logger.debug(`Creating namespace for application ${appid}`)
-      await this.appCore.createAppNamespace(appid, app.createdBy)
+      await this.clusterService.createAppNamespace(region, appid, app.createdBy)
       return
     }
 
@@ -132,11 +138,14 @@ export class ApplicationTaskService {
    */
   private async reconcileDeletingPhase(app: Application) {
     const appid = app.appid
+    // get app region
+    const region = await this.regionService.findByAppId(appid)
+    assert(region, `Region ${app.regionName} not found`)
 
     // delete namespace
-    const namespace = await this.appCore.getAppNamespace(appid)
+    const namespace = await this.clusterService.getAppNamespace(region, appid)
     if (namespace) {
-      await this.appCore.removeAppNamespace(appid)
+      await this.clusterService.removeAppNamespace(region, appid)
       return
     }
 

server/src/core/core.module.ts

@@ -1,11 +1,11 @@
 import { Global, Module } from '@nestjs/common'
-import { ApplicationCoreService } from './application.cr.service'
+import { RegionModule } from 'src/region/region.module'
 import { GatewayCoreService } from './gateway.cr.service'
-import { KubernetesService } from './kubernetes.service'
 
 @Global()
 @Module({
-  providers: [KubernetesService, ApplicationCoreService, GatewayCoreService],
-  exports: [KubernetesService, ApplicationCoreService, GatewayCoreService],
+  imports: [RegionModule],
+  providers: [GatewayCoreService],
+  exports: [GatewayCoreService],
 })
 export class CoreModule {}

server/src/core/gateway.cr.service.ts

@@ -1,15 +1,19 @@
 import { Injectable, Logger } from '@nestjs/common'
 import { GetApplicationNamespaceById } from '../utils/getter'
-import { KubernetesService } from './kubernetes.service'
 import * as assert from 'node:assert'
 import { ResourceLabels } from '../constants'
 import { Gateway } from './api/gateway.cr'
+import { ClusterService } from '../region/cluster/cluster.service'
+import { RegionService } from '../region/region.service'
 
 @Injectable()
 export class GatewayCoreService {
   private readonly logger = new Logger(GatewayCoreService.name)
 
-  constructor(private readonly k8sService: KubernetesService) {}
+  constructor(
+    private readonly clusterService: ClusterService,
+    private readonly regionService: RegionService,
+  ) {}
 
   async create(appid: string) {
     const namespace = GetApplicationNamespaceById(appid)
@@ -22,8 +26,11 @@ export class GatewayCoreService {
     gw.spec.appid = appid
     gw.spec.buckets = []
 
+    const region = await this.regionService.findByAppId(appid)
+    const objectApi = this.clusterService.makeObjectApi(region)
+
     try {
-      const res = await this.k8sService.objectApi.create(gw)
+      const res = await objectApi.create(gw)
       return Gateway.fromObject(res.body)
     } catch (error) {
       this.logger.error(error)
@@ -35,15 +42,16 @@ export class GatewayCoreService {
     assert(appid, 'appid is required')
     const namespace = GetApplicationNamespaceById(appid)
     const name = appid
+    const region = await this.regionService.findByAppId(appid)
+    const customObjectApi = this.clusterService.makeCustomObjectApi(region)
     try {
-      const res =
-        await this.k8sService.customObjectApi.getNamespacedCustomObject(
-          Gateway.GVK.group,
-          Gateway.GVK.version,
-          namespace,
-          Gateway.GVK.plural,
-          name,
-        )
+      const res = await customObjectApi.getNamespacedCustomObject(
+        Gateway.GVK.group,
+        Gateway.GVK.version,
+        namespace,
+        Gateway.GVK.plural,
+        name,
+      )
       return Gateway.fromObject(res.body)
     } catch (err) {
       if (err?.response?.body?.reason === 'NotFound') return null
@@ -52,24 +60,4 @@ export class GatewayCoreService {
       return null
     }
   }
-
-  async update(gw: Gateway) {
-    try {
-      const res = await this.k8sService.patchCustomObject(gw)
-      return Gateway.fromObject(res)
-    } catch (error) {
-      this.logger.error(error, error.response?.body)
-      return null
-    }
-  }
-
-  async remove(user: Gateway) {
-    try {
-      const res = await this.k8sService.deleteCustomObject(user)
-      return res
-    } catch (error) {
-      this.logger.error(error, error.response?.body)
-      return null
-    }
-  }
 }