From 1ba9de7dc3411308ff5fa1bea99a3a14eb51d7e6 Mon Sep 17 00:00:00 2001 From: maslow <wangfugen@126.com> Date: Thu, 23 Mar 2023 23:01:28 +0800 Subject: [PATCH 1/3] feat(server): support website custom domain ssl cert auto-gen --- server/src/constants.ts | 4 ++++ server/src/gateway/apisix.service.ts | 14 ++++++++++++++ server/src/gateway/gateway.module.ts | 2 ++ server/src/instance/instance.service.ts | 5 +++-- server/src/region/cluster/cluster.service.ts | 12 ++++++------ 5 files changed, 29 insertions(+), 8 deletions(-) diff --git a/server/src/constants.ts b/server/src/constants.ts index 82b1b00fa0..24ad11bdde 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -100,6 +100,10 @@ export class ServerConfig { return process.env.API_SERVER_URL || 'http://localhost:3000' } + static get certManagerIssuerName() { + return process.env.CERT_MANAGER_ISSUER_NAME || 'laf-issuer' + } + /** default region conf */ static get DEFAULT_REGION_DATABASE_URL() { return process.env.DEFAULT_REGION_DATABASE_URL diff --git a/server/src/gateway/apisix.service.ts b/server/src/gateway/apisix.service.ts index 9d43bade0f..75d510855d 100644 --- a/server/src/gateway/apisix.service.ts +++ b/server/src/gateway/apisix.service.ts @@ -162,6 +162,20 @@ export class ApisixService { } } + async getRoute(region: Region, id: string) { + const conf = region.gatewayConf + const api_url = `${conf.apiUrl}/routes/${id}` + + const res = await this.httpService.axiosRef.get(api_url, { + headers: { + 'X-API-KEY': conf.apiKey, + 'Content-Type': 'application/json', + }, + }) + + return res.data + } + async deleteRoute(region: Region, id: string) { const conf = region.gatewayConf const api_url = `${conf.apiUrl}/routes/${id}` diff --git a/server/src/gateway/gateway.module.ts b/server/src/gateway/gateway.module.ts index 74d072422d..46bebcd25c 100644 --- a/server/src/gateway/gateway.module.ts +++ b/server/src/gateway/gateway.module.ts @@ -6,6 +6,7 @@ import { BucketDomainService } from './bucket-domain.service' import { WebsiteTaskService } from './website-task.service' import { BucketDomainTaskService } from './bucket-domain-task.service' import { RuntimeDomainTaskService } from './runtime-domain-task.service' +import { ApisixCrdService } from './apisix-crd.service'; @Module({ imports: [HttpModule], @@ -16,6 +17,7 @@ import { RuntimeDomainTaskService } from './runtime-domain-task.service' WebsiteTaskService, BucketDomainTaskService, RuntimeDomainTaskService, + ApisixCrdService, ], exports: [RuntimeDomainService, BucketDomainService], }) diff --git a/server/src/instance/instance.service.ts b/server/src/instance/instance.service.ts index 8930826ce2..11351c7bba 100644 --- a/server/src/instance/instance.service.ts +++ b/server/src/instance/instance.service.ts @@ -310,7 +310,7 @@ export class InstanceService { return res.body } catch (error) { if (error?.response?.body?.reason === 'NotFound') return null - return null + throw error } } @@ -324,7 +324,8 @@ export class InstanceService { const res = await coreV1Api.readNamespacedService(serviceName, namespace) return res.body } catch (error) { - return null + if (error?.response?.body?.reason === 'NotFound') return null + throw error } } } diff --git a/server/src/region/cluster/cluster.service.ts b/server/src/region/cluster/cluster.service.ts index 10a945085c..4653e0c8d4 100644 --- a/server/src/region/cluster/cluster.service.ts +++ b/server/src/region/cluster/cluster.service.ts @@ -52,8 +52,8 @@ export class ClusterService { return res.body } catch (err) { this.logger.error(err) - this.logger.debug(err?.response?.body) - return null + this.logger.error(err?.response?.body) + throw err } } @@ -67,8 +67,8 @@ export class ClusterService { } catch (err) { if (err?.response?.body?.reason === 'NotFound') return null this.logger.error(err) - this.logger.debug(err?.response?.body) - return null + this.logger.error(err?.response?.body) + throw err } } @@ -81,8 +81,8 @@ export class ClusterService { return res } catch (err) { this.logger.error(err) - this.logger.debug(err?.response?.body) - return null + this.logger.error(err?.response?.body) + throw err } } From c82fb69d792df726eabf765dc883605bea2b3b7c Mon Sep 17 00:00:00 2001 From: maslow <wangfugen@126.com> Date: Thu, 23 Mar 2023 23:02:43 +0800 Subject: [PATCH 2/3] feat(server): support website custom domain ssl cert auto-gen --- .../laf-server/templates/cert-issuer.yaml | 14 ++ server/src/gateway/apisix-crd.service.ts | 172 ++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 deploy/build/charts/laf-server/templates/cert-issuer.yaml create mode 100644 server/src/gateway/apisix-crd.service.ts diff --git a/deploy/build/charts/laf-server/templates/cert-issuer.yaml b/deploy/build/charts/laf-server/templates/cert-issuer.yaml new file mode 100644 index 0000000000..5c26c288b4 --- /dev/null +++ b/deploy/build/charts/laf-server/templates/cert-issuer.yaml @@ -0,0 +1,14 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: laf-issuer +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: admin@sealos.io + privateKeySecretRef: + name: letsencrypt-prod + solvers: + - http01: + ingress: + class: apisix \ No newline at end of file diff --git a/server/src/gateway/apisix-crd.service.ts b/server/src/gateway/apisix-crd.service.ts new file mode 100644 index 0000000000..a0cb1c901b --- /dev/null +++ b/server/src/gateway/apisix-crd.service.ts @@ -0,0 +1,172 @@ +import { Injectable, Logger } from '@nestjs/common' +import { Region, WebsiteHosting } from '@prisma/client' +import { LABEL_KEY_APP_ID, ServerConfig } from 'src/constants' +import { ClusterService } from 'src/region/cluster/cluster.service' +import { GetApplicationNamespaceByAppId } from 'src/utils/getter' + +// This class handles the creation and deletion of website domain certificates +// and ApisixTls resources using Kubernetes Custom Resource Definitions (CRDs). +@Injectable() +export class ApisixCrdService { + private readonly logger = new Logger(ApisixCrdService.name) + constructor(private readonly clusterService: ClusterService) {} + + // Read a certificate for a given website using cert-manager.io CRD + async readWebsiteDomainCert(region: Region, website: WebsiteHosting) { + try { + // Get the namespace based on the application ID + const namespace = GetApplicationNamespaceByAppId(website.appid) + // Create a Kubernetes API client for the specified region + const api = this.clusterService.makeObjectApi(region) + + // Make a request to read the Certificate resource + const res = await api.read({ + apiVersion: 'cert-manager.io/v1', + kind: 'Certificate', + metadata: { + name: website.id, + namespace, + }, + }) + return res.body + } catch (err) { + if (err?.response?.body?.reason === 'NotFound') return null + this.logger.error(err) + this.logger.error(err?.response?.body) + throw err + } + } + + // Create a certificate for a given website using cert-manager.io CRD + async createWebsiteDomainCert(region: Region, website: WebsiteHosting) { + // Get the namespace based on the application ID + const namespace = GetApplicationNamespaceByAppId(website.appid) + // Create a Kubernetes API client for the specified region + const api = this.clusterService.makeObjectApi(region) + + // Make a request to create the Certificate resource + const res = await api.create({ + apiVersion: 'cert-manager.io/v1', + kind: 'Certificate', + // Set the metadata for the Certificate resource + metadata: { + name: website.id, + namespace, + labels: { + 'laf.dev/website': website.id, + 'laf.dev/website-domain': website.domain, + [LABEL_KEY_APP_ID]: website.appid, + }, + }, + // Define the specification for the Certificate resource + spec: { + secretName: website.id, + dnsNames: [website.domain], + issuerRef: { + name: ServerConfig.certManagerIssuerName, + kind: 'ClusterIssuer', + }, + }, + }) + return res.body + } + + // Delete a certificate for a given website using cert-manager.io CRD + async deleteWebsiteDomainCert(region: Region, website: WebsiteHosting) { + // Get the namespace based on the application ID + const namespace = GetApplicationNamespaceByAppId(website.appid) + // Create a Kubernetes API client for the specified region + const api = this.clusterService.makeObjectApi(region) + + // Make a request to delete the Certificate resource + const res = await api.delete({ + apiVersion: 'cert-manager.io/v1', + kind: 'Certificate', + metadata: { + name: website.id, + namespace, + }, + }) + return res.body + } + + // Read an ApisixTls resource for a given website using apisix.apache.org CRD + async readWebsiteDomainApisixTls(region: Region, website: WebsiteHosting) { + try { + // Get the namespace based on the application ID + const namespace = GetApplicationNamespaceByAppId(website.appid) + // Create an API object for the specified region + const api = this.clusterService.makeObjectApi(region) + + // Make a request to read the ApisixTls resource + const res = await api.read({ + apiVersion: 'apisix.apache.org/v2', + kind: 'ApisixTls', + metadata: { + name: website.id, + namespace, + }, + }) + return res.body + } catch (err) { + if (err?.response?.body?.reason === 'NotFound') return null + this.logger.error(err) + this.logger.error(err?.response?.body) + throw err + } + } + + // Create an ApisixTls resource for a given website using apisix.apache.org CRD + async createWebsiteDomainApisixTls(region: Region, website: WebsiteHosting) { + // Get the namespace based on the application ID + const namespace = GetApplicationNamespaceByAppId(website.appid) + // Create an API object for the specified region + const api = this.clusterService.makeObjectApi(region) + + // Make a request to create the ApisixTls resource + const res = await api.create({ + apiVersion: 'apisix.apache.org/v2', + kind: 'ApisixTls', + // Set the metadata for the ApisixTls resource + metadata: { + name: website.id, + namespace, + labels: { + 'laf.dev/website': website.id, + 'laf.dev/website-domain': website.domain, + [LABEL_KEY_APP_ID]: website.appid, + }, + }, + // Define the specification for the ApisixTls resource + spec: { + hosts: [website.domain], + secret: { + name: website.id, + namespace, + }, + }, + }) + return res.body + } + + // Deletes the APISIX TLS configuration for a specific website domain + async deleteWebsiteDomainApisixTls(region: Region, website: WebsiteHosting) { + // Get the application namespace using the website's appid + const namespace = GetApplicationNamespaceByAppId(website.appid) + + // Create an API object for the specified region + const api = this.clusterService.makeObjectApi(region) + + // Send a delete request to remove the APISIX TLS configuration + const res = await api.delete({ + apiVersion: 'apisix.apache.org/v2', + kind: 'ApisixTls', + metadata: { + name: website.id, + namespace, + }, + }) + + return res.body + } +} From 3a49958f15d49abd4d69d2d89bff8dc6bf20c698 Mon Sep 17 00:00:00 2001 From: maslow <wangfugen@126.com> Date: Fri, 24 Mar 2023 15:02:40 +0800 Subject: [PATCH 3/3] feat(server): impl website task to support cert auto gen --- ...rvice.ts => apisix-custom-cert.service.ts} | 39 +++--- server/src/gateway/apisix.service.ts | 27 ++-- server/src/gateway/gateway.module.ts | 4 +- server/src/gateway/website-task.service.ts | 119 ++++++++++++++---- 4 files changed, 136 insertions(+), 53 deletions(-) rename server/src/gateway/{apisix-crd.service.ts => apisix-custom-cert.service.ts} (89%) diff --git a/server/src/gateway/apisix-crd.service.ts b/server/src/gateway/apisix-custom-cert.service.ts similarity index 89% rename from server/src/gateway/apisix-crd.service.ts rename to server/src/gateway/apisix-custom-cert.service.ts index a0cb1c901b..c147eddf15 100644 --- a/server/src/gateway/apisix-crd.service.ts +++ b/server/src/gateway/apisix-custom-cert.service.ts @@ -7,8 +7,8 @@ import { GetApplicationNamespaceByAppId } from 'src/utils/getter' // This class handles the creation and deletion of website domain certificates // and ApisixTls resources using Kubernetes Custom Resource Definitions (CRDs). @Injectable() -export class ApisixCrdService { - private readonly logger = new Logger(ApisixCrdService.name) +export class ApisixCustomCertService { + private readonly logger = new Logger(ApisixCustomCertService.name) constructor(private readonly clusterService: ClusterService) {} // Read a certificate for a given website using cert-manager.io CRD @@ -17,17 +17,17 @@ export class ApisixCrdService { // Get the namespace based on the application ID const namespace = GetApplicationNamespaceByAppId(website.appid) // Create a Kubernetes API client for the specified region - const api = this.clusterService.makeObjectApi(region) + const api = this.clusterService.makeCustomObjectApi(region) // Make a request to read the Certificate resource - const res = await api.read({ - apiVersion: 'cert-manager.io/v1', - kind: 'Certificate', - metadata: { - name: website.id, - namespace, - }, - }) + const res = await api.getNamespacedCustomObject( + 'cert-manager.io', + 'v1', + namespace, + 'certificates', + website.id, + ) + return res.body } catch (err) { if (err?.response?.body?.reason === 'NotFound') return null @@ -96,17 +96,16 @@ export class ApisixCrdService { // Get the namespace based on the application ID const namespace = GetApplicationNamespaceByAppId(website.appid) // Create an API object for the specified region - const api = this.clusterService.makeObjectApi(region) + const api = this.clusterService.makeCustomObjectApi(region) // Make a request to read the ApisixTls resource - const res = await api.read({ - apiVersion: 'apisix.apache.org/v2', - kind: 'ApisixTls', - metadata: { - name: website.id, - namespace, - }, - }) + const res = await api.getNamespacedCustomObject( + 'apisix.apache.org', + 'v2', + namespace, + 'apisixtlses', + website.id, + ) return res.body } catch (err) { if (err?.response?.body?.reason === 'NotFound') return null diff --git a/server/src/gateway/apisix.service.ts b/server/src/gateway/apisix.service.ts index 75d510855d..2d0cb705c5 100644 --- a/server/src/gateway/apisix.service.ts +++ b/server/src/gateway/apisix.service.ts @@ -14,6 +14,7 @@ export class ApisixService { const namespace = GetApplicationNamespaceByAppId(appid) const upstreamNode = `${appid}.${namespace}:8000` + // TODO: use appid as route id instead of `app-{appid} const id = `app-${appid}` const data = { name: id, @@ -46,6 +47,7 @@ export class ApisixService { } async deleteAppRoute(region: Region, appid: string) { + // TODO: use appid as route id instead of `app-{appid}` const id = `app-${appid}` const res = await this.deleteRoute(region, id) return res @@ -57,6 +59,7 @@ export class ApisixService { const minioUrl = new URL(region.storageConf.internalEndpoint) const upstreamNode = minioUrl.host + // TODO: use bucket object id as route id instead of bucket name const id = `bucket-${bucketName}` const data = { name: id, @@ -88,6 +91,7 @@ export class ApisixService { } async deleteBucketRoute(region: Region, bucketName: string) { + // TODO: use bucket object id as route id instead of bucket name const id = `bucket-${bucketName}` const res = await this.deleteRoute(region, id) return res @@ -166,14 +170,21 @@ export class ApisixService { const conf = region.gatewayConf const api_url = `${conf.apiUrl}/routes/${id}` - const res = await this.httpService.axiosRef.get(api_url, { - headers: { - 'X-API-KEY': conf.apiKey, - 'Content-Type': 'application/json', - }, - }) - - return res.data + try { + const res = await this.httpService.axiosRef.get(api_url, { + headers: { + 'X-API-KEY': conf.apiKey, + 'Content-Type': 'application/json', + }, + }) + return res.data + } catch (error) { + if (error?.response?.status === 404) { + return null + } + this.logger.error(error, error.response?.data) + return error + } } async deleteRoute(region: Region, id: string) { diff --git a/server/src/gateway/gateway.module.ts b/server/src/gateway/gateway.module.ts index 46bebcd25c..9bb5d48223 100644 --- a/server/src/gateway/gateway.module.ts +++ b/server/src/gateway/gateway.module.ts @@ -6,7 +6,7 @@ import { BucketDomainService } from './bucket-domain.service' import { WebsiteTaskService } from './website-task.service' import { BucketDomainTaskService } from './bucket-domain-task.service' import { RuntimeDomainTaskService } from './runtime-domain-task.service' -import { ApisixCrdService } from './apisix-crd.service'; +import { ApisixCustomCertService } from './apisix-custom-cert.service' @Module({ imports: [HttpModule], @@ -17,7 +17,7 @@ import { ApisixCrdService } from './apisix-crd.service'; WebsiteTaskService, BucketDomainTaskService, RuntimeDomainTaskService, - ApisixCrdService, + ApisixCustomCertService, ], exports: [RuntimeDomainService, BucketDomainService], }) diff --git a/server/src/gateway/website-task.service.ts b/server/src/gateway/website-task.service.ts index b75ec239f9..40efadee0e 100644 --- a/server/src/gateway/website-task.service.ts +++ b/server/src/gateway/website-task.service.ts @@ -12,6 +12,7 @@ import { SystemDatabase } from 'src/database/system-database' import { RegionService } from 'src/region/region.service' import * as assert from 'node:assert' import { ApisixService } from './apisix.service' +import { ApisixCustomCertService } from './apisix-custom-cert.service' @Injectable() export class WebsiteTaskService { @@ -22,6 +23,7 @@ export class WebsiteTaskService { constructor( private readonly regionService: RegionService, private readonly apisixService: ApisixService, + private readonly customCertService: ApisixCustomCertService, ) {} @Cron(CronExpression.EVERY_SECOND) @@ -62,22 +64,19 @@ export class WebsiteTaskService { .findOneAndUpdate( { phase: DomainPhase.Creating, - lockedAt: { - $lt: new Date(Date.now() - 1000 * this.lockTimeout), - }, - }, - { - $set: { - lockedAt: new Date(), - }, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, }, + { $set: { lockedAt: new Date() } }, ) if (!res.value) return this.logger.debug(res.value) // get region by appid - const site = res.value + const site = { + ...res.value, + id: res.value._id.toString(), + } const region = await this.regionService.findByAppId(site.appid) assert(region, 'region not found') @@ -97,7 +96,48 @@ export class WebsiteTaskService { site, bucketDomain.domain, ) - this.logger.debug(`create website route: `, route) + this.logger.log(`create website route: ${route?.node?.key}`) + + // create website custom certificate if custom domain is set + if (site.isCustom) { + // create custom domain certificate + let cert = await this.customCertService.readWebsiteDomainCert( + region, + site, + ) + if (!cert) { + cert = await this.customCertService.createWebsiteDomainCert( + region, + site, + ) + this.logger.log(`create website cert: ${site._id}`) + } + + // return to try to create cert again in next tick if cert is not found + if (!cert) { + this.logger.error(`create website cert failed: ${site._id}`) + return + } + + // config custom domain certificate to apisix + let apisixTls = await this.customCertService.readWebsiteDomainApisixTls( + region, + site, + ) + if (!apisixTls) { + apisixTls = await this.customCertService.createWebsiteDomainApisixTls( + region, + site, + ) + this.logger.log(`create website apisix tls: ${site._id}`) + } + + // return to try to create cert again in next tick if cert is not found + if (!apisixTls) { + this.logger.error(`create website apisix tls failed: ${site._id}`) + return + } + } // update phase to `Created` const updated = await db @@ -117,7 +157,10 @@ export class WebsiteTaskService { if (updated.modifiedCount !== 1) { this.logger.error(`update website hosting phase failed: ${site._id}`) + return } + + this.logger.log(`update website phase to 'Created': ${site._id}`) } /** @@ -133,28 +176,55 @@ export class WebsiteTaskService { .findOneAndUpdate( { phase: DomainPhase.Deleting, - lockedAt: { - $lt: new Date(Date.now() - 1000 * this.lockTimeout), - }, - }, - { - $set: { - lockedAt: new Date(), - }, + lockedAt: { $lt: new Date(Date.now() - 1000 * this.lockTimeout) }, }, + { $set: { lockedAt: new Date() } }, ) if (!res.value) return // get region by appid - const site = res.value + const site = { + ...res.value, + id: res.value._id.toString(), + } const region = await this.regionService.findByAppId(site.appid) assert(region, 'region not found') - // delete website route - const route = await this.apisixService.deleteWebsiteRoute(region, site) - - this.logger.debug(`delete website route: `, route) + // delete website route if exists + const route = await this.apisixService.getRoute(region, site._id.toString()) + if (route) { + await this.apisixService.deleteWebsiteRoute(region, site) + const res = await this.apisixService.deleteWebsiteRoute(region, site) + this.logger.log(`delete website route: ${res?.key}`) + this.logger.debug('delete website route', res) + } + // delete website custom certificate if custom domain is set + if (site.isCustom) { + // delete custom domain certificate + const cert = await this.customCertService.readWebsiteDomainCert( + region, + site, + ) + if (cert) { + await this.customCertService.deleteWebsiteDomainCert(region, site) + this.logger.log(`delete website cert: ${site._id}`) + // return to wait for cert to be deleted + return + } + + // delete custom domain certificate from apisix + const apisixTls = await this.customCertService.readWebsiteDomainApisixTls( + region, + site, + ) + if (apisixTls) { + await this.customCertService.deleteWebsiteDomainApisixTls(region, site) + this.logger.log(`delete website apisix tls: ${site._id}`) + // return to wait for cert to be deleted + return + } + } // update phase to `Deleted` const updated = await db @@ -174,7 +244,10 @@ export class WebsiteTaskService { if (updated.modifiedCount > 1) { this.logger.error(`update website hosting phase failed: ${site._id}`) + return } + + this.logger.log(`update website phase to 'Deleted': ${site._id}`) } /**