Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(server): add metered billing module, remove subscription module, discard Prisma #1187

Merged
merged 57 commits into from
May 29, 2023
Merged
Changes from 1 commit
Commits
Show all changes
57 commits
Select commit Hold shift + click to select a range
19acb20
design metering schema
maslow May 9, 2023
c85d99e
Merge remote-tracking branch 'upstream/main' into design-metering
maslow May 10, 2023
f65e737
Merge remote-tracking branch 'upstream/main' into design-metering
maslow May 12, 2023
e87187a
Merge remote-tracking branch 'upstream/main' into design-metering
maslow May 14, 2023
ef434e7
add region bundle schema
maslow May 14, 2023
bc36980
remove prisma in some modules
maslow May 16, 2023
e7f360d
remove prisma in gateway, website, storage
maslow May 16, 2023
d3b6888
remove prisma in dependency module
maslow May 16, 2023
45c0dd3
remove prisma in account module
maslow May 17, 2023
29ee47b
remove prisma in instance module
maslow May 17, 2023
7bc52d4
remove prisma in auth and user module
maslow May 17, 2023
54550c8
remove prisma in function and trigger module
maslow May 17, 2023
5076425
remove prisma totally
maslow May 17, 2023
9f5b31d
Merge remote-tracking branch 'upstream/main' into design-metering
maslow May 17, 2023
398c3a1
add resource option & template api
maslow May 19, 2023
a2dc9db
rename resource template to bundle
maslow May 19, 2023
fec7757
design metering schema
maslow May 9, 2023
1f51402
add region bundle schema
maslow May 14, 2023
393eda0
remove prisma in some modules
maslow May 16, 2023
52d926d
remove prisma in gateway, website, storage
maslow May 16, 2023
1f51c57
remove prisma in dependency module
maslow May 16, 2023
5938807
remove prisma in account module
maslow May 17, 2023
7e3fbf6
remove prisma in instance module
maslow May 17, 2023
e65b211
remove prisma in auth and user module
maslow May 17, 2023
cba015c
remove prisma in function and trigger module
maslow May 17, 2023
2d2a116
remove prisma totally
maslow May 17, 2023
2093afb
add resource option & template api
maslow May 19, 2023
b3a7d95
rename resource template to bundle
maslow May 19, 2023
10e8d6f
Merge remote-tracking branch 'upstream/metering' into design-metering
maslow May 22, 2023
935c7b2
add billing module & price calculation api
maslow May 23, 2023
1f23d43
add response api typings
maslow May 23, 2023
9130c52
Design metering (#1169)
maslow May 23, 2023
cdfeee4
update app bundle api
maslow May 24, 2023
1238998
impl billing task
maslow May 24, 2023
b6bf134
feat(server): add application billings http api
maslow May 24, 2023
e3f5886
impl billing payment task
maslow May 25, 2023
c415158
restart app while updating bundle
maslow May 25, 2023
5e045ff
update user profile api
maslow May 25, 2023
2f1cb16
fix account api response body
maslow May 26, 2023
1f7d298
feat(web): new metering design (#1171)
LeezQ May 26, 2023
cd72e16
add pagination response decoration
maslow May 26, 2023
34b1f35
feat(web): metering web (#1173)
LeezQ May 26, 2023
cd44872
chore: update entity typings
maslow May 26, 2023
172f9cc
fix(web): modal height (#1175)
LeezQ May 26, 2023
6a240c9
add application trial tier limit
maslow May 26, 2023
b51af3d
feat(server): add invite code feature and billings pagination (#1183)
HUAHUAI23 May 29, 2023
aab7d8e
feat(server): process free trial billing
maslow May 29, 2023
5c06979
fix(web): update app state api & rules add api (#1184)
LeezQ May 29, 2023
1033a18
fix(server): update response struct of rules, add response types
maslow May 29, 2023
ef1d85a
Merge remote-tracking branch 'upstream/design-metering' into design-m…
maslow May 29, 2023
7b558ea
refactor(web): api definitions (#1185)
LeezQ May 29, 2023
8440f77
fix(server): fix trial app logic
maslow May 29, 2023
4815947
Merge remote-tracking branch 'upstream/design-metering' into design-m…
maslow May 29, 2023
5234138
Feat definition (#1186)
LeezQ May 29, 2023
6ed2885
merge main into design-metering
maslow May 29, 2023
f89d90c
add metering yaml to laf helm charts
maslow May 29, 2023
4e4f460
Merge branch 'main' into design-metering
maslow May 29, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
remove prisma in instance module
maslow committed May 17, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
commit 29ee47bbd30a8c282b72c8ececb35a7b50bab052
1 change: 0 additions & 1 deletion server/src/constants.ts
Original file line number Diff line number Diff line change
@@ -133,7 +133,6 @@ export class ServerConfig {
export const LABEL_KEY_USER_ID = 'laf.dev/user.id'
export const LABEL_KEY_APP_ID = 'laf.dev/appid'
export const LABEL_KEY_NAMESPACE_TYPE = 'laf.dev/namespace.type'
export const LABEL_KEY_BUNDLE = 'laf.dev/bundle'
export const LABEL_KEY_NODE_TYPE = 'laf.dev/node.type'
export enum NodeType {
Runtime = 'runtime',
16 changes: 10 additions & 6 deletions server/src/instance/instance-task.service.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
import { Injectable, Logger } from '@nestjs/common'
import { Cron, CronExpression } from '@nestjs/schedule'
import { Application, ApplicationPhase, ApplicationState } from '@prisma/client'
import { isConditionTrue } from '../utils/getter'
import { InstanceService } from './instance.service'
import { ServerConfig, TASK_LOCK_INIT_TIME } from 'src/constants'
import { SystemDatabase } from 'src/database/system-database'
import { CronJobService } from 'src/trigger/cron-job.service'
import {
Application,
ApplicationPhase,
ApplicationState,
} from 'src/application/entities/application'

@Injectable()
export class InstanceTaskService {
@@ -100,7 +104,7 @@ export class InstanceTaskService {
const app = res.value

// create instance
await this.instanceService.create(app)
await this.instanceService.create(app.appid)

// if waiting time is more than 5 minutes, stop the application
const waitingTime = Date.now() - app.updatedAt.getTime()
@@ -122,7 +126,7 @@ export class InstanceTaskService {
}

const appid = app.appid
const instance = await this.instanceService.get(app)
const instance = await this.instanceService.get(appid)
const unavailable =
instance.deployment?.status?.unavailableReplicas || false
if (unavailable) {
@@ -218,16 +222,16 @@ export class InstanceTaskService {
const waitingTime = Date.now() - app.updatedAt.getTime()

// check if the instance is removed
const instance = await this.instanceService.get(app)
const instance = await this.instanceService.get(app.appid)
if (instance.deployment) {
await this.instanceService.remove(app)
await this.instanceService.remove(app.appid)
await this.relock(appid, waitingTime)
return
}

// check if the service is removed
if (instance.service) {
await this.instanceService.remove(app)
await this.instanceService.remove(app.appid)
await this.relock(appid, waitingTime)
return
}
3 changes: 2 additions & 1 deletion server/src/instance/instance.module.ts
Original file line number Diff line number Diff line change
@@ -4,9 +4,10 @@ import { InstanceTaskService } from './instance-task.service'
import { StorageModule } from '../storage/storage.module'
import { DatabaseModule } from '../database/database.module'
import { TriggerModule } from 'src/trigger/trigger.module'
import { ApplicationModule } from 'src/application/application.module'

@Module({
imports: [StorageModule, DatabaseModule, TriggerModule],
imports: [StorageModule, DatabaseModule, TriggerModule, ApplicationModule],
providers: [InstanceService, InstanceTaskService],
})
export class InstanceModule {}
231 changes: 93 additions & 138 deletions server/src/instance/instance.service.ts
Original file line number Diff line number Diff line change
@@ -3,93 +3,140 @@ import { Injectable, Logger } from '@nestjs/common'
import { GetApplicationNamespaceByAppId } from '../utils/getter'
import {
LABEL_KEY_APP_ID,
LABEL_KEY_BUNDLE,
LABEL_KEY_NODE_TYPE,
MB,
NodeType,
} from '../constants'
import { PrismaService } from '../prisma/prisma.service'
import { StorageService } from '../storage/storage.service'
import { DatabaseService } from 'src/database/database.service'
import { ClusterService } from 'src/region/cluster/cluster.service'
import {
Application,
ApplicationBundle,
ApplicationConfiguration,
Runtime,
} from '@prisma/client'
import { RegionService } from 'src/region/region.service'
import { Region } from 'src/region/entities/region'

type ApplicationWithRegion = Application & { region: Region }
import { SystemDatabase } from 'src/database/system-database'
import { ApplicationWithRelations } from 'src/application/entities/application'
import { ApplicationService } from 'src/application/application.service'

@Injectable()
export class InstanceService {
private logger = new Logger('InstanceService')
private readonly logger = new Logger('InstanceService')
private readonly db = SystemDatabase.db

constructor(
private readonly clusterService: ClusterService,
private readonly regionService: RegionService,
private readonly cluster: ClusterService,
private readonly storageService: StorageService,
private readonly databaseService: DatabaseService,
private readonly prisma: PrismaService,
private readonly applicationService: ApplicationService,
) {}

async create(app: Application) {
const appid = app.appid
public async create(appid: string) {
const app = await this.applicationService.findOneUnsafe(appid)
const labels = { [LABEL_KEY_APP_ID]: appid }
const region = await this.regionService.findByAppId(appid)
const appWithRegion = { ...app, region } as ApplicationWithRegion
const region = app.region

// Although a namespace has already been created during application creation,
// we still need to check it again here in order to handle situations where the cluster is rebuilt.
const namespace = await this.clusterService.getAppNamespace(region, appid)
const namespace = await this.cluster.getAppNamespace(region, appid)
if (!namespace) {
this.logger.debug(`Creating namespace for application ${appid}`)
await this.clusterService.createAppNamespace(region, appid, app.createdBy)
await this.cluster.createAppNamespace(
region,
appid,
app.createdBy.toString(),
)
}

const res = await this.get(appWithRegion)
// ensure deployment created
const res = await this.get(app.appid)
if (!res.deployment) {
await this.createDeployment(appid, labels)
await this.createDeployment(app, labels)
}

// ensure service created
if (!res.service) {
await this.createService(appWithRegion, labels)
await this.createService(app, labels)
}
}

async createDeployment(appid: string, labels: any) {
public async remove(appid: string) {
const app = await this.applicationService.findOneUnsafe(appid)
const region = app.region
const { deployment, service } = await this.get(appid)

const namespace = await this.cluster.getAppNamespace(region, app.appid)
if (!namespace) return // namespace not found, nothing to do

const appsV1Api = this.cluster.makeAppsV1Api(region)
const coreV1Api = this.cluster.makeCoreV1Api(region)

// ensure deployment deleted
if (deployment) {
await appsV1Api.deleteNamespacedDeployment(appid, namespace.metadata.name)
}

// ensure service deleted
if (service) {
const name = appid
await coreV1Api.deleteNamespacedService(name, namespace.metadata.name)
}
this.logger.log(`remove k8s deployment ${deployment?.metadata?.name}`)
}

public async get(appid: string) {
const app = await this.applicationService.findOneUnsafe(appid)
const region = app.region
const namespace = await this.cluster.getAppNamespace(region, app.appid)
if (!namespace) {
return { deployment: null, service: null }
}

const deployment = await this.getDeployment(app)
const service = await this.getService(app)
return { deployment, service }
}

public async restart(appid: string) {
const app = await this.applicationService.findOneUnsafe(appid)
const region = app.region
const { deployment } = await this.get(appid)
if (!deployment) {
await this.create(appid)
return
}

deployment.spec = await this.makeDeploymentSpec(
app,
deployment.spec.template.metadata.labels,
)
const appsV1Api = this.cluster.makeAppsV1Api(region)
const namespace = GetApplicationNamespaceByAppId(appid)
const app = await this.prisma.application.findUnique({
where: { appid },
include: {
configuration: true,
bundle: true,
runtime: true,
region: true,
},
})
const res = await appsV1Api.replaceNamespacedDeployment(
app.appid,
namespace,
deployment,
)

this.logger.log(`restart k8s deployment ${res.body?.metadata?.name}`)
}

// add bundle label
labels[LABEL_KEY_BUNDLE] = app.bundle.name
private async createDeployment(app: ApplicationWithRelations, labels: any) {
const appid = app.appid
const namespace = GetApplicationNamespaceByAppId(appid)

// create deployment
const data = new V1Deployment()
data.metadata = { name: app.appid, labels }
data.spec = await this.makeDeploymentSpec(app, labels)

const appsV1Api = this.clusterService.makeAppsV1Api(app.region)
const appsV1Api = this.cluster.makeAppsV1Api(app.region)
const res = await appsV1Api.createNamespacedDeployment(namespace, data)

this.logger.log(`create k8s deployment ${res.body?.metadata?.name}`)

return res.body
}

async createService(app: ApplicationWithRegion, labels: any) {
private async createService(app: ApplicationWithRelations, labels: any) {
const namespace = GetApplicationNamespaceByAppId(app.appid)
const serviceName = app.appid
const coreV1Api = this.clusterService.makeCoreV1Api(app.region)
const coreV1Api = this.cluster.makeCoreV1Api(app.region)
const res = await coreV1Api.createNamespacedService(namespace, {
metadata: { name: serviceName, labels },
spec: {
@@ -102,49 +149,9 @@ export class InstanceService {
return res.body
}

async remove(app: Application) {
private async getDeployment(app: ApplicationWithRelations) {
const appid = app.appid
const region = await this.regionService.findByAppId(appid)
const { deployment, service } = await this.get(app)

const namespace = await this.clusterService.getAppNamespace(
region,
app.appid,
)
if (!namespace) return

const appsV1Api = this.clusterService.makeAppsV1Api(region)
const coreV1Api = this.clusterService.makeCoreV1Api(region)

if (deployment) {
await appsV1Api.deleteNamespacedDeployment(appid, namespace.metadata.name)
}
if (service) {
const name = appid
await coreV1Api.deleteNamespacedService(name, namespace.metadata.name)
}
this.logger.log(`remove k8s deployment ${deployment?.metadata?.name}`)
}

async get(app: Application) {
const region = await this.regionService.findByAppId(app.appid)
const namespace = await this.clusterService.getAppNamespace(
region,
app.appid,
)
if (!namespace) {
return { deployment: null, service: null }
}

const appWithRegion = { ...app, region }
const deployment = await this.getDeployment(appWithRegion)
const service = await this.getService(appWithRegion)
return { deployment, service }
}

async getDeployment(app: ApplicationWithRegion) {
const appid = app.appid
const appsV1Api = this.clusterService.makeAppsV1Api(app.region)
const appsV1Api = this.cluster.makeAppsV1Api(app.region)
try {
const namespace = GetApplicationNamespaceByAppId(appid)
const res = await appsV1Api.readNamespacedDeployment(appid, namespace)
@@ -155,9 +162,9 @@ export class InstanceService {
}
}

async getService(app: ApplicationWithRegion) {
private async getService(app: ApplicationWithRelations) {
const appid = app.appid
const coreV1Api = this.clusterService.makeCoreV1Api(app.region)
const coreV1Api = this.cluster.makeCoreV1Api(app.region)

try {
const serviceName = appid
@@ -170,45 +177,8 @@ export class InstanceService {
}
}

async restart(appid: string) {
const app = await this.prisma.application.findUnique({
where: { appid },
include: {
configuration: true,
bundle: true,
runtime: true,
region: true,
},
})
const { deployment } = await this.get(app)
if (!deployment) {
await this.create(app)
return
}

deployment.spec = await this.makeDeploymentSpec(
app,
deployment.spec.template.metadata.labels,
)
const region = await this.regionService.findByAppId(app.appid)
const appsV1Api = this.clusterService.makeAppsV1Api(region)
const namespace = GetApplicationNamespaceByAppId(app.appid)
const res = await appsV1Api.replaceNamespacedDeployment(
app.appid,
namespace,
deployment,
)

this.logger.log(`restart k8s deployment ${res.body?.metadata?.name}`)
}

async makeDeploymentSpec(
app: Application & {
region: Region
bundle: ApplicationBundle
configuration: ApplicationConfiguration
runtime: Runtime
},
private async makeDeploymentSpec(
app: ApplicationWithRelations,
labels: any,
): Promise<V1DeploymentSpec> {
// prepare params
@@ -386,21 +356,6 @@ export class InstanceService {
},
],
},
// preferred to schedule on bundle matched node
preferredDuringSchedulingIgnoredDuringExecution: [
{
weight: 10,
preference: {
matchExpressions: [
{
key: LABEL_KEY_BUNDLE,
operator: 'In',
values: [app.bundle.name],
},
],
},
},
],
}, // end of nodeAffinity {}
}, // end of affinity {}
}, // end of spec {}