From 4faa378d583b54e3c701cd7348c7045dfe1cb348 Mon Sep 17 00:00:00 2001 From: ewwu1 Date: Sun, 3 Nov 2024 23:24:08 +0800 Subject: [PATCH 1/7] added k8 files for deployment, service, stateful set and hpa; docker-compose now requires .env with dockerhub id for easy pushing --- docker-compose.yml | 5 ++++ .../collaboration-service-deployment.yaml | 24 ++++++++++++++++ k8/deployments/frontend-deployment.yaml | 19 +++++++++++++ .../matching-service-deployment.yaml | 28 +++++++++++++++++++ k8/deployments/metrics-server-deployment.yaml | 26 +++++++++++++++++ .../question-service-deployment.yaml | 28 +++++++++++++++++++ k8/deployments/redis-deployment.yaml | 19 +++++++++++++ k8/deployments/user-service-deployment.yaml | 22 +++++++++++++++ k8/hpas/collaboration-service-hpa.yaml | 18 ++++++++++++ k8/hpas/frontend-hpa.yaml | 18 ++++++++++++ k8/hpas/matching-service-hpa.yaml | 18 ++++++++++++ k8/hpas/question-service-hpa.yaml | 18 ++++++++++++ k8/hpas/user-service-hpa.yaml | 18 ++++++++++++ .../collaboration-service-service.yaml | 11 ++++++++ k8/services/frontend-service.yaml | 11 ++++++++ k8/services/matching-service-service.yaml | 11 ++++++++ k8/services/question-service-service.yaml | 11 ++++++++ k8/services/redis-service.yaml | 11 ++++++++ k8/services/user-service-service.yaml | 11 ++++++++ k8/statefulsets/question-db-statefulset.yaml | 20 +++++++++++++ k8/statefulsets/user-db-statefulset.yaml | 20 +++++++++++++ 21 files changed, 367 insertions(+) create mode 100644 k8/deployments/collaboration-service-deployment.yaml create mode 100644 k8/deployments/frontend-deployment.yaml create mode 100644 k8/deployments/matching-service-deployment.yaml create mode 100644 k8/deployments/metrics-server-deployment.yaml create mode 100644 k8/deployments/question-service-deployment.yaml create mode 100644 k8/deployments/redis-deployment.yaml create mode 100644 k8/deployments/user-service-deployment.yaml create mode 100644 k8/hpas/collaboration-service-hpa.yaml create mode 100644 k8/hpas/frontend-hpa.yaml create mode 100644 k8/hpas/matching-service-hpa.yaml create mode 100644 k8/hpas/question-service-hpa.yaml create mode 100644 k8/hpas/user-service-hpa.yaml create mode 100644 k8/services/collaboration-service-service.yaml create mode 100644 k8/services/frontend-service.yaml create mode 100644 k8/services/matching-service-service.yaml create mode 100644 k8/services/question-service-service.yaml create mode 100644 k8/services/redis-service.yaml create mode 100644 k8/services/user-service-service.yaml create mode 100644 k8/statefulsets/question-db-statefulset.yaml create mode 100644 k8/statefulsets/user-db-statefulset.yaml diff --git a/docker-compose.yml b/docker-compose.yml index 3c6b9cf67f..4ade7d2143 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ services: context: ./services/question-service args: - PORT=${QUESTION_PORT} + image: ${DOCKER_USERNAME}/question-service:latest ports: - "${QUESTION_PORT}:${QUESTION_PORT}" environment: @@ -26,6 +27,7 @@ services: context: ./services/user-service args: - PORT=${USER_PORT} + image: ${DOCKER_USERNAME}/user-service:latest ports: - "${USER_PORT}:${USER_PORT}" environment: @@ -51,6 +53,7 @@ services: - REDIS_PORT=6379 ports: - "${MATCHING_PORT}:${MATCHING_PORT}" + image: ${DOCKER_USERNAME}/matching-service:latest environment: PORT: ${MATCHING_PORT} REDIS_HOST: redis @@ -64,6 +67,7 @@ services: context: ./services/collaboration-service args: - PORT=${COLLAB_PORT} + image: ${DOCKER_USERNAME}/collaboration-service:latest ports: - "${COLLAB_PORT}:${COLLAB_PORT}" environment: @@ -77,6 +81,7 @@ services: frontend: build: ./frontend + image: ${DOCKER_USERNAME}/frontend:latest ports: - "3000:3000" volumes: diff --git a/k8/deployments/collaboration-service-deployment.yaml b/k8/deployments/collaboration-service-deployment.yaml new file mode 100644 index 0000000000..8bded202be --- /dev/null +++ b/k8/deployments/collaboration-service-deployment.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: collaboration-service +spec: + replicas: 1 + selector: + matchLabels: + app: collaboration-service + template: + metadata: + labels: + app: collaboration-service + spec: + containers: + - name: collaboration-service + image: wuemily/collaboration-service:latest + ports: + - containerPort: 8083 + env: + - name: PORT + value: "8083" + - name: ENV + value: "${ENV}" diff --git a/k8/deployments/frontend-deployment.yaml b/k8/deployments/frontend-deployment.yaml new file mode 100644 index 0000000000..ce09f21598 --- /dev/null +++ b/k8/deployments/frontend-deployment.yaml @@ -0,0 +1,19 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: frontend +spec: + replicas: 1 + selector: + matchLabels: + app: frontend + template: + metadata: + labels: + app: frontend + spec: + containers: + - name: frontend + image: wuemily/frontend:latest + ports: + - containerPort: 3000 diff --git a/k8/deployments/matching-service-deployment.yaml b/k8/deployments/matching-service-deployment.yaml new file mode 100644 index 0000000000..ef08a9f6a5 --- /dev/null +++ b/k8/deployments/matching-service-deployment.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: matching-service +spec: + replicas: 1 + selector: + matchLabels: + app: matching-service + template: + metadata: + labels: + app: matching-service + spec: + containers: + - name: matching-service + image: wuemily/matching-service:latest + ports: + - containerPort: 8082 + env: + - name: PORT + value: "8082" + - name: REDIS_HOST + value: "redis" + - name: REDIS_PORT + value: "6379" + - name: ENV + value: "${ENV}" diff --git a/k8/deployments/metrics-server-deployment.yaml b/k8/deployments/metrics-server-deployment.yaml new file mode 100644 index 0000000000..1956bd0d5e --- /dev/null +++ b/k8/deployments/metrics-server-deployment.yaml @@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metrics-server + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: metrics-server + template: + metadata: + labels: + k8s-app: metrics-server + spec: + containers: + - name: metrics-server + image: k8s.gcr.io/metrics-server/metrics-server:v0.7.2 # Check for the latest version + args: + - --secure-port=443 + - --metric-resolution=30s + - --cert-dir=/tmp + - --kubelet-insecure-tls + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + ports: + - containerPort: 443 diff --git a/k8/deployments/question-service-deployment.yaml b/k8/deployments/question-service-deployment.yaml new file mode 100644 index 0000000000..2388996ada --- /dev/null +++ b/k8/deployments/question-service-deployment.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: question-service +spec: + replicas: 1 + selector: + matchLabels: + app: question-service + template: + metadata: + labels: + app: question-service + spec: + containers: + - name: question-service + image: wuemily/question-service:latest + ports: + - containerPort: 8081 + env: + - name: PORT + value: "8081" + - name: DB_CLOUD_URI + value: "${QUESTION_DB_CLOUD_URI}" + - name: DB_LOCAL_URI + value: "${QUESTION_DB_LOCAL_URI}" + - name: ENV + value: "${ENV}" diff --git a/k8/deployments/redis-deployment.yaml b/k8/deployments/redis-deployment.yaml new file mode 100644 index 0000000000..a6a7f7d9f8 --- /dev/null +++ b/k8/deployments/redis-deployment.yaml @@ -0,0 +1,19 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis:alpine + ports: + - containerPort: 6379 diff --git a/k8/deployments/user-service-deployment.yaml b/k8/deployments/user-service-deployment.yaml new file mode 100644 index 0000000000..debabf61de --- /dev/null +++ b/k8/deployments/user-service-deployment.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: user-service +spec: + replicas: 1 + selector: + matchLabels: + app: user-service + template: + metadata: + labels: + app: user-service + spec: + containers: + - name: user-service + image: wuemily/user-service:latest + ports: + - containerPort: 8080 # Adjust based on your service's port + env: + - name: PORT + value: "8080" diff --git a/k8/hpas/collaboration-service-hpa.yaml b/k8/hpas/collaboration-service-hpa.yaml new file mode 100644 index 0000000000..17721e578f --- /dev/null +++ b/k8/hpas/collaboration-service-hpa.yaml @@ -0,0 +1,18 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: collaboration-service-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: collaboration-service + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 diff --git a/k8/hpas/frontend-hpa.yaml b/k8/hpas/frontend-hpa.yaml new file mode 100644 index 0000000000..dbf0a4098a --- /dev/null +++ b/k8/hpas/frontend-hpa.yaml @@ -0,0 +1,18 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: frontend-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: frontend + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 diff --git a/k8/hpas/matching-service-hpa.yaml b/k8/hpas/matching-service-hpa.yaml new file mode 100644 index 0000000000..4bda2d2f05 --- /dev/null +++ b/k8/hpas/matching-service-hpa.yaml @@ -0,0 +1,18 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: matching-service-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: matching-service + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 diff --git a/k8/hpas/question-service-hpa.yaml b/k8/hpas/question-service-hpa.yaml new file mode 100644 index 0000000000..7d72698ebc --- /dev/null +++ b/k8/hpas/question-service-hpa.yaml @@ -0,0 +1,18 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: question-service-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: question-service + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 diff --git a/k8/hpas/user-service-hpa.yaml b/k8/hpas/user-service-hpa.yaml new file mode 100644 index 0000000000..a6a892c3f4 --- /dev/null +++ b/k8/hpas/user-service-hpa.yaml @@ -0,0 +1,18 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: user-service-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: user-service + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 diff --git a/k8/services/collaboration-service-service.yaml b/k8/services/collaboration-service-service.yaml new file mode 100644 index 0000000000..1661365c98 --- /dev/null +++ b/k8/services/collaboration-service-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: collaboration-service +spec: + type: ClusterIP + ports: + - port: 8083 + targetPort: 8083 + selector: + app: collaboration-service diff --git a/k8/services/frontend-service.yaml b/k8/services/frontend-service.yaml new file mode 100644 index 0000000000..60b2061ec1 --- /dev/null +++ b/k8/services/frontend-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: frontend +spec: + type: LoadBalancer + ports: + - port: 3000 + targetPort: 3000 + selector: + app: frontend diff --git a/k8/services/matching-service-service.yaml b/k8/services/matching-service-service.yaml new file mode 100644 index 0000000000..13d2f22acf --- /dev/null +++ b/k8/services/matching-service-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: matching-service +spec: + type: ClusterIP + ports: + - port: 8082 + targetPort: 8082 + selector: + app: matching-service diff --git a/k8/services/question-service-service.yaml b/k8/services/question-service-service.yaml new file mode 100644 index 0000000000..a2ca99b200 --- /dev/null +++ b/k8/services/question-service-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: question-service +spec: + type: ClusterIP + ports: + - port: 8081 + targetPort: 8081 + selector: + app: question-service diff --git a/k8/services/redis-service.yaml b/k8/services/redis-service.yaml new file mode 100644 index 0000000000..c6587d8cac --- /dev/null +++ b/k8/services/redis-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis +spec: + ports: + - port: 6379 + targetPort: 6379 + selector: + app: redis + type: ClusterIP diff --git a/k8/services/user-service-service.yaml b/k8/services/user-service-service.yaml new file mode 100644 index 0000000000..61ccdb3e62 --- /dev/null +++ b/k8/services/user-service-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: user-service +spec: + type: ClusterIP # Use LoadBalancer for external access (with `minikube tunnel`) + ports: + - port: 8080 + targetPort: 8080 + selector: + app: user-service diff --git a/k8/statefulsets/question-db-statefulset.yaml b/k8/statefulsets/question-db-statefulset.yaml new file mode 100644 index 0000000000..9a08b4644b --- /dev/null +++ b/k8/statefulsets/question-db-statefulset.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: question-db +spec: + serviceName: "question-db" + replicas: 1 + selector: + matchLabels: + app: question-db + template: + metadata: + labels: + app: question-db + spec: + containers: + - name: question-db + image: mongodb/mongodb-atlas-local + ports: + - containerPort: 27017 # Default MongoDB port diff --git a/k8/statefulsets/user-db-statefulset.yaml b/k8/statefulsets/user-db-statefulset.yaml new file mode 100644 index 0000000000..34cac99638 --- /dev/null +++ b/k8/statefulsets/user-db-statefulset.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: user-db +spec: + serviceName: "user-db" + replicas: 1 + selector: + matchLabels: + app: user-db + template: + metadata: + labels: + app: user-db + spec: + containers: + - name: user-db + image: mongodb/mongodb-atlas-local + ports: + - containerPort: 27017 # Default MongoDB port From 27aeaa1b5fc3bd7771960971b74a5fcb1f4be111 Mon Sep 17 00:00:00 2001 From: ewwu1 Date: Mon, 4 Nov 2024 13:05:53 +0800 Subject: [PATCH 2/7] added resource limits to deployments --- k8/deployments/collaboration-service-deployment.yaml | 6 ++++++ k8/deployments/frontend-deployment.yaml | 5 +++++ k8/deployments/matching-service-deployment.yaml | 5 +++++ k8/deployments/question-service-deployment.yaml | 5 +++++ k8/deployments/user-service-deployment.yaml | 7 ++++++- 5 files changed, 27 insertions(+), 1 deletion(-) diff --git a/k8/deployments/collaboration-service-deployment.yaml b/k8/deployments/collaboration-service-deployment.yaml index 8bded202be..b37ea861e7 100644 --- a/k8/deployments/collaboration-service-deployment.yaml +++ b/k8/deployments/collaboration-service-deployment.yaml @@ -17,8 +17,14 @@ spec: image: wuemily/collaboration-service:latest ports: - containerPort: 8083 + resources: + limits: + cpu: 500m + requests: + cpu: 200m env: - name: PORT value: "8083" - name: ENV value: "${ENV}" + diff --git a/k8/deployments/frontend-deployment.yaml b/k8/deployments/frontend-deployment.yaml index ce09f21598..f4dc643045 100644 --- a/k8/deployments/frontend-deployment.yaml +++ b/k8/deployments/frontend-deployment.yaml @@ -17,3 +17,8 @@ spec: image: wuemily/frontend:latest ports: - containerPort: 3000 + resources: + limits: + cpu: 500m + requests: + cpu: 200m \ No newline at end of file diff --git a/k8/deployments/matching-service-deployment.yaml b/k8/deployments/matching-service-deployment.yaml index ef08a9f6a5..8ebbf008db 100644 --- a/k8/deployments/matching-service-deployment.yaml +++ b/k8/deployments/matching-service-deployment.yaml @@ -17,6 +17,11 @@ spec: image: wuemily/matching-service:latest ports: - containerPort: 8082 + resources: + limits: + cpu: 500m + requests: + cpu: 200m env: - name: PORT value: "8082" diff --git a/k8/deployments/question-service-deployment.yaml b/k8/deployments/question-service-deployment.yaml index 2388996ada..572b2a2c95 100644 --- a/k8/deployments/question-service-deployment.yaml +++ b/k8/deployments/question-service-deployment.yaml @@ -17,6 +17,11 @@ spec: image: wuemily/question-service:latest ports: - containerPort: 8081 + resources: + limits: + cpu: 500m + requests: + cpu: 200m env: - name: PORT value: "8081" diff --git a/k8/deployments/user-service-deployment.yaml b/k8/deployments/user-service-deployment.yaml index debabf61de..f14706b3d0 100644 --- a/k8/deployments/user-service-deployment.yaml +++ b/k8/deployments/user-service-deployment.yaml @@ -16,7 +16,12 @@ spec: - name: user-service image: wuemily/user-service:latest ports: - - containerPort: 8080 # Adjust based on your service's port + - containerPort: 8080 + resources: + limits: + cpu: 500m + requests: + cpu: 200m env: - name: PORT value: "8080" From 8e305aa1d115a02982fe429f86a4099bedaee1a1 Mon Sep 17 00:00:00 2001 From: ewwu1 Date: Tue, 5 Nov 2024 16:26:03 +0800 Subject: [PATCH 3/7] added app-config file, api-gateway .yaml files --- docker-compose.yml | 2 + k8/app-config.yaml | 17 ++ k8/components.yaml | 203 ++++++++++++++++++ k8/deployments/api-gateway-deplyoment.yaml | 22 ++ .../collaboration-service-deployment.yaml | 6 +- k8/deployments/frontend-deployment.yaml | 5 +- .../matching-service-deployment.yaml | 5 +- k8/deployments/metrics-server-deployment.yaml | 2 +- .../question-service-deployment.yaml | 9 +- k8/deployments/user-service-deployment.yaml | 3 + k8/services/api-gateway-service.yaml | 12 ++ k8/services/frontend-service.yaml | 9 +- 12 files changed, 278 insertions(+), 17 deletions(-) create mode 100644 k8/app-config.yaml create mode 100644 k8/components.yaml create mode 100644 k8/deployments/api-gateway-deplyoment.yaml create mode 100644 k8/services/api-gateway-service.yaml diff --git a/docker-compose.yml b/docker-compose.yml index 314003e748..b614ce08fc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,6 +33,7 @@ services: context: ./services/user-service args: - PORT=${USER_PORT} + image: ${DOCKER_USERNAME}/user-service:latest ports: - "${USER_PORT}:${USER_PORT}" environment: @@ -110,6 +111,7 @@ services: - MATCHING_URL=http://matching-service - QUESTION_URL=http://question-service - COLLAB_URL=http://collaboration-service + image: ${DOCKER_USERNAME}/api-gateway:latest ports: - "${GATEWAY_PORT}:${GATEWAY_PORT}" environment: diff --git a/k8/app-config.yaml b/k8/app-config.yaml new file mode 100644 index 0000000000..22685b26d4 --- /dev/null +++ b/k8/app-config.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: app-config +data: + QUESTION_PORT: "3001" + USER_PORT: "3002" + MATCHING_PORT: "8080" + COLLAB_PORT: "8081" + GATEWAY_PORT: "4000" + ENV: "PROD" + REACT_APP_QUESTION_API_URL: "http://question-service:3001" + REACT_APP_USER_API_URL: "http://user-service:3002" + REACT_APP_MATCHING_API_URL: "http://matching-service:8080" + REACT_APP_COLLAB_WS_URL: "ws://collaboration-service:8081" + REACT_APP_COLLAB_API_URL: "http://collaboration-service:8081" + REACT_APP_GATEWAY_URL: "http://api-gateway:4000/api" \ No newline at end of file diff --git a/k8/components.yaml b/k8/components.yaml new file mode 100644 index 0000000000..cd1ece8479 --- /dev/null +++ b/k8/components.yaml @@ -0,0 +1,203 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader +rules: +- apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server +rules: +- apiGroups: + - "" + resources: + - nodes/metrics + verbs: + - get +- apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + k8s-app: metrics-server + name: metrics-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: metrics-server + name: metrics-server:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + k8s-app: metrics-server +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: metrics-server + strategy: + rollingUpdate: + maxUnavailable: 0 + template: + metadata: + labels: + k8s-app: metrics-server + spec: + containers: + - args: + - --cert-dir=/tmp + - --secure-port=10250 + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --metric-resolution=15s + - --secure-port=443 + - --kubelet-insecure-tls + image: registry.k8s.io/metrics-server/metrics-server:v0.7.2 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + name: metrics-server + ports: + - containerPort: 10250 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /tmp + name: tmp-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: metrics-server + volumes: + - emptyDir: {} + name: tmp-dir +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + labels: + k8s-app: metrics-server + name: v1beta1.metrics.k8s.io +spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: metrics-server + namespace: kube-system + version: v1beta1 + versionPriority: 100 diff --git a/k8/deployments/api-gateway-deplyoment.yaml b/k8/deployments/api-gateway-deplyoment.yaml new file mode 100644 index 0000000000..f5c0bff934 --- /dev/null +++ b/k8/deployments/api-gateway-deplyoment.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: api-gateway +spec: + replicas: 1 + selector: + matchLabels: + app: api-gateway + template: + metadata: + labels: + app: api-gateway + spec: + containers: + - name: api-gateway + image: wuemily/api-gateway:latest + ports: + - containerPort: 4000 + envFrom: + - configMapRef: + name: app-config \ No newline at end of file diff --git a/k8/deployments/collaboration-service-deployment.yaml b/k8/deployments/collaboration-service-deployment.yaml index b37ea861e7..fa4d4088ed 100644 --- a/k8/deployments/collaboration-service-deployment.yaml +++ b/k8/deployments/collaboration-service-deployment.yaml @@ -25,6 +25,6 @@ spec: env: - name: PORT value: "8083" - - name: ENV - value: "${ENV}" - + envFrom: + - configMapRef: + name: app-config diff --git a/k8/deployments/frontend-deployment.yaml b/k8/deployments/frontend-deployment.yaml index f4dc643045..b6ba19bab7 100644 --- a/k8/deployments/frontend-deployment.yaml +++ b/k8/deployments/frontend-deployment.yaml @@ -21,4 +21,7 @@ spec: limits: cpu: 500m requests: - cpu: 200m \ No newline at end of file + cpu: 200m + envFrom: + - configMapRef: + name: app-config \ No newline at end of file diff --git a/k8/deployments/matching-service-deployment.yaml b/k8/deployments/matching-service-deployment.yaml index 8ebbf008db..14d9e547f5 100644 --- a/k8/deployments/matching-service-deployment.yaml +++ b/k8/deployments/matching-service-deployment.yaml @@ -29,5 +29,6 @@ spec: value: "redis" - name: REDIS_PORT value: "6379" - - name: ENV - value: "${ENV}" + envFrom: + - configMapRef: + name: app-config diff --git a/k8/deployments/metrics-server-deployment.yaml b/k8/deployments/metrics-server-deployment.yaml index 1956bd0d5e..4c2e8ec901 100644 --- a/k8/deployments/metrics-server-deployment.yaml +++ b/k8/deployments/metrics-server-deployment.yaml @@ -15,7 +15,7 @@ spec: spec: containers: - name: metrics-server - image: k8s.gcr.io/metrics-server/metrics-server:v0.7.2 # Check for the latest version + image: k8s.gcr.io/metrics-server/metrics-server:v0.7.2 args: - --secure-port=443 - --metric-resolution=30s diff --git a/k8/deployments/question-service-deployment.yaml b/k8/deployments/question-service-deployment.yaml index 572b2a2c95..1678687d5d 100644 --- a/k8/deployments/question-service-deployment.yaml +++ b/k8/deployments/question-service-deployment.yaml @@ -25,9 +25,6 @@ spec: env: - name: PORT value: "8081" - - name: DB_CLOUD_URI - value: "${QUESTION_DB_CLOUD_URI}" - - name: DB_LOCAL_URI - value: "${QUESTION_DB_LOCAL_URI}" - - name: ENV - value: "${ENV}" + envFrom: + - configMapRef: + name: app-config \ No newline at end of file diff --git a/k8/deployments/user-service-deployment.yaml b/k8/deployments/user-service-deployment.yaml index f14706b3d0..eb2bdc93de 100644 --- a/k8/deployments/user-service-deployment.yaml +++ b/k8/deployments/user-service-deployment.yaml @@ -25,3 +25,6 @@ spec: env: - name: PORT value: "8080" + envFrom: + - configMapRef: + name: app-config diff --git a/k8/services/api-gateway-service.yaml b/k8/services/api-gateway-service.yaml new file mode 100644 index 0000000000..5e496be558 --- /dev/null +++ b/k8/services/api-gateway-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: api-gateway +spec: + selector: + app: api-gateway-deployment + ports: + - protocol: TCP + port: 4000 + targetPort: 4000 + type: LoadBalancer \ No newline at end of file diff --git a/k8/services/frontend-service.yaml b/k8/services/frontend-service.yaml index 60b2061ec1..43990c95a9 100644 --- a/k8/services/frontend-service.yaml +++ b/k8/services/frontend-service.yaml @@ -3,9 +3,10 @@ kind: Service metadata: name: frontend spec: - type: LoadBalancer + selector: + app: frontend-service ports: - - port: 3000 + - protocol: TCP + port: 3000 targetPort: 3000 - selector: - app: frontend + type: LoadBalancer From d242dd8135c5859a18e8a7d1b7a0d6666084a7c1 Mon Sep 17 00:00:00 2001 From: ewwu1 Date: Tue, 5 Nov 2024 16:30:44 +0800 Subject: [PATCH 4/7] delete obsolete file --- k8/deployments/metrics-server-deployment.yaml | 26 ------------------- 1 file changed, 26 deletions(-) delete mode 100644 k8/deployments/metrics-server-deployment.yaml diff --git a/k8/deployments/metrics-server-deployment.yaml b/k8/deployments/metrics-server-deployment.yaml deleted file mode 100644 index 4c2e8ec901..0000000000 --- a/k8/deployments/metrics-server-deployment.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: metrics-server - namespace: kube-system -spec: - replicas: 1 - selector: - matchLabels: - k8s-app: metrics-server - template: - metadata: - labels: - k8s-app: metrics-server - spec: - containers: - - name: metrics-server - image: k8s.gcr.io/metrics-server/metrics-server:v0.7.2 - args: - - --secure-port=443 - - --metric-resolution=30s - - --cert-dir=/tmp - - --kubelet-insecure-tls - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - ports: - - containerPort: 443 From 50b199ca65e57a887fedabeb69c3381ff26ea11d Mon Sep 17 00:00:00 2001 From: ewwu1 Date: Thu, 7 Nov 2024 21:52:39 +0800 Subject: [PATCH 5/7] added history service kubernetes implementations; updated config file (api-gateway deployment bug fix) --- docker-compose.yml | 12 ++++---- k8/app-config.yaml | 2 ++ .../collaboration-service-deployment.yaml | 4 +-- .../history-service-deployment.yaml | 30 +++++++++++++++++++ .../matching-service-deployment.yaml | 4 +-- .../question-service-deployment.yaml | 4 +-- k8/deployments/user-service-deployment.yaml | 4 +-- k8/hpas/history-service-hpa.yaml | 18 +++++++++++ k8/services/api-gateway-service.yaml | 2 +- .../collaboration-service-service.yaml | 4 +-- k8/services/history-service-service.yaml | 11 +++++++ k8/services/matching-service-service.yaml | 4 +-- k8/services/question-service-service.yaml | 4 +-- k8/services/user-service-service.yaml | 6 ++-- 14 files changed, 86 insertions(+), 23 deletions(-) create mode 100644 k8/deployments/history-service-deployment.yaml create mode 100644 k8/hpas/history-service-hpa.yaml create mode 100644 k8/services/history-service-service.yaml diff --git a/docker-compose.yml b/docker-compose.yml index b614ce08fc..d9fb887eea 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -144,16 +144,12 @@ services: networks: - app-network - -networks: - app-network: - driver: bridge - history-service: build: context: ./services/history-service args: - PORT=${HISTORY_PORT} + image: ${DOCKER_USERNAME}/history-service:latest ports: - "${HISTORY_PORT}:${HISTORY_PORT}" environment: @@ -169,3 +165,9 @@ networks: ports: - "27020:27020" +networks: + app-network: + driver: bridge + + + diff --git a/k8/app-config.yaml b/k8/app-config.yaml index 22685b26d4..24ac2d0c5f 100644 --- a/k8/app-config.yaml +++ b/k8/app-config.yaml @@ -8,7 +8,9 @@ data: MATCHING_PORT: "8080" COLLAB_PORT: "8081" GATEWAY_PORT: "4000" + HISTORY_PORT: "8082" ENV: "PROD" + MATCHING_URL: "https://matching-service" REACT_APP_QUESTION_API_URL: "http://question-service:3001" REACT_APP_USER_API_URL: "http://user-service:3002" REACT_APP_MATCHING_API_URL: "http://matching-service:8080" diff --git a/k8/deployments/collaboration-service-deployment.yaml b/k8/deployments/collaboration-service-deployment.yaml index fa4d4088ed..3054aba7ad 100644 --- a/k8/deployments/collaboration-service-deployment.yaml +++ b/k8/deployments/collaboration-service-deployment.yaml @@ -16,7 +16,7 @@ spec: - name: collaboration-service image: wuemily/collaboration-service:latest ports: - - containerPort: 8083 + - containerPort: 8081 resources: limits: cpu: 500m @@ -24,7 +24,7 @@ spec: cpu: 200m env: - name: PORT - value: "8083" + value: "8081" envFrom: - configMapRef: name: app-config diff --git a/k8/deployments/history-service-deployment.yaml b/k8/deployments/history-service-deployment.yaml new file mode 100644 index 0000000000..2123741145 --- /dev/null +++ b/k8/deployments/history-service-deployment.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: history-service +spec: + replicas: 1 + selector: + matchLabels: + app: history-service + template: + metadata: + labels: + app: history-service + spec: + containers: + - name: history-service + image: wuemily/history-service:latest + ports: + - containerPort: 8082 + resources: + limits: + cpu: 500m + requests: + cpu: 200m + env: + - name: PORT + value: "8082" + envFrom: + - configMapRef: + name: app-config \ No newline at end of file diff --git a/k8/deployments/matching-service-deployment.yaml b/k8/deployments/matching-service-deployment.yaml index 14d9e547f5..e75e240f27 100644 --- a/k8/deployments/matching-service-deployment.yaml +++ b/k8/deployments/matching-service-deployment.yaml @@ -16,7 +16,7 @@ spec: - name: matching-service image: wuemily/matching-service:latest ports: - - containerPort: 8082 + - containerPort: 8080 resources: limits: cpu: 500m @@ -24,7 +24,7 @@ spec: cpu: 200m env: - name: PORT - value: "8082" + value: "8080" - name: REDIS_HOST value: "redis" - name: REDIS_PORT diff --git a/k8/deployments/question-service-deployment.yaml b/k8/deployments/question-service-deployment.yaml index 1678687d5d..f6b4b3da71 100644 --- a/k8/deployments/question-service-deployment.yaml +++ b/k8/deployments/question-service-deployment.yaml @@ -16,7 +16,7 @@ spec: - name: question-service image: wuemily/question-service:latest ports: - - containerPort: 8081 + - containerPort: 3001 resources: limits: cpu: 500m @@ -24,7 +24,7 @@ spec: cpu: 200m env: - name: PORT - value: "8081" + value: "3001" envFrom: - configMapRef: name: app-config \ No newline at end of file diff --git a/k8/deployments/user-service-deployment.yaml b/k8/deployments/user-service-deployment.yaml index eb2bdc93de..ee06474d75 100644 --- a/k8/deployments/user-service-deployment.yaml +++ b/k8/deployments/user-service-deployment.yaml @@ -16,7 +16,7 @@ spec: - name: user-service image: wuemily/user-service:latest ports: - - containerPort: 8080 + - containerPort: 3002 resources: limits: cpu: 500m @@ -24,7 +24,7 @@ spec: cpu: 200m env: - name: PORT - value: "8080" + value: "3002" envFrom: - configMapRef: name: app-config diff --git a/k8/hpas/history-service-hpa.yaml b/k8/hpas/history-service-hpa.yaml new file mode 100644 index 0000000000..b0517f7dc1 --- /dev/null +++ b/k8/hpas/history-service-hpa.yaml @@ -0,0 +1,18 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: history-service-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: history-service + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 diff --git a/k8/services/api-gateway-service.yaml b/k8/services/api-gateway-service.yaml index 5e496be558..cfb9b7746b 100644 --- a/k8/services/api-gateway-service.yaml +++ b/k8/services/api-gateway-service.yaml @@ -4,7 +4,7 @@ metadata: name: api-gateway spec: selector: - app: api-gateway-deployment + app: api-gateway ports: - protocol: TCP port: 4000 diff --git a/k8/services/collaboration-service-service.yaml b/k8/services/collaboration-service-service.yaml index 1661365c98..91339b0dfb 100644 --- a/k8/services/collaboration-service-service.yaml +++ b/k8/services/collaboration-service-service.yaml @@ -5,7 +5,7 @@ metadata: spec: type: ClusterIP ports: - - port: 8083 - targetPort: 8083 + - port: 8081 + targetPort: 8081 selector: app: collaboration-service diff --git a/k8/services/history-service-service.yaml b/k8/services/history-service-service.yaml new file mode 100644 index 0000000000..20468b23e9 --- /dev/null +++ b/k8/services/history-service-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: history-service +spec: + type: ClusterIP + ports: + - port: 8080 + targetPort: 8080 + selector: + app: history-service diff --git a/k8/services/matching-service-service.yaml b/k8/services/matching-service-service.yaml index 13d2f22acf..d9b42cc4c3 100644 --- a/k8/services/matching-service-service.yaml +++ b/k8/services/matching-service-service.yaml @@ -5,7 +5,7 @@ metadata: spec: type: ClusterIP ports: - - port: 8082 - targetPort: 8082 + - port: 8080 + targetPort: 8080 selector: app: matching-service diff --git a/k8/services/question-service-service.yaml b/k8/services/question-service-service.yaml index a2ca99b200..7276b9f975 100644 --- a/k8/services/question-service-service.yaml +++ b/k8/services/question-service-service.yaml @@ -5,7 +5,7 @@ metadata: spec: type: ClusterIP ports: - - port: 8081 - targetPort: 8081 + - port: 3001 + targetPort: 3001 selector: app: question-service diff --git a/k8/services/user-service-service.yaml b/k8/services/user-service-service.yaml index 61ccdb3e62..b057918e2b 100644 --- a/k8/services/user-service-service.yaml +++ b/k8/services/user-service-service.yaml @@ -3,9 +3,9 @@ kind: Service metadata: name: user-service spec: - type: ClusterIP # Use LoadBalancer for external access (with `minikube tunnel`) + type: ClusterIP ports: - - port: 8080 - targetPort: 8080 + - port: 3002 + targetPort: 3002 selector: app: user-service From 418b095c2d7ff4d55d209257c98ed9a616f780dd Mon Sep 17 00:00:00 2001 From: ewwu1 Date: Tue, 12 Nov 2024 08:37:09 +0800 Subject: [PATCH 6/7] added hpa set up instructions file --- HPA-SETUP.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 HPA-SETUP.md diff --git a/HPA-SETUP.md b/HPA-SETUP.md new file mode 100644 index 0000000000..583edee02f --- /dev/null +++ b/HPA-SETUP.md @@ -0,0 +1,46 @@ +N8: Scalability +## Kubernetes and Horizontal Pod Auto-scaler (HPA) +The application uses Horizontal Pod Autoscalers (HPAs) to manage scalability based on CPU utilization. The HPAs automatically adjust the number of pods for each service based on the load. + +### Configuration Infomation +Each microservice has its own HPA configuration file located in the `./k8` directory. +- The HPA settings by default are: + - Min Replicas: The minimum number of pod replicas (e.g., 1). + - Max Replicas: The maximum number of pod replicas (e.g., 10). + - Target CPU Utilization: The desired average CPU utilization (e.g., 50%). +- The HPA configuration files for each service are: + - Frontend: `k8/hpas/frontend-hpa.yaml` + - Collaboration Service: `k8/hpas/collaboration-service-hpa.yaml` + - Matching Service: `k8/hpas/matching-service-hpa.yaml` + - Question Service: `k8/hpas/question-service-hpa.yaml` + - User Service: `k8/hpas/user-service-hpa.yaml` + - History Service: `k8/hpas/history-service-hpa.yaml` + +### Setup Instructions +- **Step 1: Setting up Docker Images.** +Ensure repository is cloned, and `.env` file is added. **Note:** `./k8/app-config.yaml` should hold the same ports and urls variables as the .env file. If there are discrepencies, update them to match. If you would like to change the ports for the services, you need to update ports in the app-config, and the relevant deployment and service `.yaml` files. + - If you want to use your own Docker Hub repository (where you will need to then push to), specify your Docker Hub ID in the .env **and** change image names in `.yaml`s in `./k8/deployments` for the microservices (so excluding redis and metrics-server deployments) to replace `wuemily` with your docker hub ID. + - After building the Docker images (e.g. with `docker compose up`), push them with: +`docker push dockerhubid/question-service:latest` +`docker push dockerhubid/user-service:latest` +`docker push dockerhubid/matching-service:latest` +`docker push dockerhubid/collaboration-service:latest` +`docker push dockerhubid/frontend:latest` +`docker push dockerhubid/api-gateway:latest` +`docker push dockerhubid/history-service:latest` + - Alternatively, you can also use existing public reposity by setting the Docker Hub ID to wuemily; no need to push images or make changes in `.yaml` files. +- **Step 2: Setting up Minikube and Kubernetes.** +This assumes some basic understanding of Minikube and Kubernetes and they are both installed. + - Start Minikube with: ``minikube start --driver=docker`` + - To apply the HPA configurations to your Kubernetes cluster, use the following commands (from root folder): + `kubectl apply -f ./k8` + `kubectl apply -f ./k8/deployments` + `kubectl apply -f ./k8/services` + `kubectl apply -f ./k8/statefulsets` + `kubectl apply -f ./k8/hpas` + - If load balancers appear to be not working, it may be because of the local hosting. Make sure `minikube tunnel` is running. +- **Step 3: Monitoring** +You can monitor the scaling behavior using `kubectl` commands. +- `kubectl get hpa`displays the current status of all HPAs in the cluster, including the current and desired number of replicas based on CPU usage. + - If CPU usage is `unknown`, check if Minikube metrics server addon is enabled with `minikube adddons list`, if it isn't, enable with `minikube addons enable metrics-server`. +- Can also observe pods and their status with `kubectl get pods` and `minikube dashboard`, depending on your preference. \ No newline at end of file From 27d4f548d353d9e870143f9cf4de1a7e2806c751 Mon Sep 17 00:00:00 2001 From: ewwu1 Date: Tue, 12 Nov 2024 21:24:48 +0800 Subject: [PATCH 7/7] stateful sets for collab and history db --- .../collaboration-db-statefulset.yaml | 20 +++++++++++++++++++ k8/statefulsets/history-db-statefulset.yaml | 20 +++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 k8/statefulsets/collaboration-db-statefulset.yaml create mode 100644 k8/statefulsets/history-db-statefulset.yaml diff --git a/k8/statefulsets/collaboration-db-statefulset.yaml b/k8/statefulsets/collaboration-db-statefulset.yaml new file mode 100644 index 0000000000..2925bb640b --- /dev/null +++ b/k8/statefulsets/collaboration-db-statefulset.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: collaboration-db +spec: + serviceName: "collaboration-db" + replicas: 1 + selector: + matchLabels: + app: collaboration-db + template: + metadata: + labels: + app: collaboration-db + spec: + containers: + - name: collaboration-db + image: mongodb/mongodb-atlas-local + ports: + - containerPort: 27017 # Default MongoDB port diff --git a/k8/statefulsets/history-db-statefulset.yaml b/k8/statefulsets/history-db-statefulset.yaml new file mode 100644 index 0000000000..64d6bb3c61 --- /dev/null +++ b/k8/statefulsets/history-db-statefulset.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: history-db +spec: + serviceName: "history-db" + replicas: 1 + selector: + matchLabels: + app: history-db + template: + metadata: + labels: + app: history-db + spec: + containers: + - name: history-db + image: mongodb/mongodb-atlas-local + ports: + - containerPort: 27017 # Default MongoDB port