Merge pull request #382 from openinfradev/byok1

feature. add endpoint secret creation step when importing
openinfradev · Aug 26, 2024 · c76ec47 · c76ec47
2 parents 8efefdf + 8cf7b38
commit c76ec47
Show file tree

Hide file tree

Showing 4 changed files with 266 additions and 10 deletions.
diff --git a/deploy_apps/tks-primary-cluster.yaml b/deploy_apps/tks-primary-cluster.yaml
@@ -494,7 +494,7 @@ spec:
           fi
         fi
 
-        LOKI_USER_SERVICE=$(kubectl get secret -n ${primary_cluster} tks-endpoint-secret -o jsonpath='{.data.LOKI_USER}'| base64 -d )
+        LOKI_USER_SERVICE=$(kubectl get secret -n ${primary_cluster} tks-endpoint-secret -o jsonpath='{.data.loki_user}'| base64 -d )
 
         if [[ "$LOKI_USER_SERVICE" == "" ]]; then
           while [ -z $(kubectl --kubeconfig=kubeconfig get svc -n lma loki-user-loki-distributed-gateway -o jsonpath="{.status.loadBalancer.ingress[*].hostname}") ]

diff --git a/tks-cli/tks-cli.yaml b/tks-cli/tks-cli.yaml
@@ -12,7 +12,7 @@ spec:
   - name: login-tks-api
     container:
       name: login-tks-api
-      image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.2.1
+      image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.2.2
       envFrom:
       - secretRef:
           name: "tks-api-secret"       
@@ -31,7 +31,7 @@ spec:
       - name: description
     container:
       name: create-organization
-      image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.2.1
+      image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.2.2
       envFrom:
       - secretRef:
           name: "tks-api-secret"       
@@ -95,7 +95,7 @@ spec:
       - name: cluster_domains
     container:
       name: create-usercluster
-      image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.2.1
+      image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.2.2
       envFrom:
       - secretRef:
           name: "tks-api-secret"       
@@ -166,7 +166,7 @@ spec:
       - name: organization_id
     container:
       name: install-usercluster
-      image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.2.1
+      image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.2.2
       envFrom:
       - secretRef:
           name: "tks-api-secret"       
@@ -212,14 +212,81 @@ spec:
           default: "Something wrong"
           path: /mnt/out/cluster_id.txt
 
+  - name: import-usercluster
+    inputs:
+      parameters:
+      - name: cluster_name
+      - name: stack_template_id
+      - name: organization_id
+      - name: creator
+      - name: description
+      - name: policy_ids
+      - name: cluster_domains
+      - name: kubeconfig_string
+    container:
+      name: import-usercluster
+      image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.2.2
+      imagePullPolicy: Always
+      envFrom:
+      - secretRef:
+          name: "tks-api-secret"       
+      command:
+        - /bin/bash
+        - '-exc'
+        - |
+          tks login {{workflow.parameters.tks_api_url}} --organization-id ${ORGANIZATION_ID} --account-id ${ACCOUNT_ID} --password ${PASSWORD}
+
+          CL_NAME="{{inputs.parameters.cluster_name}}"
+
+          echo "* Import $CL_NAME cluster"
+          tks cluster import ${CL_NAME} \
+            --stack-template-id "{{inputs.parameters.stack_template_id}}" \
+            --organization-id "{{inputs.parameters.organization_id}}" \
+            --description "{{inputs.parameters.description}}" \
+            --cluster-type "USER" \
+            --kubeconfig-string "{{inputs.parameters.kubeconfig_string}}" \
+            --domains "{{inputs.parameters.cluster_domains}}" \
+            --policy-ids "{{inputs.parameters.policy_ids}}" 
+
+          threshold=720
+          for i in $(seq 1 $threshold)
+          do
+            CL_STATUS=$(tks cluster list "{{inputs.parameters.organization_id}}" | grep -w $CL_NAME | awk '{ print $4 }')
+            if [ "$CL_STATUS" = "RUNNING" ]; then
+              break
+            elif [ "$CL_STATUS" = "ERROR" ]; then
+              exit 1
+            fi
+
+            if [ "$i" -ge "$threshold" ]; then
+              echo "Timed out waiting for user-cluster to be ready."
+              exit 1
+            fi
+            sleep 5
+          done
+
+          tks cluster list "{{inputs.parameters.organization_id}}" | grep -w $CL_NAME | awk '{print $3}' | tee /mnt/out/cluster_id.txt
+      volumeMounts:
+        - name: out
+          mountPath: /mnt/out
+    volumes:
+      - name: out
+        emptyDir: { }
+    outputs:
+      parameters:
+      - name: cluster-id
+        valueFrom:
+          default: "Something wrong"
+          path: /mnt/out/cluster_id.txt
+
   - name: delete-usercluster
     inputs:
       parameters:
       - name: organization_id
       - name: cluster_id
     container:
       name: delete-usercluster
-      image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.2.1
+      image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.2.2
       envFrom:
       - secretRef:
           name: "tks-api-secret"       
@@ -262,7 +329,7 @@ spec:
       - name: description
     container:
       name: create-appgroup
-      image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.2.1
+      image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.2.2
       envFrom:
       - secretRef:
           name: "tks-api-secret"       
@@ -312,7 +379,7 @@ spec:
       - name: appgroup_id
     container:
       name: delete-appgroup
-      image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.2.1
+      image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.2.2
       envFrom:
       - secretRef:
           name: "tks-api-secret"       
@@ -352,7 +419,7 @@ spec:
       - name: name
     container:
       name: get-appgroup-id
-      image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.2.1
+      image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.2.2
       envFrom:
       - secretRef:
           name: "tks-api-secret"       

diff --git a/tks-cluster/import-usercluster-wftpl.yaml b/tks-cluster/import-usercluster-wftpl.yaml
@@ -15,6 +15,10 @@ spec:
       value: "P0010010a"
     - name: cluster_id
       value: "C011b88fa"
+    - name: site_name
+      value: "{{workflow.parameters.cluster_id}}"
+    - name: app_prefix
+      value: "{{workflow.parameters.cluster_id}}"
     - name: kubeconfig
       value: "KUBECONFIG_string"
     - name: git_account
@@ -25,6 +29,10 @@ spec:
       value: "NULL"
     - name: base_repo_branch
       value: "main"
+    - name: policy_ids
+      value: ""
+    - name: revision
+      value: "main"
 
   volumes:
   - name: kubeconfig-adm
@@ -80,6 +88,15 @@ spec:
               value: "{{ workflow.parameters.base_repo_branch }}"
         when: "{{steps.tks-get-cluster-info.outputs.parameters.cluster_type}} != ADMIN"
 
+    - - name: tks-create-config-secret
+        template: create-endpoint-secret
+        arguments:
+          parameters:
+            - name: cluster_domains
+              value: "{{steps.tks-get-cluster-info.outputs.parameters.cluster_domains}}"
+            - name: cluster_id
+              value: "{{ workflow.parameters.cluster_id }}"
+
     - - name: init-cluster-for-tks
         template: init-cluster-for-tks
         arguments:
@@ -95,6 +112,9 @@ spec:
             - name: contract_id
               value: "{{ workflow.parameters.contract_id }}"
 
+    - - name: suspend
+        template: suspend
+
     - - name: create-default-rbac-resources
         templateRef:
           name: create-tks-usercluster
@@ -104,10 +124,24 @@ spec:
             - name: cluster_id
               value: "{{ workflow.parameters.cluster_id }}"
 
+    - - name: install-policy-management
+        templateRef:
+          name: tks-policy
+          template: deploy
+        arguments:
+          parameters:
+            - name: cluster_id
+              value: "{{ workflow.parameters.cluster_id }}"
+            - name: contract_id
+              value: "{{ workflow.parameters.contract_id }}"
+            - name: policy_ids
+              value: "{{ workflow.parameters.policy_ids }}"
 
   #######################
   # Template Definition #
   #######################
+  - name: suspend
+    suspend: {}
 
   - name: import-cluster
     inputs:
@@ -160,14 +194,28 @@ spec:
         - name: cluster_id
     container:
       name: cluster-init
-      image: harbor.taco-cat.xyz/tks/python_kubectl_argo:v1.1.0
+      image: harbor.taco-cat.xyz/tks/tks-cluster-init:v1.0.0
       command:
         - /bin/bash
         - '-exc'
         - |
           cp /kube/value kubeconfig_adm
           export KUBECONFIG=kubeconfig_adm
 
+          ####### add tks info. on namespace ########
+          kubectl label ns ${CLUSTER_ID} tks.io/organization=${CONTRACT_ID}
+          if [ $(kubectl get ns -l tks.io/organization=${CONTRACT_ID} --ignore-not-found=true | grep -v NAME | awk '{print $1}' | wc -l ) -le 1 ]; then
+            kubectl label ns ${CLUSTER_ID} tks.io/policy=${CLUSTER_ID}
+          else
+            POLICY_NS=$(kubectl get ns $(kubectl get ns -l tks.io/organization=${CONTRACT_ID} --ignore-not-found=true | grep -v NAME | awk '{print $1}' | head -n 1 )  --ignore-not-found=true -o jsonpath='{.metadata.labels.tks\.io\/policy}' )
+            if [ -z "$POLICY_NS" ]; then
+              kubectl label ns ${CLUSTER_ID} tks.io/policy=${CLUSTER_ID}
+            else
+              kubectl label ns ${CLUSTER_ID} tks.io/policy=${POLICY_NS}
+            fi
+          fi
+          ###########################################
+
           KUBECONFIG_WORKLOAD=$(kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-tks-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
           cat <<< "$KUBECONFIG_WORKLOAD" > kubeconfig_workload
 
@@ -212,3 +260,48 @@ spec:
       - name: CLUSTER_ID
         value: "{{ inputs.parameters.cluster_id }}"
 
+
+  - name: create-endpoint-secret
+    inputs:
+      parameters:
+        - name: cluster_domains
+        - name: cluster_id
+    container:
+      name: create-namespace
+      image: harbor.taco-cat.xyz/tks/hyperkube:v1.18.6
+      command:
+        - /bin/bash
+        - '-c'
+        - |
+          CLUSTER_ID={{inputs.parameters.cluster_id}}
+
+          GRAFANA_URL=$(echo $CLUSTER_DOMAINS | jq -r '. | map(select(.domainType | contains("grafana"))|.url)'[])
+          LOKI_URL=$(echo $CLUSTER_DOMAINS | jq -r '. | map(select(.domainType | contains("loki"))|.url)'[])
+          MINIO_URL=$(echo $CLUSTER_DOMAINS | jq -r '. | map(select(.domainType | contains("minio"))|.url)'[])
+          PROMETHEUS_URL=$(echo $CLUSTER_DOMAINS | jq -r '. | map(select(.domainType | contains("prometheus"))|.url)'[])
+          THANOS_URL=$(echo $CLUSTER_DOMAINS | jq -r '. | map(select(.domainType | contains("thanos"))|.url)'[])
+          KIALI_URL=$(echo $CLUSTER_DOMAINS | jq -r '. | map(select(.domainType | contains("kiali"))|.url)'[])
+          JAEGER_URL=$(echo $CLUSTER_DOMAINS | jq -r '. | map(select(.domainType | contains("jaeger"))|.url)'[])
+
+          cat <<EOF > tks-endpoint-secret.yaml
+          ---
+          apiVersion: v1
+          kind: Secret
+          metadata:
+            name: tks-endpoint-secret
+            namespace: ${CLUSTER_ID}
+          data:
+            grafana: $(echo ${GRAFANA_URL} | base64)  # 30001
+            loki: $(echo ${LOKI_URL} | base64)   # 30002
+            minio: $(echo ${MINIO_URL} | base64)   # 30003
+            prometheus: $(echo ${PROMETHEUS_URL} | base64)    # 30004
+            thanos: $(echo ${THANOS_URL} | base64)  # 30005 (queryfrontend만 합시다...)
+            kiali: $(echo ${KIALI_URL} | base64)    # 30011
+            jaeger: $(echo ${JAEGER_URL} | base64)    # 30012
+          EOF
+          kubectl apply -f tks-endpoint-secret.yaml
+      env:
+      - name: CLUSTER_DOMAINS
+        value: "{{inputs.parameters.cluster_domains}}"
+
+    activeDeadlineSeconds: 30