From 2eb121fc22a5f7375e84816cbf8a74abde99be2d Mon Sep 17 00:00:00 2001 From: Damian Avila Date: Fri, 10 Sep 2021 17:24:11 -0300 Subject: [PATCH 1/7] Add a new required stateStore property if kops is enabled We currently manage EKS and kops clusters on AWS land. With this commit we are adding support and validation (by jsonschema) for the kops-based cluster that requires the state store to properly get the state of the cluster and succeed in the next step at the time to get export the kubeconfig. Notice I am using the if and then keywords [1] to require the stateStore property only if the kops value is configured. The encrypted key file referenced in the condfig file will be provided in a later commit. [1] https://json-schema.org/understanding-json-schema/reference/conditionals.html#if-then-else --- config/hubs/farallon.cluster.yaml | 10 +++++++--- config/hubs/schema.yaml | 12 ++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/config/hubs/farallon.cluster.yaml b/config/hubs/farallon.cluster.yaml index 2c13dfc7d..bbb94dd6a 100644 --- a/config/hubs/farallon.cluster.yaml +++ b/config/hubs/farallon.cluster.yaml @@ -1,7 +1,11 @@ name: farallon -provider: kubeconfig -kubeconfig: - file: secrets/farallon.yaml +provider: aws +aws: + key: secrets/farallon.json + clusterType: kops + clusterName: farallonhub.k8s.local + region: us-east-2 + stateStore: s3://2i2c-farallon-kops-state hubs: - name: staging domain: staging.farallon.2i2c.cloud diff --git a/config/hubs/schema.yaml b/config/hubs/schema.yaml index edbeb9b64..7b4620b2b 100644 --- a/config/hubs/schema.yaml +++ b/config/hubs/schema.yaml @@ -102,11 +102,23 @@ properties: type: string description: | The AWS region the cluster is in. + stateStore: + type: string + description: | + A dedicated S3 bucket for kops to use in order to store the state + (and the representation) of your of your kops cluster. required: - key - clusterType - clusterName - region + if: + properties: + clusterType: + const: kops + then: + required: + - stateStore hubs: type: array description: | From 28bb1f152665fe23a1804ef156584c58827ce81e Mon Sep 17 00:00:00 2001 From: Damian Avila Date: Fri, 10 Sep 2021 17:32:22 -0300 Subject: [PATCH 2/7] Add the encrypted file containing deployer credentials We identified a 2i2c-engineers groups in Farallon AWS and we have created a deployer user under that group. We used common awscli commands to perform this task and retrieve the credentials. More details in [1]. Finally, we encrypted the file with sops accordingly with the current established workflows to manage secret files. [1] https://github.com/2i2c-org/pilot-hubs/issues/381#issuecomment-917170711 --- secrets/farallon.json | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 secrets/farallon.json diff --git a/secrets/farallon.json b/secrets/farallon.json new file mode 100644 index 000000000..f255bd912 --- /dev/null +++ b/secrets/farallon.json @@ -0,0 +1,27 @@ +{ + "AccessKey": { + "UserName": "ENC[AES256_GCM,data:xzejx3ekJ3o=,iv:6pkDM1SlOUZam95khZZsRkszGpf8rzE87ZZFTYfPuNA=,tag:TYF7ctbZ3szuY88kA8MyCg==,type:str]", + "AccessKeyId": "ENC[AES256_GCM,data:baIaSl6LkHTPI6VlIDFU/vhLjHI=,iv:lS1kSvvyB0BCnJ2ROt5ZYqKwRHD5+8PGttyrJRrfNt4=,tag:aATT82STHnqA0gJ18Z6h1w==,type:str]", + "Status": "ENC[AES256_GCM,data:dITKVJJU,iv:zNPAZiM9WlBGFcqTtgYZMCIUYFLGJeIkgnolyL70paU=,tag:y5dsK27jgopRA4YTjFeBTw==,type:str]", + "SecretAccessKey": "ENC[AES256_GCM,data:dv1lLpX+oNckM99jq9XyNDtmvatshvw0vG1R7um3eSWAAK5mhR8lPQ==,iv:vQijh5x9aXKkvwGtC+tz4K7c/2cC+4RS7ZhuR1nH8aE=,tag:DUVVUWjMQVzQ1+COT3gZjQ==,type:str]", + "CreateDate": "ENC[AES256_GCM,data:lTRvSgoBvv78lZKaMyo7FFaM1TY=,iv:GCLCdzuNmKebeQ36vFw1j6hLxzBXbauytA+b9rjpXyE=,tag:QQu4b+NFPCbO3VmP8eTo4Q==,type:str]" + }, + "sops": { + "kms": null, + "gcp_kms": [ + { + "resource_id": "projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs", + "created_at": "2021-09-10T19:47:17Z", + "enc": "CiQA4OM7eADZ6//P01khGg4CZO59PqPjFbWve5/BrnloowSkutESSQC9ZQbLAKgNPACzbKGS+Na1TnvxQ5HfjKGuRpe28hjxPRxLdYtjrFAJS9sLzMpMUOS10chi3N6SWLLbngM0mDIpCpx5nAzCjiI=" + } + ], + "azure_kv": null, + "hc_vault": null, + "age": null, + "lastmodified": "2021-09-10T19:47:19Z", + "mac": "ENC[AES256_GCM,data:n/hHhm2IdIQOy/s0yyRr9XzRU0sVTLa4VbzqdSWJ91ZtUiGdbBB6sw81lxRFv7x2MXnlGrBVs6QS07urB31QB7ryr1BhRDJeA0tVzceGf6KGmpqih3luWqEEtrtIwQSuqKgNueEQ4zcNmLq7KavfQkijE3MGYnoAudU/FavChFY=,iv:TO/jhfTpM03n71o9Sx/krrCmtuB/qyQg9FFBTdks5ec=,tag:EVg4qv2iP9SFnj0j0q7sIA==,type:str]", + "pgp": null, + "unencrypted_suffix": "_unencrypted", + "version": "3.7.1" + } +} \ No newline at end of file From 3fcdeb46e3992900aa98870b67c80c619a02d660 Mon Sep 17 00:00:00 2001 From: Damian Avila Date: Fri, 10 Sep 2021 17:39:41 -0300 Subject: [PATCH 3/7] Add logic in the auth_aws function to also export kops kubeconfig Previously, we have added an auth_aws function to retrieve the kubeconfig and connect to EKS-based cluster. With this commit, we are adding a new subprocess "brach" to conditionally `kops export` the kubeconfig (if the kops option is configured) and get access to kops-based clusters, such as the Farallon one. --- deployer/hub.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/deployer/hub.py b/deployer/hub.py index 46a9f25c4..ce17a99f7 100644 --- a/deployer/hub.py +++ b/deployer/hub.py @@ -146,7 +146,8 @@ def auth_aws(self): """ Reads `aws` nested config and temporarily sets environment variables like `KUBECONFIG`, `AWS_ACCESS_KEY_ID`, and `AWS_SECRET_ACCESS_KEY` - before trying to authenticate with the aws eks update-kubeconfig command. + before trying to authenticate with the `aws eks update-kubeconfig` or + the `kops export kubecfg --admin` commands. Finally get those environment variables to the original values to prevent side-effects on existing local configuration. @@ -157,6 +158,9 @@ def auth_aws(self): cluster_name = config['clusterName'] region = config['region'] + if cluster_type == 'kops': + state_store = config['stateStore'] + with tempfile.NamedTemporaryFile() as kubeconfig: orig_kubeconfig = os.environ.get('KUBECONFIG', None) orig_access_key_id = os.environ.get('AWS_ACCESS_KEY_ID', None) @@ -176,11 +180,18 @@ def auth_aws(self): os.environ['KUBECONFIG'] = kubeconfig.name - subprocess.check_call([ - 'aws', 'eks', 'update-kubeconfig', - f'--name={cluster_name}', - f'--region={region}' - ]) + if cluster_type == 'kops': + subprocess.check_call([ + 'kops', 'export', 'kubecfg', '--admin', + f'--name {cluster_name}', + f'--state {state_store}' + ]) + else: + subprocess.check_call([ + 'aws', 'eks', 'update-kubeconfig', + f'--name={cluster_name}', + f'--region={region}' + ]) yield finally: From 19529b897e4eed21d56f59bce2439b3a7aaf0171 Mon Sep 17 00:00:00 2001 From: Damian Avila Date: Fri, 10 Sep 2021 17:47:35 -0300 Subject: [PATCH 4/7] Add a new kops action into the CI to enable Farallon CI-based deployment I am still not sure how the action performs for real but it is the only one available in the marketplace and we should probably give it a try. I have also added a commented openscapes line for the future ;-) --- .github/workflows/deploy-hubs.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 1f0fecad6..316ba0bd2 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -27,6 +27,8 @@ jobs: - 2i2c - cloudbank - carbonplan + - farallon + # - openscapes - meom-ige - pangeo-181919 - pangeo-hubs @@ -81,6 +83,12 @@ jobs: (steps.config_files.outputs.hub_config == 'true') uses: mdgreenwald/mozilla-sops-action@v1 + - name: Setup kops + if: | + (steps.base_files.outputs.files == 'true') || + (steps.config_files.outputs.hub_config == 'true') + uses: hiberbee/github-action-kops@1.0.0 + - name: Deploy ${{ matrix.cluster_name }} if: | (steps.base_files.outputs.files == 'true') || From b73b8e723d8a010627ceff542f82095348becc51 Mon Sep 17 00:00:00 2001 From: Damian Avila Date: Mon, 13 Sep 2021 16:19:42 -0300 Subject: [PATCH 5/7] Add missing equal signs in the subprocess kops call --- deployer/hub.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployer/hub.py b/deployer/hub.py index ce17a99f7..c6b69112b 100644 --- a/deployer/hub.py +++ b/deployer/hub.py @@ -183,8 +183,8 @@ def auth_aws(self): if cluster_type == 'kops': subprocess.check_call([ 'kops', 'export', 'kubecfg', '--admin', - f'--name {cluster_name}', - f'--state {state_store}' + f'--name={cluster_name}', + f'--state={state_store}' ]) else: subprocess.check_call([ From 10cb749e534313c54ebafc9ac3d5dde11ce70646 Mon Sep 17 00:00:00 2001 From: Damian Avila Date: Mon, 13 Sep 2021 18:06:29 -0300 Subject: [PATCH 6/7] Install kops as per kops docs because the existing action is not working --- .github/workflows/deploy-hubs.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index 316ba0bd2..f9583c68d 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -87,7 +87,10 @@ jobs: if: | (steps.base_files.outputs.files == 'true') || (steps.config_files.outputs.hub_config == 'true') - uses: hiberbee/github-action-kops@1.0.0 + run: | + curl -Lo kops https://github.com/kubernetes/kops/releases/download/$(curl -s https://api.github.com/repos/kubernetes/kops/releases/latest | grep tag_name | cut -d '"' -f 4)/kops-linux-amd64 + chmod +x kops + sudo mv kops /usr/local/bin/kops - name: Deploy ${{ matrix.cluster_name }} if: | From f3da5eca6319c27e98435bc80e835e272ce0cd5b Mon Sep 17 00:00:00 2001 From: Damian Avila Date: Mon, 13 Sep 2021 20:31:47 -0300 Subject: [PATCH 7/7] Parametrize kops version and suggest when uncomment openscapes --- .github/workflows/deploy-hubs.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml index f9583c68d..7e2b23b01 100644 --- a/.github/workflows/deploy-hubs.yaml +++ b/.github/workflows/deploy-hubs.yaml @@ -28,6 +28,7 @@ jobs: - cloudbank - carbonplan - farallon + # Uncomment openscapes once a deployer user is created in openscapes AWS land # - openscapes - meom-ige - pangeo-181919 @@ -88,9 +89,11 @@ jobs: (steps.base_files.outputs.files == 'true') || (steps.config_files.outputs.hub_config == 'true') run: | - curl -Lo kops https://github.com/kubernetes/kops/releases/download/$(curl -s https://api.github.com/repos/kubernetes/kops/releases/latest | grep tag_name | cut -d '"' -f 4)/kops-linux-amd64 + curl -Lo kops https://github.com/kubernetes/kops/releases/download/$KOPS_VERSION/kops-linux-amd64 chmod +x kops sudo mv kops /usr/local/bin/kops + env: + KOPS_VERSION: "v1.21.1" - name: Deploy ${{ matrix.cluster_name }} if: |