From 7c74f8a9050fece2e6c111f1b6aca6a9b89be5eb Mon Sep 17 00:00:00 2001 From: Chico Venancio Date: Thu, 8 Aug 2019 11:18:46 -0300 Subject: [PATCH 01/44] add registry_class config value --- binderhub/app.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/binderhub/app.py b/binderhub/app.py index 5f41f4bc2..ab17e0376 100644 --- a/binderhub/app.py +++ b/binderhub/app.py @@ -20,7 +20,16 @@ import tornado.log from tornado.log import app_log import tornado.web -from traitlets import Unicode, Integer, Bool, Dict, validate, TraitError, default +from traitlets import ( + Unicode, + Integer, + Bool, + Dict, + validate, + TraitError, + default, + Type, +) from traitlets.config import Application from jupyterhub.services.auth import HubOAuthCallbackHandler @@ -195,6 +204,14 @@ def _valid_badge_base_url(self, proposal): config=True, ) + registry_class = Type( + DockerRegistry, + help=""" + Registry class implementation, change to define your own + """, + config=True + ) + use_registry = Bool( True, help=""" @@ -511,7 +528,7 @@ def initialize(self, *args, **kwargs): ]) jinja_env = Environment(loader=loader, **jinja_options) if self.use_registry and self.builder_required: - registry = DockerRegistry(parent=self) + registry = self.registry_class(parent=self) else: registry = None From 81f92ca6695c9cc23b6e8212bc0319a2a0d5962d Mon Sep 17 00:00:00 2001 From: Chico Venancio Date: Sat, 10 Aug 2019 11:02:22 -0300 Subject: [PATCH 02/44] add boto3 requirement --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index f34cb78aa..9370884c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ python-json-logger jupyterhub jsonschema pycurl +boto3 From 7d6f7756438692b6cec753d8bc7c8cba5c8bf941 Mon Sep 17 00:00:00 2001 From: Chico Venancio Date: Tue, 27 Aug 2019 13:04:03 -0300 Subject: [PATCH 03/44] add AWSElasticContainerRegistry class --- binderhub/registry.py | 63 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/binderhub/registry.py b/binderhub/registry.py index 8891041b2..fb5439bfc 100644 --- a/binderhub/registry.py +++ b/binderhub/registry.py @@ -6,10 +6,13 @@ import os from urllib.parse import urlparse +import boto3 +import kubernetes.client +import kubernetes.config from tornado import gen, httpclient from tornado.httputil import url_concat +from traitlets import default, Dict, Unicode, Any from traitlets.config import LoggingConfigurable -from traitlets import Dict, Unicode, default DEFAULT_DOCKER_REGISTRY_URL = "https://registry.hub.docker.com" DEFAULT_DOCKER_AUTH_URL = "https://index.docker.io/v1" @@ -224,3 +227,61 @@ def get_image_manifest(self, image, tag): raise else: return json.loads(resp.body.decode("utf-8")) + + +class AWSElasticContainerRegistry(DockerRegistry): + aws_region = Unicode( + config=True, + help=""" + AWS region for ECR service + """, + ) + + ecr_client = Any() + + @default("ecr_client") + def _get_ecr_client(self): + return boto3.client("ecr", region_name=self.aws_region) + + username = "AWS" + + kubernetes.config.load_incluster_config() + kube_client = kubernetes.client.CoreV1Api() + + def _get_ecr_auth(self): + return self.ecr_client.get_authorization_token()["authorizationData"][0] + + @default("url") + def _default_url(self): + return self._get_ecr_auth()["proxyEndpoint"] + + def _patch_docker_config_secret(self, auth): + """Patch binder-push-secret""" + secret_data = {"auths": {self.url: {"auth": auth["authorizationToken"]}}} + secret_data = base64.b64encode(json.dumps(secret_data).encode("utf8")).decode( + "utf8" + ) + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as f: + namespace = f.read() + self.kube_client.patch_namespaced_secret( + "binder-push-secret", namespace, {"data": {"config.json": secret_data}} + ) + + @default("password") + def _get_ecr_pawssord(self): + """Get ecr password""" + auth = self._get_ecr_auth() + self.password_expires = auth["expiresAt"] + self._patch_docker_config_secret(auth) + return base64.b64decode(auth['authorizationToken']).decode("utf-8").split(':')[1] + + async def get_image_manifest(self, image, tag): + try: + repo_name = image.split("/", 1)[1] + self.ecr_client.create_repository(repositoryName=repo_name) + self.log.info("Creating ECR repo {}".format(repo_name)) + except self.ecr_client.exceptions.RepositoryAlreadyExistsException: + self.log.info("ECR repo {} already exists".format(repo_name)) + # TODO: check for expiration before reseting password + self.password = self._get_ecr_pawssord() + return await super().get_image_manifest(repo_name, tag) From b0b438c96d1d67751e1afb8fc4570a39d6d2560f Mon Sep 17 00:00:00 2001 From: Chico Venancio Date: Tue, 27 Aug 2019 15:52:31 -0300 Subject: [PATCH 04/44] rbac.yaml: allow manipulation of secrets --- helm-chart/binderhub/templates/rbac.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/helm-chart/binderhub/templates/rbac.yaml b/helm-chart/binderhub/templates/rbac.yaml index 210e62df0..d249d4e09 100644 --- a/helm-chart/binderhub/templates/rbac.yaml +++ b/helm-chart/binderhub/templates/rbac.yaml @@ -14,6 +14,10 @@ rules: - apiGroups: [""] resources: ["pods/log"] verbs: ["get"] +- apiGroups: + - "" + resources: ["secrets"] + verbs: ["get", "patch"] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 From 6cb5b6a54706de21abd48c70e428e2606b66f76d Mon Sep 17 00:00:00 2001 From: Chico Venancio Date: Tue, 27 Aug 2019 18:03:45 -0300 Subject: [PATCH 05/44] doc-requirements: add boto3 --- doc/doc-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/doc-requirements.txt b/doc/doc-requirements.txt index 96b78c0a8..0763e3dd1 100644 --- a/doc/doc-requirements.txt +++ b/doc/doc-requirements.txt @@ -20,3 +20,4 @@ python-json-logger jupyterhub jsonschema #pycurl Do not install for docs as it breaks the RTD build. Its primary use is for mocks in testing . +boto3 From e25e59763ef2e33fb829e2feb36329721b28e015 Mon Sep 17 00:00:00 2001 From: Chico Venancio Date: Fri, 8 Nov 2019 18:52:21 -0300 Subject: [PATCH 06/44] Fix syntax --- binderhub/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/binderhub/app.py b/binderhub/app.py index 81a7a89ac..4a9cd4640 100644 --- a/binderhub/app.py +++ b/binderhub/app.py @@ -211,6 +211,7 @@ def _valid_badge_base_url(self, proposal): Registry class implementation, change to define your own """, config=True + ) sticky_builds = Bool( False, From bd961a85e3da6e4549528f2a1d7c023c92a3e3c4 Mon Sep 17 00:00:00 2001 From: Ivan Gomes Date: Mon, 20 Jan 2020 16:45:16 -0800 Subject: [PATCH 07/44] Revert RBAC modification and remove handling in AWSElasticContainerRegistry for secret management --- binderhub/registry.py | 46 +++++------------------- helm-chart/binderhub/templates/rbac.yaml | 4 --- 2 files changed, 9 insertions(+), 41 deletions(-) diff --git a/binderhub/registry.py b/binderhub/registry.py index fb5439bfc..8c8bc9c7c 100644 --- a/binderhub/registry.py +++ b/binderhub/registry.py @@ -7,8 +7,6 @@ from urllib.parse import urlparse import boto3 -import kubernetes.client -import kubernetes.config from tornado import gen, httpclient from tornado.httputil import url_concat from traitlets import default, Dict, Unicode, Any @@ -237,51 +235,25 @@ class AWSElasticContainerRegistry(DockerRegistry): """, ) + username = "AWS" + + # fetch password every time as ECR password expires every 12 hours + # and is refreshed by k8s externally + @property + def password(self): + return super()._default_password() + ecr_client = Any() @default("ecr_client") def _get_ecr_client(self): return boto3.client("ecr", region_name=self.aws_region) - username = "AWS" - - kubernetes.config.load_incluster_config() - kube_client = kubernetes.client.CoreV1Api() - - def _get_ecr_auth(self): - return self.ecr_client.get_authorization_token()["authorizationData"][0] - - @default("url") - def _default_url(self): - return self._get_ecr_auth()["proxyEndpoint"] - - def _patch_docker_config_secret(self, auth): - """Patch binder-push-secret""" - secret_data = {"auths": {self.url: {"auth": auth["authorizationToken"]}}} - secret_data = base64.b64encode(json.dumps(secret_data).encode("utf8")).decode( - "utf8" - ) - with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as f: - namespace = f.read() - self.kube_client.patch_namespaced_secret( - "binder-push-secret", namespace, {"data": {"config.json": secret_data}} - ) - - @default("password") - def _get_ecr_pawssord(self): - """Get ecr password""" - auth = self._get_ecr_auth() - self.password_expires = auth["expiresAt"] - self._patch_docker_config_secret(auth) - return base64.b64decode(auth['authorizationToken']).decode("utf-8").split(':')[1] - async def get_image_manifest(self, image, tag): try: repo_name = image.split("/", 1)[1] self.ecr_client.create_repository(repositoryName=repo_name) - self.log.info("Creating ECR repo {}".format(repo_name)) + self.log.info("ECR repo {} created".format(repo_name)) except self.ecr_client.exceptions.RepositoryAlreadyExistsException: self.log.info("ECR repo {} already exists".format(repo_name)) - # TODO: check for expiration before reseting password - self.password = self._get_ecr_pawssord() return await super().get_image_manifest(repo_name, tag) diff --git a/helm-chart/binderhub/templates/rbac.yaml b/helm-chart/binderhub/templates/rbac.yaml index d249d4e09..210e62df0 100644 --- a/helm-chart/binderhub/templates/rbac.yaml +++ b/helm-chart/binderhub/templates/rbac.yaml @@ -14,10 +14,6 @@ rules: - apiGroups: [""] resources: ["pods/log"] verbs: ["get"] -- apiGroups: - - "" - resources: ["secrets"] - verbs: ["get", "patch"] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 From af56e934d262b42197ada40b33b7fb976c233bff Mon Sep 17 00:00:00 2001 From: Ivan Gomes Date: Mon, 20 Jan 2020 18:52:38 -0800 Subject: [PATCH 08/44] Add password and url fetching for ECR --- binderhub/registry.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/binderhub/registry.py b/binderhub/registry.py index 8c8bc9c7c..b13c427f3 100644 --- a/binderhub/registry.py +++ b/binderhub/registry.py @@ -235,20 +235,28 @@ class AWSElasticContainerRegistry(DockerRegistry): """, ) - username = "AWS" - - # fetch password every time as ECR password expires every 12 hours - # and is refreshed by k8s externally - @property - def password(self): - return super()._default_password() - ecr_client = Any() @default("ecr_client") def _get_ecr_client(self): return boto3.client("ecr", region_name=self.aws_region) + # TODO: cache auth if not expired - authorizationData.0.expiresAt + def _get_ecr_auth(self): + return self.ecr_client.get_authorization_token()["authorizationData"][0] + + username = "AWS" + + @property + def password(self): + # Fetch password every time as ECR password expires + auth = self._get_ecr_auth() + return base64.b64decode(auth['authorizationToken']).decode("utf-8").split(':')[1] + + @default("url") + def _default_url(self): + return self._get_ecr_auth()["proxyEndpoint"] + async def get_image_manifest(self, image, tag): try: repo_name = image.split("/", 1)[1] From 296772bd20db32da2c61ddb33d89ec19c5233508 Mon Sep 17 00:00:00 2001 From: Ivan Gomes Date: Mon, 20 Jan 2020 21:46:10 -0800 Subject: [PATCH 09/44] Add AWS ECR documentation --- doc/setup-binderhub.rst | 47 +++++++++++++++++++++ doc/setup-registry.rst | 91 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) diff --git a/doc/setup-binderhub.rst b/doc/setup-binderhub.rst index 9c5bf54b9..6e56e09ff 100644 --- a/doc/setup-binderhub.rst +++ b/doc/setup-binderhub.rst @@ -116,6 +116,22 @@ where: * `` is the AppID of the Service Principal with AcrPush role assignment, * `` is the password for the Service Principal. +If you are using Amazon Elastic Container Registry +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Update `secret.yaml` to include the following:: + + registry: + url: https://.dkr.ecr..amazonaws.com + +where: + +* ```` is the identifier of your AWS account +* ```` is the AWS region of the ECR registry, e.g. ``us-east-1`` + +As ECR uses AWS IAM for authorization, specifying ``username`` and ``password`` +is not necessary. + Create ``config.yaml`` ---------------------- @@ -183,6 +199,37 @@ where: If this is not provided, you may find BinderHub rebuilds images every launch instead of pulling them from the ACR. Suggestions for `` could be `ACR_NAME` or the name you give your BinderHub. +If you are using Amazon Elastic Container Registry +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you want your BinderHub to push and pull images from an Amazon Elastic +Container Registry (ECR), then your `config.yaml` file will look as follows:: + + config: + BinderHub: + use_registry: true + registry_class: binderhub.registry.AWSElasticContainerRegistry + image_prefix: "-" + AWSElasticContainerRegistry: + aws_region: + +where: + +* ```` is the AWS region of the ECR registry, e.g. ``us-east-1``. +* ```` can be any string, and will be prepended to image names. We + recommend something descriptive such as ``binder-dev-`` or ``binder-prod-`` + (ending with a `-` is useful). + +If you opted to use an IAM User with programmatic access instead of assuming +the role in the previous step you will additionally need to add the following +to your `config.yaml`:: + + extraEnv: + - name: AWS_ACCESS_KEY_ID + value: "xxx" + - name: AWS_SECRET_ACCESS_KEY + value: "yyy" + If you are using a custom registry ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/setup-registry.rst b/doc/setup-registry.rst index 5921e826f..9fd81ed56 100644 --- a/doc/setup-registry.rst +++ b/doc/setup-registry.rst @@ -137,6 +137,97 @@ where: SERVICE_PRINCIPAL_PASSWORD=$(az ad sp create-for-rbac --name --role AcrPush --scopes --query password --output tsv) SERVICE_PRINCIPAL_ID=$(az ad sp show --id http:// --query appId --output tsv) +.. _use-ecr: + +Set up Amazon Elastic Container Registry +---------------------------------------- + +To use Amazon Elastic Container Registry (ECR), you'll need to use AWS IAM to +authorize the machine or pod running BinderHub so it can push images. There +are a number of options on how to do this with IAM and Kubernetes, but we +will highlight two: assume IAM role and IAM user with programmatic access. + +Start by creating an IAM policy that grants access to create repositories and +read/write images from them. An example IAM permissions policy is provided +below. For more information and examples see `Identity and Access Management for Amazon Elastic Container Registry `_. + +.. code-block:: json + + { + "Statement": [ + { + "Action": [ + "ecr:ListImages" + ], + "Effect": "Allow", + "Resource": "arn:aws:ecr:::-*", + "Sid": "ListImagesInRepository" + }, + { + "Action": [ + "ecr:GetAuthorizationToken" + ], + "Effect": "Allow", + "Resource": "*", + "Sid": "GetAuthorizationToken" + }, + { + "Action": [ + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:DescribeImages", + "ecr:BatchGetImage", + "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", + "ecr:CompleteLayerUpload", + "ecr:PutImage" + ], + "Effect": "Allow", + "Resource": "arn:aws:ecr:::-*", + "Sid": "ManageRepositoryContents" + }, + { + "Action": [ + "ecr:CreateRepository" + ], + "Effect": "Allow", + "Resource": "arn:aws:ecr:::-*", + "Sid": "CreateRepository" + }, + ], + "Version": "2012-10-17" + } + +If you are using AWS EC2, EKS, etc. to set up your Kubernetes cluster you can +add this policy to the IAM Role assumed by the nodes of the cluster, e.g. +``nodes..k8s.local`` if you followed `Zero to JupyterHub with Kubernetes `_ +and used kops. The IAM permissions policy will need to accompanied with an IAM +trust policy to allow it to be assumed. An example for EC2 is provided below. +For more information see `Granting a User Permissions to Pass a Role to an AWS Service `_. +This is the recommended method if your Kubernetes cluster is provisioned on +AWS. + +.. code-block:: json + + { + "Version": "2012-10-17", + "Statement": { + "Sid": "TrustPolicyStatementThatAllowsEC2ServiceToAssumeTheAttachedRole", + "Effect": "Allow", + "Principal": { "Service": "ec2.amazonaws.com" }, + "Action": "sts:AssumeRole" + } + } + +Alternatively, you can create an IAM user with programmatic access (see +`Creating an IAM User in Your AWS Account `_) +and specify the ``AWS_ACCESS_KEY_ID`` and ``AWS_SECRET_ACCESS_KEY`` environment +variables in the following step. + Next step --------- From ade3a8b4c88a6b7e2a9e542b87df4da1ff5ae3c6 Mon Sep 17 00:00:00 2001 From: Ivan Gomes Date: Mon, 20 Jan 2020 21:50:03 -0800 Subject: [PATCH 10/44] Fix wording in AWS ECR documentation --- doc/setup-registry.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/setup-registry.rst b/doc/setup-registry.rst index 9fd81ed56..65ff58302 100644 --- a/doc/setup-registry.rst +++ b/doc/setup-registry.rst @@ -202,8 +202,8 @@ below. For more information and examples see `Identity and Access Management for "Version": "2012-10-17" } -If you are using AWS EC2, EKS, etc. to set up your Kubernetes cluster you can -add this policy to the IAM Role assumed by the nodes of the cluster, e.g. +If you used AWS services like EC2 or EKS to set up your Kubernetes cluster you +can add this policy to the IAM Role assumed by the nodes of the cluster, e.g. ``nodes..k8s.local`` if you followed `Zero to JupyterHub with Kubernetes `_ and used kops. The IAM permissions policy will need to accompanied with an IAM trust policy to allow it to be assumed. An example for EC2 is provided below. From c1c37ddb94348cc522b5e78e2675f961518c16b3 Mon Sep 17 00:00:00 2001 From: Ivan Gomes Date: Tue, 21 Jan 2020 09:13:14 -0800 Subject: [PATCH 11/44] Add ECR host to `image_prefix` in documentation --- doc/setup-binderhub.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/setup-binderhub.rst b/doc/setup-binderhub.rst index 6e56e09ff..18aecce85 100644 --- a/doc/setup-binderhub.rst +++ b/doc/setup-binderhub.rst @@ -209,13 +209,14 @@ Container Registry (ECR), then your `config.yaml` file will look as follows:: BinderHub: use_registry: true registry_class: binderhub.registry.AWSElasticContainerRegistry - image_prefix: "-" + image_prefix: ".dkr.ecr..amazonaws.com/-" AWSElasticContainerRegistry: - aws_region: + aws_region: where: -* ```` is the AWS region of the ECR registry, e.g. ``us-east-1``. +* ```` is the identifier of your AWS account +* ```` is the AWS region of the ECR registry, e.g. ``us-east-1``. * ```` can be any string, and will be prepended to image names. We recommend something descriptive such as ``binder-dev-`` or ``binder-prod-`` (ending with a `-` is useful). From b9ecea4b6406c8c9ad60b1a01ee37bbd1443d7f6 Mon Sep 17 00:00:00 2001 From: Ivan Gomes Date: Tue, 21 Jan 2020 11:18:57 -0800 Subject: [PATCH 12/44] Re-add binder-push-secret patching. Add check for correct auth token. --- binderhub/registry.py | 28 +++++++++++++++++++----- helm-chart/binderhub/templates/rbac.yaml | 3 +++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/binderhub/registry.py b/binderhub/registry.py index b13c427f3..c81143467 100644 --- a/binderhub/registry.py +++ b/binderhub/registry.py @@ -7,6 +7,8 @@ from urllib.parse import urlparse import boto3 +import kubernetes.client +import kubernetes.config from tornado import gen, httpclient from tornado.httputil import url_concat from traitlets import default, Dict, Unicode, Any @@ -241,9 +243,12 @@ class AWSElasticContainerRegistry(DockerRegistry): def _get_ecr_client(self): return boto3.client("ecr", region_name=self.aws_region) - # TODO: cache auth if not expired - authorizationData.0.expiresAt + # TODO: cache auth if not expired - authorizationData[i]["expiresAt"] def _get_ecr_auth(self): - return self.ecr_client.get_authorization_token()["authorizationData"][0] + auths = self.ecr_client.get_authorization_token()["authorizationData"] + auth = next(x for x in auths if x["proxyEndpoint"] == self.url) + self._patch_docker_config_secret(auth) + return auth username = "AWS" @@ -253,10 +258,6 @@ def password(self): auth = self._get_ecr_auth() return base64.b64decode(auth['authorizationToken']).decode("utf-8").split(':')[1] - @default("url") - def _default_url(self): - return self._get_ecr_auth()["proxyEndpoint"] - async def get_image_manifest(self, image, tag): try: repo_name = image.split("/", 1)[1] @@ -265,3 +266,18 @@ async def get_image_manifest(self, image, tag): except self.ecr_client.exceptions.RepositoryAlreadyExistsException: self.log.info("ECR repo {} already exists".format(repo_name)) return await super().get_image_manifest(repo_name, tag) + + kubernetes.config.load_incluster_config() + kube_client = kubernetes.client.CoreV1Api() + + def _patch_docker_config_secret(self, auth): + """Patch binder-push-secret""" + secret_data = {"auths": {self.url: {"auth": auth["authorizationToken"]}}} + secret_data = base64.b64encode(json.dumps(secret_data).encode("utf8")).decode( + "utf8" + ) + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as f: + namespace = f.read() + self.kube_client.patch_namespaced_secret( + "binder-push-secret", namespace, {"data": {"config.json": secret_data}} + ) diff --git a/helm-chart/binderhub/templates/rbac.yaml b/helm-chart/binderhub/templates/rbac.yaml index 210e62df0..2942ac2e3 100644 --- a/helm-chart/binderhub/templates/rbac.yaml +++ b/helm-chart/binderhub/templates/rbac.yaml @@ -14,6 +14,9 @@ rules: - apiGroups: [""] resources: ["pods/log"] verbs: ["get"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "patch"] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 From f9fd86c8fa9e31d5d8a994cbcf25421140d078a0 Mon Sep 17 00:00:00 2001 From: Ivan Date: Thu, 30 Jan 2020 10:17:15 -0800 Subject: [PATCH 13/44] Remove duplicate GetAuthorizationToken in AWS ECR doc --- doc/setup-registry.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/setup-registry.rst b/doc/setup-registry.rst index 65ff58302..9c700e7ae 100644 --- a/doc/setup-registry.rst +++ b/doc/setup-registry.rst @@ -173,7 +173,6 @@ below. For more information and examples see `Identity and Access Management for }, { "Action": [ - "ecr:GetAuthorizationToken", "ecr:BatchCheckLayerAvailability", "ecr:GetDownloadUrlForLayer", "ecr:GetRepositoryPolicy", From a5edcd324ed18077798f874706f22f42eff6936f Mon Sep 17 00:00:00 2001 From: Ivan Gomes Date: Sat, 8 Feb 2020 20:37:29 -0800 Subject: [PATCH 14/44] Lazily initialize kube_client in AWSElasticContainerRegistry to prevent ConfigException when outside k8s --- binderhub/registry.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/binderhub/registry.py b/binderhub/registry.py index c81143467..f99d05534 100644 --- a/binderhub/registry.py +++ b/binderhub/registry.py @@ -267,8 +267,12 @@ async def get_image_manifest(self, image, tag): self.log.info("ECR repo {} already exists".format(repo_name)) return await super().get_image_manifest(repo_name, tag) - kubernetes.config.load_incluster_config() - kube_client = kubernetes.client.CoreV1Api() + kube_client = Any() + + @default("kube_client") + def _get_kube_client(self): + kubernetes.config.load_incluster_config() + return kubernetes.client.CoreV1Api() def _patch_docker_config_secret(self, auth): """Patch binder-push-secret""" From a9aa576884a8f0ad71e376e03504840616da2318 Mon Sep 17 00:00:00 2001 From: Ivan Gomes Date: Wed, 12 Feb 2020 22:21:38 -0800 Subject: [PATCH 15/44] Update description of `registry_class` --- binderhub/app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/binderhub/app.py b/binderhub/app.py index 2847b189d..4402658d0 100644 --- a/binderhub/app.py +++ b/binderhub/app.py @@ -210,7 +210,8 @@ def _valid_badge_base_url(self, proposal): registry_class = Type( DockerRegistry, help=""" - Registry class implementation, change to define your own + Change this to support different Docker container registries. + The default works with GCR, ACR and DockerHub. Use `AWSElasticContainerRegistry` for AWS ECR. """, config=True ) From 1047f9e22bef31e3e4e27c0f467ef1de803be3d2 Mon Sep 17 00:00:00 2001 From: Ivan Gomes Date: Tue, 12 May 2020 21:10:14 -0700 Subject: [PATCH 16/44] Rename `registry_class` to `docker_registry_class` for clarity --- binderhub/app.py | 4 ++-- doc/zero-to-binderhub/setup-binderhub.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/binderhub/app.py b/binderhub/app.py index 493c36947..e776976b0 100755 --- a/binderhub/app.py +++ b/binderhub/app.py @@ -207,7 +207,7 @@ def _valid_badge_base_url(self, proposal): config=True, ) - registry_class = Type( + docker_registry_class = Type( DockerRegistry, help=""" Change this to support different Docker container registries. @@ -564,7 +564,7 @@ def initialize(self, *args, **kwargs): ]) jinja_env = Environment(loader=loader, **jinja_options) if self.use_registry and self.builder_required: - registry = self.registry_class(parent=self) + registry = self.docker_registry_class(parent=self) else: registry = None diff --git a/doc/zero-to-binderhub/setup-binderhub.rst b/doc/zero-to-binderhub/setup-binderhub.rst index eab147acd..0421412c0 100644 --- a/doc/zero-to-binderhub/setup-binderhub.rst +++ b/doc/zero-to-binderhub/setup-binderhub.rst @@ -208,7 +208,7 @@ Container Registry (ECR), then your `config.yaml` file will look as follows:: config: BinderHub: use_registry: true - registry_class: binderhub.registry.AWSElasticContainerRegistry + docker_registry_class: binderhub.registry.AWSElasticContainerRegistry image_prefix: ".dkr.ecr..amazonaws.com/-" AWSElasticContainerRegistry: aws_region: From 88e4914a59a7c312deb0abd4b2146bafc0dcb5ac Mon Sep 17 00:00:00 2001 From: Ivan Gomes Date: Fri, 5 Jun 2020 17:37:47 -0700 Subject: [PATCH 17/44] Move boto3 and kubernetes calls to own threadpool --- binderhub/registry.py | 64 +++++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/binderhub/registry.py b/binderhub/registry.py index f99d05534..59958829c 100644 --- a/binderhub/registry.py +++ b/binderhub/registry.py @@ -1,7 +1,9 @@ """ Interaction with the Docker Registry """ +import asyncio import base64 +from concurrent.futures import ThreadPoolExecutor import json import os from urllib.parse import urlparse @@ -11,7 +13,7 @@ import kubernetes.config from tornado import gen, httpclient from tornado.httputil import url_concat -from traitlets import default, Dict, Unicode, Any +from traitlets import default, Dict, Unicode, Any, Integer from traitlets.config import LoggingConfigurable DEFAULT_DOCKER_REGISTRY_URL = "https://registry.hub.docker.com" @@ -243,29 +245,24 @@ class AWSElasticContainerRegistry(DockerRegistry): def _get_ecr_client(self): return boto3.client("ecr", region_name=self.aws_region) - # TODO: cache auth if not expired - authorizationData[i]["expiresAt"] - def _get_ecr_auth(self): - auths = self.ecr_client.get_authorization_token()["authorizationData"] - auth = next(x for x in auths if x["proxyEndpoint"] == self.url) - self._patch_docker_config_secret(auth) - return auth - username = "AWS" - @property - def password(self): - # Fetch password every time as ECR password expires - auth = self._get_ecr_auth() - return base64.b64decode(auth['authorizationToken']).decode("utf-8").split(':')[1] + executor_threads = Integer( + 5, + config=True, + help="""The number of threads to use for blocking calls + + Should generaly be a small number because we don't + care about high concurrency here, just not blocking the webserver. + This executor is not used for long-running tasks (e.g. builds). + """, + ) - async def get_image_manifest(self, image, tag): - try: - repo_name = image.split("/", 1)[1] - self.ecr_client.create_repository(repositoryName=repo_name) - self.log.info("ECR repo {} created".format(repo_name)) - except self.ecr_client.exceptions.RepositoryAlreadyExistsException: - self.log.info("ECR repo {} already exists".format(repo_name)) - return await super().get_image_manifest(repo_name, tag) + executor = Any() + + @default("executor") + def _get_executor(self): + return ThreadPoolExecutor(self.executor_threads) kube_client = Any() @@ -273,6 +270,31 @@ async def get_image_manifest(self, image, tag): def _get_kube_client(self): kubernetes.config.load_incluster_config() return kubernetes.client.CoreV1Api() + + async def get_image_manifest(self, image, tag): + image = image.split("/", 1)[1] + await asyncio.wrap_future(self.executor.submit(self._pre_get_image_manifest, image, tag)) + return await super().get_image_manifest(image, tag) + + def _pre_get_image_manifest(self, image, tag): + self._create_repository(image, tag) + self._refresh_password() + + def _create_repository(self, image, tag): + try: + self.ecr_client.create_repository(repositoryName=image) + self.log.info("ECR repo {} created".format(image)) + except self.ecr_client.exceptions.RepositoryAlreadyExistsException: + self.log.info("ECR repo {} already exists".format(image)) + + # An IAM principal is used to generate an auth token that is valid for 12 hours + # ref: https://docs.aws.amazon.com/AmazonECR/latest/userguide/Registries.html + # TODO: cache auth if not expired - authorizationData[i]["expiresAt"] + def _refresh_password(self): + auths = self.ecr_client.get_authorization_token()["authorizationData"] + auth = next(x for x in auths if x["proxyEndpoint"] == self.url) + self._patch_docker_config_secret(auth) + self.password = base64.b64decode(auth['authorizationToken']).decode("utf-8").split(':')[1] def _patch_docker_config_secret(self, auth): """Patch binder-push-secret""" From 019a9fe7ce5338c81a3ede6c1f438920c96916fa Mon Sep 17 00:00:00 2001 From: Ivan Gomes Date: Fri, 31 Jul 2020 10:56:12 -0700 Subject: [PATCH 18/44] Restore newline lost during merge --- doc/zero-to-binderhub/setup-binderhub.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/zero-to-binderhub/setup-binderhub.rst b/doc/zero-to-binderhub/setup-binderhub.rst index d434ef250..f5e1c98a8 100644 --- a/doc/zero-to-binderhub/setup-binderhub.rst +++ b/doc/zero-to-binderhub/setup-binderhub.rst @@ -219,6 +219,7 @@ If you are using OVH Container Registry ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you want your BinderHub to push and pull images from an OVH Container Registry, then the `config.yaml` file will look the following:: + config: BinderHub: use_registry: true From a74d814bef49b7004441056419be9f7451dbe164 Mon Sep 17 00:00:00 2001 From: Tomas Beuzen Date: Fri, 18 Sep 2020 10:42:39 -0700 Subject: [PATCH 19/44] general word-smithing for clarity --- doc/zero-to-binderhub/setup-registry.rst | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/doc/zero-to-binderhub/setup-registry.rst b/doc/zero-to-binderhub/setup-registry.rst index 4c6b3d9e0..2d886c699 100644 --- a/doc/zero-to-binderhub/setup-registry.rst +++ b/doc/zero-to-binderhub/setup-registry.rst @@ -175,11 +175,12 @@ Set up Amazon Elastic Container Registry To use Amazon Elastic Container Registry (ECR), you'll need to use AWS IAM to authorize the machine or pod running BinderHub so it can push images. There are a number of options on how to do this with IAM and Kubernetes, but we -will highlight two: assume IAM role and IAM user with programmatic access. +will highlight two: define and assign an IAM role, or assume an IAM user with programmatic access. Start by creating an IAM policy that grants access to create repositories and -read/write images from them. An example IAM permissions policy is provided -below. For more information and examples see `Identity and Access Management for Amazon Elastic Container Registry `_. +read/write images from them. You can create policies using the AWS console, CLI +or API as detailed in the documentation `Creating IAM policies `_. +An example IAM permissions policy is provided below. For more information and examples see `Identity and Access Management for Amazon Elastic Container Registry `_. .. code-block:: json @@ -234,11 +235,13 @@ below. For more information and examples see `Identity and Access Management for If you used AWS services like EC2 or EKS to set up your Kubernetes cluster you can add this policy to the IAM Role assumed by the nodes of the cluster, e.g. ``nodes..k8s.local`` if you followed `Zero to JupyterHub with Kubernetes `_ -and used kops. The IAM permissions policy will need to accompanied with an IAM -trust policy to allow it to be assumed. An example for EC2 is provided below. -For more information see `Granting a User Permissions to Pass a Role to an AWS Service `_. -This is the recommended method if your Kubernetes cluster is provisioned on -AWS. +and used kops. One way to do this from the AWS Console is to navigate to the IAM service and click "Roles" in the side bar, +then find and select the IAM Role assumed by the nodes of your cluster, click "Permissions" and then "Attach policies" to attach +the IAM Policy we just created. The IAM permissions policy will need to accompanied with an IAM +trust policy to allow it to be assumed. A suitable trust policy may already be defined for your node's IAM Role, +you can view and edit the trust policy by clicking the "Trust relationships" tab is next to the "Permissions" tab in the AWS console. +An example trust policy for EC2 is provided below. For more information see `Granting a User Permissions to Pass a Role to an AWS Service `_. +This is the recommended method if your Kubernetes cluster is provisioned on AWS. .. code-block:: json @@ -252,7 +255,7 @@ AWS. } } -Alternatively, you can create an IAM user with programmatic access (see +Alternatively to the above steps, you can create an IAM user with programmatic access (see `Creating an IAM User in Your AWS Account `_) and specify the ``AWS_ACCESS_KEY_ID`` and ``AWS_SECRET_ACCESS_KEY`` environment variables in the following step. From dddecf8781a9c3833f2ba640f597b2b4f9e75b52 Mon Sep 17 00:00:00 2001 From: Tomas Beuzen Date: Fri, 18 Sep 2020 10:43:06 -0700 Subject: [PATCH 20/44] remove stray comma that AWS doesn't like in the policy --- doc/zero-to-binderhub/setup-registry.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/zero-to-binderhub/setup-registry.rst b/doc/zero-to-binderhub/setup-registry.rst index 2d886c699..506f2aaae 100644 --- a/doc/zero-to-binderhub/setup-registry.rst +++ b/doc/zero-to-binderhub/setup-registry.rst @@ -227,7 +227,7 @@ An example IAM permissions policy is provided below. For more information and ex "Effect": "Allow", "Resource": "arn:aws:ecr:::-*", "Sid": "CreateRepository" - }, + } ], "Version": "2012-10-17" } From a729986c51e0739cf50e0d9d2a129d371f4934cb Mon Sep 17 00:00:00 2001 From: Tomas Beuzen Date: Fri, 18 Sep 2020 10:45:32 -0700 Subject: [PATCH 21/44] one more small clarification --- doc/zero-to-binderhub/setup-registry.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/zero-to-binderhub/setup-registry.rst b/doc/zero-to-binderhub/setup-registry.rst index 506f2aaae..fd31ba876 100644 --- a/doc/zero-to-binderhub/setup-registry.rst +++ b/doc/zero-to-binderhub/setup-registry.rst @@ -177,7 +177,7 @@ authorize the machine or pod running BinderHub so it can push images. There are a number of options on how to do this with IAM and Kubernetes, but we will highlight two: define and assign an IAM role, or assume an IAM user with programmatic access. -Start by creating an IAM policy that grants access to create repositories and +For the former, start by creating an IAM policy that grants access to create repositories and read/write images from them. You can create policies using the AWS console, CLI or API as detailed in the documentation `Creating IAM policies `_. An example IAM permissions policy is provided below. For more information and examples see `Identity and Access Management for Amazon Elastic Container Registry `_. From 9a4ad32d0c32ff6b94023380132199b9196c9469 Mon Sep 17 00:00:00 2001 From: Chico Venancio Date: Sat, 10 Aug 2019 11:02:22 -0300 Subject: [PATCH 22/44] add boto3 requirement --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 9fadb2d19..e5e40a632 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ pyjwt>=2 python-json-logger tornado>=5.1 traitlets +boto3 From dfcb9215c025bf344c440622232a22f22f5c84b1 Mon Sep 17 00:00:00 2001 From: Chico Venancio Date: Tue, 27 Aug 2019 13:04:03 -0300 Subject: [PATCH 23/44] add AWSElasticContainerRegistry class --- binderhub/registry.py | 63 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/binderhub/registry.py b/binderhub/registry.py index 2c54a7673..daf262512 100644 --- a/binderhub/registry.py +++ b/binderhub/registry.py @@ -6,9 +6,12 @@ import os from urllib.parse import urlparse +import boto3 +import kubernetes.client +import kubernetes.config from tornado import httpclient from tornado.httputil import url_concat -from traitlets import Dict, Unicode, default +from traitlets import Dict, Unicode, default, Any from traitlets.config import LoggingConfigurable DEFAULT_DOCKER_REGISTRY_URL = "https://registry-1.docker.io" @@ -233,6 +236,64 @@ async def get_image_manifest(self, image, tag): return json.loads(resp.body.decode("utf-8")) +class AWSElasticContainerRegistry(DockerRegistry): + aws_region = Unicode( + config=True, + help=""" + AWS region for ECR service + """, + ) + + ecr_client = Any() + + @default("ecr_client") + def _get_ecr_client(self): + return boto3.client("ecr", region_name=self.aws_region) + + username = "AWS" + + kubernetes.config.load_incluster_config() + kube_client = kubernetes.client.CoreV1Api() + + def _get_ecr_auth(self): + return self.ecr_client.get_authorization_token()["authorizationData"][0] + + @default("url") + def _default_url(self): + return self._get_ecr_auth()["proxyEndpoint"] + + def _patch_docker_config_secret(self, auth): + """Patch binder-push-secret""" + secret_data = {"auths": {self.url: {"auth": auth["authorizationToken"]}}} + secret_data = base64.b64encode(json.dumps(secret_data).encode("utf8")).decode( + "utf8" + ) + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as f: + namespace = f.read() + self.kube_client.patch_namespaced_secret( + "binder-push-secret", namespace, {"data": {"config.json": secret_data}} + ) + + @default("password") + def _get_ecr_pawssord(self): + """Get ecr password""" + auth = self._get_ecr_auth() + self.password_expires = auth["expiresAt"] + self._patch_docker_config_secret(auth) + return base64.b64decode(auth['authorizationToken']).decode("utf-8").split(':')[1] + + async def get_image_manifest(self, image, tag): + try: + repo_name = image.split("/", 1)[1] + self.ecr_client.create_repository(repositoryName=repo_name) + self.log.info("Creating ECR repo {}".format(repo_name)) + except self.ecr_client.exceptions.RepositoryAlreadyExistsException: + self.log.info("ECR repo {} already exists".format(repo_name)) + # TODO: check for expiration before reseting password + self.password = self._get_ecr_pawssord() + return await super().get_image_manifest(repo_name, tag) + + class FakeRegistry(DockerRegistry): """ Fake registry that contains no images From a2a670a0450e480f60925f21ad0f6cb583763c87 Mon Sep 17 00:00:00 2001 From: Chico Venancio Date: Tue, 27 Aug 2019 15:52:31 -0300 Subject: [PATCH 24/44] rbac.yaml: allow manipulation of secrets --- helm-chart/binderhub/templates/rbac.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/helm-chart/binderhub/templates/rbac.yaml b/helm-chart/binderhub/templates/rbac.yaml index 5716b5abf..643fd9051 100644 --- a/helm-chart/binderhub/templates/rbac.yaml +++ b/helm-chart/binderhub/templates/rbac.yaml @@ -14,6 +14,10 @@ rules: - apiGroups: [""] resources: ["pods/log"] verbs: ["get"] +- apiGroups: + - "" + resources: ["secrets"] + verbs: ["get", "patch"] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 From 25faed2c5e85c6dfe64c544db5caabc47230a510 Mon Sep 17 00:00:00 2001 From: Ivan Gomes Date: Mon, 20 Jan 2020 16:45:16 -0800 Subject: [PATCH 25/44] Revert RBAC modification and remove handling in AWSElasticContainerRegistry for secret management --- binderhub/registry.py | 46 +++++------------------- helm-chart/binderhub/templates/rbac.yaml | 4 --- 2 files changed, 9 insertions(+), 41 deletions(-) diff --git a/binderhub/registry.py b/binderhub/registry.py index daf262512..d804538ba 100644 --- a/binderhub/registry.py +++ b/binderhub/registry.py @@ -7,8 +7,6 @@ from urllib.parse import urlparse import boto3 -import kubernetes.client -import kubernetes.config from tornado import httpclient from tornado.httputil import url_concat from traitlets import Dict, Unicode, default, Any @@ -244,53 +242,27 @@ class AWSElasticContainerRegistry(DockerRegistry): """, ) + username = "AWS" + + # fetch password every time as ECR password expires every 12 hours + # and is refreshed by k8s externally + @property + def password(self): + return super()._default_password() + ecr_client = Any() @default("ecr_client") def _get_ecr_client(self): return boto3.client("ecr", region_name=self.aws_region) - username = "AWS" - - kubernetes.config.load_incluster_config() - kube_client = kubernetes.client.CoreV1Api() - - def _get_ecr_auth(self): - return self.ecr_client.get_authorization_token()["authorizationData"][0] - - @default("url") - def _default_url(self): - return self._get_ecr_auth()["proxyEndpoint"] - - def _patch_docker_config_secret(self, auth): - """Patch binder-push-secret""" - secret_data = {"auths": {self.url: {"auth": auth["authorizationToken"]}}} - secret_data = base64.b64encode(json.dumps(secret_data).encode("utf8")).decode( - "utf8" - ) - with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as f: - namespace = f.read() - self.kube_client.patch_namespaced_secret( - "binder-push-secret", namespace, {"data": {"config.json": secret_data}} - ) - - @default("password") - def _get_ecr_pawssord(self): - """Get ecr password""" - auth = self._get_ecr_auth() - self.password_expires = auth["expiresAt"] - self._patch_docker_config_secret(auth) - return base64.b64decode(auth['authorizationToken']).decode("utf-8").split(':')[1] - async def get_image_manifest(self, image, tag): try: repo_name = image.split("/", 1)[1] self.ecr_client.create_repository(repositoryName=repo_name) - self.log.info("Creating ECR repo {}".format(repo_name)) + self.log.info("ECR repo {} created".format(repo_name)) except self.ecr_client.exceptions.RepositoryAlreadyExistsException: self.log.info("ECR repo {} already exists".format(repo_name)) - # TODO: check for expiration before reseting password - self.password = self._get_ecr_pawssord() return await super().get_image_manifest(repo_name, tag) diff --git a/helm-chart/binderhub/templates/rbac.yaml b/helm-chart/binderhub/templates/rbac.yaml index 643fd9051..5716b5abf 100644 --- a/helm-chart/binderhub/templates/rbac.yaml +++ b/helm-chart/binderhub/templates/rbac.yaml @@ -14,10 +14,6 @@ rules: - apiGroups: [""] resources: ["pods/log"] verbs: ["get"] -- apiGroups: - - "" - resources: ["secrets"] - verbs: ["get", "patch"] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 From 60c8ed4db3f46481abdbe894bc9608f44abeff59 Mon Sep 17 00:00:00 2001 From: Ivan Gomes Date: Mon, 20 Jan 2020 18:52:38 -0800 Subject: [PATCH 26/44] Add password and url fetching for ECR --- binderhub/registry.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/binderhub/registry.py b/binderhub/registry.py index d804538ba..7422ed324 100644 --- a/binderhub/registry.py +++ b/binderhub/registry.py @@ -242,20 +242,28 @@ class AWSElasticContainerRegistry(DockerRegistry): """, ) - username = "AWS" - - # fetch password every time as ECR password expires every 12 hours - # and is refreshed by k8s externally - @property - def password(self): - return super()._default_password() - ecr_client = Any() @default("ecr_client") def _get_ecr_client(self): return boto3.client("ecr", region_name=self.aws_region) + # TODO: cache auth if not expired - authorizationData.0.expiresAt + def _get_ecr_auth(self): + return self.ecr_client.get_authorization_token()["authorizationData"][0] + + username = "AWS" + + @property + def password(self): + # Fetch password every time as ECR password expires + auth = self._get_ecr_auth() + return base64.b64decode(auth['authorizationToken']).decode("utf-8").split(':')[1] + + @default("url") + def _default_url(self): + return self._get_ecr_auth()["proxyEndpoint"] + async def get_image_manifest(self, image, tag): try: repo_name = image.split("/", 1)[1] From 6e6a61c50cd48c26e47e8a0d27549fabac652619 Mon Sep 17 00:00:00 2001 From: Ivan Gomes Date: Mon, 20 Jan 2020 21:46:10 -0800 Subject: [PATCH 27/44] Add AWS ECR documentation --- doc/zero-to-binderhub/setup-binderhub.rst | 47 ++++++++++++ doc/zero-to-binderhub/setup-registry.rst | 91 +++++++++++++++++++++++ 2 files changed, 138 insertions(+) diff --git a/doc/zero-to-binderhub/setup-binderhub.rst b/doc/zero-to-binderhub/setup-binderhub.rst index aa1681bdd..ca680dbca 100644 --- a/doc/zero-to-binderhub/setup-binderhub.rst +++ b/doc/zero-to-binderhub/setup-binderhub.rst @@ -107,6 +107,22 @@ where: * `` is the Harbor username * `` is the Harbor password +If you are using Amazon Elastic Container Registry +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Update `secret.yaml` to include the following:: + + registry: + url: https://.dkr.ecr..amazonaws.com + +where: + +* ```` is the identifier of your AWS account +* ```` is the AWS region of the ECR registry, e.g. ``us-east-1`` + +As ECR uses AWS IAM for authorization, specifying ``username`` and ``password`` +is not necessary. + Create ``config.yaml`` ---------------------- @@ -205,6 +221,37 @@ As an example, the config should look like the following:: token_url: https://abcde.gra7.container-registry.ovh.net/service/token?service=harbor-registry +If you are using Amazon Elastic Container Registry +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you want your BinderHub to push and pull images from an Amazon Elastic +Container Registry (ECR), then your `config.yaml` file will look as follows:: + + config: + BinderHub: + use_registry: true + registry_class: binderhub.registry.AWSElasticContainerRegistry + image_prefix: "-" + AWSElasticContainerRegistry: + aws_region: + +where: + +* ```` is the AWS region of the ECR registry, e.g. ``us-east-1``. +* ```` can be any string, and will be prepended to image names. We + recommend something descriptive such as ``binder-dev-`` or ``binder-prod-`` + (ending with a `-` is useful). + +If you opted to use an IAM User with programmatic access instead of assuming +the role in the previous step you will additionally need to add the following +to your `config.yaml`:: + + extraEnv: + - name: AWS_ACCESS_KEY_ID + value: "xxx" + - name: AWS_SECRET_ACCESS_KEY + value: "yyy" + If you are using a custom registry ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/zero-to-binderhub/setup-registry.rst b/doc/zero-to-binderhub/setup-registry.rst index afbcbcfa3..e3e424869 100644 --- a/doc/zero-to-binderhub/setup-registry.rst +++ b/doc/zero-to-binderhub/setup-registry.rst @@ -167,6 +167,97 @@ To use the OVH Container Registry, log in to the `OVH Control Panel `_ +.. _use-ecr: + +Set up Amazon Elastic Container Registry +---------------------------------------- + +To use Amazon Elastic Container Registry (ECR), you'll need to use AWS IAM to +authorize the machine or pod running BinderHub so it can push images. There +are a number of options on how to do this with IAM and Kubernetes, but we +will highlight two: assume IAM role and IAM user with programmatic access. + +Start by creating an IAM policy that grants access to create repositories and +read/write images from them. An example IAM permissions policy is provided +below. For more information and examples see `Identity and Access Management for Amazon Elastic Container Registry `_. + +.. code-block:: json + + { + "Statement": [ + { + "Action": [ + "ecr:ListImages" + ], + "Effect": "Allow", + "Resource": "arn:aws:ecr:::-*", + "Sid": "ListImagesInRepository" + }, + { + "Action": [ + "ecr:GetAuthorizationToken" + ], + "Effect": "Allow", + "Resource": "*", + "Sid": "GetAuthorizationToken" + }, + { + "Action": [ + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:DescribeImages", + "ecr:BatchGetImage", + "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", + "ecr:CompleteLayerUpload", + "ecr:PutImage" + ], + "Effect": "Allow", + "Resource": "arn:aws:ecr:::-*", + "Sid": "ManageRepositoryContents" + }, + { + "Action": [ + "ecr:CreateRepository" + ], + "Effect": "Allow", + "Resource": "arn:aws:ecr:::-*", + "Sid": "CreateRepository" + }, + ], + "Version": "2012-10-17" + } + +If you are using AWS EC2, EKS, etc. to set up your Kubernetes cluster you can +add this policy to the IAM Role assumed by the nodes of the cluster, e.g. +``nodes..k8s.local`` if you followed `Zero to JupyterHub with Kubernetes `_ +and used kops. The IAM permissions policy will need to accompanied with an IAM +trust policy to allow it to be assumed. An example for EC2 is provided below. +For more information see `Granting a User Permissions to Pass a Role to an AWS Service `_. +This is the recommended method if your Kubernetes cluster is provisioned on +AWS. + +.. code-block:: json + + { + "Version": "2012-10-17", + "Statement": { + "Sid": "TrustPolicyStatementThatAllowsEC2ServiceToAssumeTheAttachedRole", + "Effect": "Allow", + "Principal": { "Service": "ec2.amazonaws.com" }, + "Action": "sts:AssumeRole" + } + } + +Alternatively, you can create an IAM user with programmatic access (see +`Creating an IAM User in Your AWS Account `_) +and specify the ``AWS_ACCESS_KEY_ID`` and ``AWS_SECRET_ACCESS_KEY`` environment +variables in the following step. + Next step --------- From b9877a71c50eab3d4a0502a29650d7a5618ec859 Mon Sep 17 00:00:00 2001 From: Ivan Gomes Date: Mon, 20 Jan 2020 21:50:03 -0800 Subject: [PATCH 28/44] Fix wording in AWS ECR documentation --- doc/zero-to-binderhub/setup-registry.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/zero-to-binderhub/setup-registry.rst b/doc/zero-to-binderhub/setup-registry.rst index e3e424869..37a335b22 100644 --- a/doc/zero-to-binderhub/setup-registry.rst +++ b/doc/zero-to-binderhub/setup-registry.rst @@ -232,8 +232,8 @@ below. For more information and examples see `Identity and Access Management for "Version": "2012-10-17" } -If you are using AWS EC2, EKS, etc. to set up your Kubernetes cluster you can -add this policy to the IAM Role assumed by the nodes of the cluster, e.g. +If you used AWS services like EC2 or EKS to set up your Kubernetes cluster you +can add this policy to the IAM Role assumed by the nodes of the cluster, e.g. ``nodes..k8s.local`` if you followed `Zero to JupyterHub with Kubernetes `_ and used kops. The IAM permissions policy will need to accompanied with an IAM trust policy to allow it to be assumed. An example for EC2 is provided below. From a5f21e8423958123070c08dd8258594ed306d723 Mon Sep 17 00:00:00 2001 From: Ivan Gomes Date: Tue, 21 Jan 2020 09:13:14 -0800 Subject: [PATCH 29/44] Add ECR host to `image_prefix` in documentation --- doc/zero-to-binderhub/setup-binderhub.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/zero-to-binderhub/setup-binderhub.rst b/doc/zero-to-binderhub/setup-binderhub.rst index ca680dbca..27bf1093d 100644 --- a/doc/zero-to-binderhub/setup-binderhub.rst +++ b/doc/zero-to-binderhub/setup-binderhub.rst @@ -231,13 +231,14 @@ Container Registry (ECR), then your `config.yaml` file will look as follows:: BinderHub: use_registry: true registry_class: binderhub.registry.AWSElasticContainerRegistry - image_prefix: "-" + image_prefix: ".dkr.ecr..amazonaws.com/-" AWSElasticContainerRegistry: - aws_region: + aws_region: where: -* ```` is the AWS region of the ECR registry, e.g. ``us-east-1``. +* ```` is the identifier of your AWS account +* ```` is the AWS region of the ECR registry, e.g. ``us-east-1``. * ```` can be any string, and will be prepended to image names. We recommend something descriptive such as ``binder-dev-`` or ``binder-prod-`` (ending with a `-` is useful). From 54c1b64eec7f94e6697e60201593e475ca5a3a9a Mon Sep 17 00:00:00 2001 From: Ivan Gomes Date: Tue, 21 Jan 2020 11:18:57 -0800 Subject: [PATCH 30/44] Re-add binder-push-secret patching. Add check for correct auth token. --- binderhub/registry.py | 28 +++++++++++++++++++----- helm-chart/binderhub/templates/rbac.yaml | 3 +++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/binderhub/registry.py b/binderhub/registry.py index 7422ed324..05be11417 100644 --- a/binderhub/registry.py +++ b/binderhub/registry.py @@ -7,6 +7,8 @@ from urllib.parse import urlparse import boto3 +import kubernetes.client +import kubernetes.config from tornado import httpclient from tornado.httputil import url_concat from traitlets import Dict, Unicode, default, Any @@ -248,9 +250,12 @@ class AWSElasticContainerRegistry(DockerRegistry): def _get_ecr_client(self): return boto3.client("ecr", region_name=self.aws_region) - # TODO: cache auth if not expired - authorizationData.0.expiresAt + # TODO: cache auth if not expired - authorizationData[i]["expiresAt"] def _get_ecr_auth(self): - return self.ecr_client.get_authorization_token()["authorizationData"][0] + auths = self.ecr_client.get_authorization_token()["authorizationData"] + auth = next(x for x in auths if x["proxyEndpoint"] == self.url) + self._patch_docker_config_secret(auth) + return auth username = "AWS" @@ -260,10 +265,6 @@ def password(self): auth = self._get_ecr_auth() return base64.b64decode(auth['authorizationToken']).decode("utf-8").split(':')[1] - @default("url") - def _default_url(self): - return self._get_ecr_auth()["proxyEndpoint"] - async def get_image_manifest(self, image, tag): try: repo_name = image.split("/", 1)[1] @@ -273,6 +274,21 @@ async def get_image_manifest(self, image, tag): self.log.info("ECR repo {} already exists".format(repo_name)) return await super().get_image_manifest(repo_name, tag) + kubernetes.config.load_incluster_config() + kube_client = kubernetes.client.CoreV1Api() + + def _patch_docker_config_secret(self, auth): + """Patch binder-push-secret""" + secret_data = {"auths": {self.url: {"auth": auth["authorizationToken"]}}} + secret_data = base64.b64encode(json.dumps(secret_data).encode("utf8")).decode( + "utf8" + ) + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as f: + namespace = f.read() + self.kube_client.patch_namespaced_secret( + "binder-push-secret", namespace, {"data": {"config.json": secret_data}} + ) + class FakeRegistry(DockerRegistry): """ diff --git a/helm-chart/binderhub/templates/rbac.yaml b/helm-chart/binderhub/templates/rbac.yaml index 5716b5abf..05c9cb6c2 100644 --- a/helm-chart/binderhub/templates/rbac.yaml +++ b/helm-chart/binderhub/templates/rbac.yaml @@ -14,6 +14,9 @@ rules: - apiGroups: [""] resources: ["pods/log"] verbs: ["get"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "patch"] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 From 6266cddbe18082a4f8b10767a27e53d3b04541e3 Mon Sep 17 00:00:00 2001 From: Ivan Date: Thu, 30 Jan 2020 10:17:15 -0800 Subject: [PATCH 31/44] Remove duplicate GetAuthorizationToken in AWS ECR doc --- doc/zero-to-binderhub/setup-registry.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/zero-to-binderhub/setup-registry.rst b/doc/zero-to-binderhub/setup-registry.rst index 37a335b22..063920514 100644 --- a/doc/zero-to-binderhub/setup-registry.rst +++ b/doc/zero-to-binderhub/setup-registry.rst @@ -203,7 +203,6 @@ below. For more information and examples see `Identity and Access Management for }, { "Action": [ - "ecr:GetAuthorizationToken", "ecr:BatchCheckLayerAvailability", "ecr:GetDownloadUrlForLayer", "ecr:GetRepositoryPolicy", From dc676c474dba0fd7590bef26c7adba6acb6bfdb3 Mon Sep 17 00:00:00 2001 From: Ivan Gomes Date: Sat, 8 Feb 2020 20:37:29 -0800 Subject: [PATCH 32/44] Lazily initialize kube_client in AWSElasticContainerRegistry to prevent ConfigException when outside k8s --- binderhub/registry.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/binderhub/registry.py b/binderhub/registry.py index 05be11417..a4d450664 100644 --- a/binderhub/registry.py +++ b/binderhub/registry.py @@ -274,8 +274,12 @@ async def get_image_manifest(self, image, tag): self.log.info("ECR repo {} already exists".format(repo_name)) return await super().get_image_manifest(repo_name, tag) - kubernetes.config.load_incluster_config() - kube_client = kubernetes.client.CoreV1Api() + kube_client = Any() + + @default("kube_client") + def _get_kube_client(self): + kubernetes.config.load_incluster_config() + return kubernetes.client.CoreV1Api() def _patch_docker_config_secret(self, auth): """Patch binder-push-secret""" From 5a4894af631c755160112e1c07828ab41ad3e093 Mon Sep 17 00:00:00 2001 From: Ivan Gomes Date: Tue, 12 May 2020 21:10:14 -0700 Subject: [PATCH 33/44] Rename `registry_class` to `docker_registry_class` for clarity --- doc/zero-to-binderhub/setup-binderhub.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/zero-to-binderhub/setup-binderhub.rst b/doc/zero-to-binderhub/setup-binderhub.rst index 27bf1093d..3caa5e0b2 100644 --- a/doc/zero-to-binderhub/setup-binderhub.rst +++ b/doc/zero-to-binderhub/setup-binderhub.rst @@ -230,7 +230,7 @@ Container Registry (ECR), then your `config.yaml` file will look as follows:: config: BinderHub: use_registry: true - registry_class: binderhub.registry.AWSElasticContainerRegistry + docker_registry_class: binderhub.registry.AWSElasticContainerRegistry image_prefix: ".dkr.ecr..amazonaws.com/-" AWSElasticContainerRegistry: aws_region: From eeb7661373695f7925b378bd056fcb1d709248c9 Mon Sep 17 00:00:00 2001 From: Ivan Gomes Date: Fri, 5 Jun 2020 17:37:47 -0700 Subject: [PATCH 34/44] Move boto3 and kubernetes calls to own threadpool --- binderhub/registry.py | 75 +++++++++++++++++++++++++++++++------------ 1 file changed, 54 insertions(+), 21 deletions(-) diff --git a/binderhub/registry.py b/binderhub/registry.py index a4d450664..69e49c98a 100644 --- a/binderhub/registry.py +++ b/binderhub/registry.py @@ -1,7 +1,9 @@ """ Interaction with the Docker Registry """ +import asyncio import base64 +from concurrent.futures import ThreadPoolExecutor import json import os from urllib.parse import urlparse @@ -11,7 +13,7 @@ import kubernetes.config from tornado import httpclient from tornado.httputil import url_concat -from traitlets import Dict, Unicode, default, Any +from traitlets import Dict, Unicode, default, Any, Integer from traitlets.config import LoggingConfigurable DEFAULT_DOCKER_REGISTRY_URL = "https://registry-1.docker.io" @@ -250,29 +252,35 @@ class AWSElasticContainerRegistry(DockerRegistry): def _get_ecr_client(self): return boto3.client("ecr", region_name=self.aws_region) - # TODO: cache auth if not expired - authorizationData[i]["expiresAt"] - def _get_ecr_auth(self): - auths = self.ecr_client.get_authorization_token()["authorizationData"] - auth = next(x for x in auths if x["proxyEndpoint"] == self.url) - self._patch_docker_config_secret(auth) - return auth - username = "AWS" - @property - def password(self): - # Fetch password every time as ECR password expires - auth = self._get_ecr_auth() - return base64.b64decode(auth['authorizationToken']).decode("utf-8").split(':')[1] + executor_threads = Integer( + 5, + config=True, + help="""The number of threads to use for blocking calls - async def get_image_manifest(self, image, tag): - try: - repo_name = image.split("/", 1)[1] - self.ecr_client.create_repository(repositoryName=repo_name) - self.log.info("ECR repo {} created".format(repo_name)) - except self.ecr_client.exceptions.RepositoryAlreadyExistsException: - self.log.info("ECR repo {} already exists".format(repo_name)) - return await super().get_image_manifest(repo_name, tag) + Should generaly be a small number because we don't + care about high concurrency here, just not blocking the webserver. + This executor is not used for long-running tasks (e.g. builds). + """, + ) + + executor_threads = Integer( + 5, + config=True, + help="""The number of threads to use for blocking calls + + Should generaly be a small number because we don't + care about high concurrency here, just not blocking the webserver. + This executor is not used for long-running tasks (e.g. builds). + """, + ) + + executor = Any() + + @default("executor") + def _get_executor(self): + return ThreadPoolExecutor(self.executor_threads) kube_client = Any() @@ -280,6 +288,31 @@ async def get_image_manifest(self, image, tag): def _get_kube_client(self): kubernetes.config.load_incluster_config() return kubernetes.client.CoreV1Api() + + async def get_image_manifest(self, image, tag): + image = image.split("/", 1)[1] + await asyncio.wrap_future(self.executor.submit(self._pre_get_image_manifest, image, tag)) + return await super().get_image_manifest(image, tag) + + def _pre_get_image_manifest(self, image, tag): + self._create_repository(image, tag) + self._refresh_password() + + def _create_repository(self, image, tag): + try: + self.ecr_client.create_repository(repositoryName=image) + self.log.info("ECR repo {} created".format(image)) + except self.ecr_client.exceptions.RepositoryAlreadyExistsException: + self.log.info("ECR repo {} already exists".format(image)) + + # An IAM principal is used to generate an auth token that is valid for 12 hours + # ref: https://docs.aws.amazon.com/AmazonECR/latest/userguide/Registries.html + # TODO: cache auth if not expired - authorizationData[i]["expiresAt"] + def _refresh_password(self): + auths = self.ecr_client.get_authorization_token()["authorizationData"] + auth = next(x for x in auths if x["proxyEndpoint"] == self.url) + self._patch_docker_config_secret(auth) + self.password = base64.b64decode(auth['authorizationToken']).decode("utf-8").split(':')[1] def _patch_docker_config_secret(self, auth): """Patch binder-push-secret""" From ad97b5acc2244e6bc29d4dd9da99bacc564aa2e1 Mon Sep 17 00:00:00 2001 From: Tomas Beuzen Date: Fri, 18 Sep 2020 10:42:39 -0700 Subject: [PATCH 35/44] general word-smithing for clarity --- doc/zero-to-binderhub/setup-registry.rst | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/doc/zero-to-binderhub/setup-registry.rst b/doc/zero-to-binderhub/setup-registry.rst index 063920514..a2f2171ed 100644 --- a/doc/zero-to-binderhub/setup-registry.rst +++ b/doc/zero-to-binderhub/setup-registry.rst @@ -175,11 +175,12 @@ Set up Amazon Elastic Container Registry To use Amazon Elastic Container Registry (ECR), you'll need to use AWS IAM to authorize the machine or pod running BinderHub so it can push images. There are a number of options on how to do this with IAM and Kubernetes, but we -will highlight two: assume IAM role and IAM user with programmatic access. +will highlight two: define and assign an IAM role, or assume an IAM user with programmatic access. Start by creating an IAM policy that grants access to create repositories and -read/write images from them. An example IAM permissions policy is provided -below. For more information and examples see `Identity and Access Management for Amazon Elastic Container Registry `_. +read/write images from them. You can create policies using the AWS console, CLI +or API as detailed in the documentation `Creating IAM policies `_. +An example IAM permissions policy is provided below. For more information and examples see `Identity and Access Management for Amazon Elastic Container Registry `_. .. code-block:: json @@ -234,11 +235,13 @@ below. For more information and examples see `Identity and Access Management for If you used AWS services like EC2 or EKS to set up your Kubernetes cluster you can add this policy to the IAM Role assumed by the nodes of the cluster, e.g. ``nodes..k8s.local`` if you followed `Zero to JupyterHub with Kubernetes `_ -and used kops. The IAM permissions policy will need to accompanied with an IAM -trust policy to allow it to be assumed. An example for EC2 is provided below. -For more information see `Granting a User Permissions to Pass a Role to an AWS Service `_. -This is the recommended method if your Kubernetes cluster is provisioned on -AWS. +and used kops. One way to do this from the AWS Console is to navigate to the IAM service and click "Roles" in the side bar, +then find and select the IAM Role assumed by the nodes of your cluster, click "Permissions" and then "Attach policies" to attach +the IAM Policy we just created. The IAM permissions policy will need to accompanied with an IAM +trust policy to allow it to be assumed. A suitable trust policy may already be defined for your node's IAM Role, +you can view and edit the trust policy by clicking the "Trust relationships" tab is next to the "Permissions" tab in the AWS console. +An example trust policy for EC2 is provided below. For more information see `Granting a User Permissions to Pass a Role to an AWS Service `_. +This is the recommended method if your Kubernetes cluster is provisioned on AWS. .. code-block:: json @@ -252,7 +255,7 @@ AWS. } } -Alternatively, you can create an IAM user with programmatic access (see +Alternatively to the above steps, you can create an IAM user with programmatic access (see `Creating an IAM User in Your AWS Account `_) and specify the ``AWS_ACCESS_KEY_ID`` and ``AWS_SECRET_ACCESS_KEY`` environment variables in the following step. From e7624d98346edbedda33ab31809efcd5b6b48fbb Mon Sep 17 00:00:00 2001 From: Tomas Beuzen Date: Fri, 18 Sep 2020 10:43:06 -0700 Subject: [PATCH 36/44] remove stray comma that AWS doesn't like in the policy --- doc/zero-to-binderhub/setup-registry.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/zero-to-binderhub/setup-registry.rst b/doc/zero-to-binderhub/setup-registry.rst index a2f2171ed..c1e4f9765 100644 --- a/doc/zero-to-binderhub/setup-registry.rst +++ b/doc/zero-to-binderhub/setup-registry.rst @@ -227,7 +227,7 @@ An example IAM permissions policy is provided below. For more information and ex "Effect": "Allow", "Resource": "arn:aws:ecr:::-*", "Sid": "CreateRepository" - }, + } ], "Version": "2012-10-17" } From 378f539fe8a252d129ca11490aa2c8fb22d3899c Mon Sep 17 00:00:00 2001 From: Tomas Beuzen Date: Fri, 18 Sep 2020 10:45:32 -0700 Subject: [PATCH 37/44] one more small clarification --- doc/zero-to-binderhub/setup-registry.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/zero-to-binderhub/setup-registry.rst b/doc/zero-to-binderhub/setup-registry.rst index c1e4f9765..5f561e742 100644 --- a/doc/zero-to-binderhub/setup-registry.rst +++ b/doc/zero-to-binderhub/setup-registry.rst @@ -177,7 +177,7 @@ authorize the machine or pod running BinderHub so it can push images. There are a number of options on how to do this with IAM and Kubernetes, but we will highlight two: define and assign an IAM role, or assume an IAM user with programmatic access. -Start by creating an IAM policy that grants access to create repositories and +For the former, start by creating an IAM policy that grants access to create repositories and read/write images from them. You can create policies using the AWS console, CLI or API as detailed in the documentation `Creating IAM policies `_. An example IAM permissions policy is provided below. For more information and examples see `Identity and Access Management for Amazon Elastic Container Registry `_. From 762443e838520b6888d8e2eb9b178ffa28241caa Mon Sep 17 00:00:00 2001 From: thomas-bc Date: Thu, 11 Aug 2022 11:37:33 -0700 Subject: [PATCH 38/44] Fix rebase mishap --- binderhub/registry.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/binderhub/registry.py b/binderhub/registry.py index 69e49c98a..42540d481 100644 --- a/binderhub/registry.py +++ b/binderhub/registry.py @@ -265,17 +265,6 @@ def _get_ecr_client(self): """, ) - executor_threads = Integer( - 5, - config=True, - help="""The number of threads to use for blocking calls - - Should generaly be a small number because we don't - care about high concurrency here, just not blocking the webserver. - This executor is not used for long-running tasks (e.g. builds). - """, - ) - executor = Any() @default("executor") From 40ddeeb556b342c14bb0dd2d7db12111e8b1e7c9 Mon Sep 17 00:00:00 2001 From: thomas-bc Date: Thu, 11 Aug 2022 11:45:20 -0700 Subject: [PATCH 39/44] update registry_class documentation for ECR --- doc/zero-to-binderhub/setup-binderhub.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/zero-to-binderhub/setup-binderhub.rst b/doc/zero-to-binderhub/setup-binderhub.rst index 3caa5e0b2..27bf1093d 100644 --- a/doc/zero-to-binderhub/setup-binderhub.rst +++ b/doc/zero-to-binderhub/setup-binderhub.rst @@ -230,7 +230,7 @@ Container Registry (ECR), then your `config.yaml` file will look as follows:: config: BinderHub: use_registry: true - docker_registry_class: binderhub.registry.AWSElasticContainerRegistry + registry_class: binderhub.registry.AWSElasticContainerRegistry image_prefix: ".dkr.ecr..amazonaws.com/-" AWSElasticContainerRegistry: aws_region: From 42a28bf869006dc5bb3a29d0efb4d53a27cd0479 Mon Sep 17 00:00:00 2001 From: thomas-bc Date: Thu, 11 Aug 2022 12:20:18 -0700 Subject: [PATCH 40/44] moving optional boto3 import inside class --- binderhub/registry.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/binderhub/registry.py b/binderhub/registry.py index 42540d481..9102127a2 100644 --- a/binderhub/registry.py +++ b/binderhub/registry.py @@ -8,7 +8,6 @@ import os from urllib.parse import urlparse -import boto3 import kubernetes.client import kubernetes.config from tornado import httpclient @@ -239,6 +238,8 @@ async def get_image_manifest(self, image, tag): class AWSElasticContainerRegistry(DockerRegistry): + import boto3 + aws_region = Unicode( config=True, help=""" From 7cf26bf016d73ae11827171ff254c08eb983fed5 Mon Sep 17 00:00:00 2001 From: thomas-bc Date: Thu, 11 Aug 2022 18:46:23 -0700 Subject: [PATCH 41/44] _patch_docker_config_secret docs and robustness --- binderhub/registry.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/binderhub/registry.py b/binderhub/registry.py index 9102127a2..d1844a079 100644 --- a/binderhub/registry.py +++ b/binderhub/registry.py @@ -302,10 +302,11 @@ def _refresh_password(self): auths = self.ecr_client.get_authorization_token()["authorizationData"] auth = next(x for x in auths if x["proxyEndpoint"] == self.url) self._patch_docker_config_secret(auth) - self.password = base64.b64decode(auth['authorizationToken']).decode("utf-8").split(':')[1] - + self.password = base64.b64decode(auth["authorizationToken"]).decode("utf-8").split(':')[1] + def _patch_docker_config_secret(self, auth): - """Patch binder-push-secret""" + """Patch push_secret. Necessary because AWS rotates auth tokens. + ref: https://docs.aws.amazon.com/AmazonECR/latest/userguide/Registries.html """ secret_data = {"auths": {self.url: {"auth": auth["authorizationToken"]}}} secret_data = base64.b64encode(json.dumps(secret_data).encode("utf8")).decode( "utf8" @@ -313,7 +314,7 @@ def _patch_docker_config_secret(self, auth): with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as f: namespace = f.read() self.kube_client.patch_namespaced_secret( - "binder-push-secret", namespace, {"data": {"config.json": secret_data}} + self.parent.push_secret, namespace, {"data": {"config.json": secret_data}} ) From d9a983f4e2859f77fb63b453cc417ae1d5f847a3 Mon Sep 17 00:00:00 2001 From: thomas-bc Date: Thu, 11 Aug 2022 19:21:41 -0700 Subject: [PATCH 42/44] make RBAC increase conditional --- doc/zero-to-binderhub/setup-binderhub.rst | 2 ++ helm-chart/binderhub/schema.yaml | 7 +++++++ helm-chart/binderhub/templates/rbac.yaml | 2 ++ 3 files changed, 11 insertions(+) diff --git a/doc/zero-to-binderhub/setup-binderhub.rst b/doc/zero-to-binderhub/setup-binderhub.rst index 27bf1093d..e7aa6d86f 100644 --- a/doc/zero-to-binderhub/setup-binderhub.rst +++ b/doc/zero-to-binderhub/setup-binderhub.rst @@ -227,6 +227,8 @@ If you are using Amazon Elastic Container Registry If you want your BinderHub to push and pull images from an Amazon Elastic Container Registry (ECR), then your `config.yaml` file will look as follows:: + rbac: + augmented: true config: BinderHub: use_registry: true diff --git a/helm-chart/binderhub/schema.yaml b/helm-chart/binderhub/schema.yaml index f55ffd1df..5f7e2e12b 100644 --- a/helm-chart/binderhub/schema.yaml +++ b/helm-chart/binderhub/schema.yaml @@ -88,6 +88,13 @@ properties: description: | Decides if RBAC resources are to be created and referenced by the the Helm chart's workloads. + augmented: + type: boolean + description: | + Allows manipulation of Secrets if set to true. + This should only be needed if using the AWSElasticContainerRegistry + registry class, because AWS rotates auth tokens. See [the + documentation](https://docs.aws.amazon.com/AmazonECR/latest/userguide/registry_auth.html) nodeSelector: &nodeSelector-spec type: object diff --git a/helm-chart/binderhub/templates/rbac.yaml b/helm-chart/binderhub/templates/rbac.yaml index 05c9cb6c2..958bb7650 100644 --- a/helm-chart/binderhub/templates/rbac.yaml +++ b/helm-chart/binderhub/templates/rbac.yaml @@ -14,9 +14,11 @@ rules: - apiGroups: [""] resources: ["pods/log"] verbs: ["get"] +{{- if .Values.rbac.augmented -}} - apiGroups: [""] resources: ["secrets"] verbs: ["get", "patch"] +{{- end }} --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 From 591be68e34d43faa7b9fd9c7829f5ecc5b9790e8 Mon Sep 17 00:00:00 2001 From: thomas-bc Date: Tue, 16 Aug 2022 14:43:40 -0700 Subject: [PATCH 43/44] rename RBAC increase to rbac.patchSecrets --- doc/zero-to-binderhub/setup-binderhub.rst | 2 +- helm-chart/binderhub/schema.yaml | 4 ++-- helm-chart/binderhub/templates/rbac.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/zero-to-binderhub/setup-binderhub.rst b/doc/zero-to-binderhub/setup-binderhub.rst index e7aa6d86f..198a6a2b9 100644 --- a/doc/zero-to-binderhub/setup-binderhub.rst +++ b/doc/zero-to-binderhub/setup-binderhub.rst @@ -228,7 +228,7 @@ If you want your BinderHub to push and pull images from an Amazon Elastic Container Registry (ECR), then your `config.yaml` file will look as follows:: rbac: - augmented: true + patchSecrets: true config: BinderHub: use_registry: true diff --git a/helm-chart/binderhub/schema.yaml b/helm-chart/binderhub/schema.yaml index 5f7e2e12b..05449632d 100644 --- a/helm-chart/binderhub/schema.yaml +++ b/helm-chart/binderhub/schema.yaml @@ -88,10 +88,10 @@ properties: description: | Decides if RBAC resources are to be created and referenced by the the Helm chart's workloads. - augmented: + patchSecrets: type: boolean description: | - Allows manipulation of Secrets if set to true. + Allows get and patch Secrets if set to true. This should only be needed if using the AWSElasticContainerRegistry registry class, because AWS rotates auth tokens. See [the documentation](https://docs.aws.amazon.com/AmazonECR/latest/userguide/registry_auth.html) diff --git a/helm-chart/binderhub/templates/rbac.yaml b/helm-chart/binderhub/templates/rbac.yaml index 958bb7650..54bc0fced 100644 --- a/helm-chart/binderhub/templates/rbac.yaml +++ b/helm-chart/binderhub/templates/rbac.yaml @@ -14,7 +14,7 @@ rules: - apiGroups: [""] resources: ["pods/log"] verbs: ["get"] -{{- if .Values.rbac.augmented -}} +{{- if .Values.rbac.patchSecrets -}} - apiGroups: [""] resources: ["secrets"] verbs: ["get", "patch"] From 8a25e45f7c1cd1d9886d0cab460827c17ef484d2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Aug 2022 22:37:57 +0000 Subject: [PATCH 44/44] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- binderhub/registry.py | 18 +++++++++++------- requirements.txt | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/binderhub/registry.py b/binderhub/registry.py index d1844a079..748fe20f3 100644 --- a/binderhub/registry.py +++ b/binderhub/registry.py @@ -3,16 +3,16 @@ """ import asyncio import base64 -from concurrent.futures import ThreadPoolExecutor import json import os +from concurrent.futures import ThreadPoolExecutor from urllib.parse import urlparse import kubernetes.client import kubernetes.config from tornado import httpclient from tornado.httputil import url_concat -from traitlets import Dict, Unicode, default, Any, Integer +from traitlets import Any, Dict, Integer, Unicode, default from traitlets.config import LoggingConfigurable DEFAULT_DOCKER_REGISTRY_URL = "https://registry-1.docker.io" @@ -281,7 +281,9 @@ def _get_kube_client(self): async def get_image_manifest(self, image, tag): image = image.split("/", 1)[1] - await asyncio.wrap_future(self.executor.submit(self._pre_get_image_manifest, image, tag)) + await asyncio.wrap_future( + self.executor.submit(self._pre_get_image_manifest, image, tag) + ) return await super().get_image_manifest(image, tag) def _pre_get_image_manifest(self, image, tag): @@ -291,9 +293,9 @@ def _pre_get_image_manifest(self, image, tag): def _create_repository(self, image, tag): try: self.ecr_client.create_repository(repositoryName=image) - self.log.info("ECR repo {} created".format(image)) + self.log.info(f"ECR repo {image} created") except self.ecr_client.exceptions.RepositoryAlreadyExistsException: - self.log.info("ECR repo {} already exists".format(image)) + self.log.info(f"ECR repo {image} already exists") # An IAM principal is used to generate an auth token that is valid for 12 hours # ref: https://docs.aws.amazon.com/AmazonECR/latest/userguide/Registries.html @@ -302,11 +304,13 @@ def _refresh_password(self): auths = self.ecr_client.get_authorization_token()["authorizationData"] auth = next(x for x in auths if x["proxyEndpoint"] == self.url) self._patch_docker_config_secret(auth) - self.password = base64.b64decode(auth["authorizationToken"]).decode("utf-8").split(':')[1] + self.password = ( + base64.b64decode(auth["authorizationToken"]).decode("utf-8").split(":")[1] + ) def _patch_docker_config_secret(self, auth): """Patch push_secret. Necessary because AWS rotates auth tokens. - ref: https://docs.aws.amazon.com/AmazonECR/latest/userguide/Registries.html """ + ref: https://docs.aws.amazon.com/AmazonECR/latest/userguide/Registries.html""" secret_data = {"auths": {self.url: {"auth": auth["authorizationToken"]}}} secret_data = base64.b64encode(json.dumps(secret_data).encode("utf8")).decode( "utf8" diff --git a/requirements.txt b/requirements.txt index e5e40a632..d0be691e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +boto3 # About pycurl: # - pycurl is an optional dependency which improves performance # - pycurl requires both `curl-config` and `gcc` to be available when installing @@ -16,4 +17,3 @@ pyjwt>=2 python-json-logger tornado>=5.1 traitlets -boto3