From a31dee3fa43be70c64f267fc86f8162b9b48dce2 Mon Sep 17 00:00:00 2001 From: Isman Firmansyah Date: Tue, 21 Feb 2023 13:58:44 +0700 Subject: [PATCH] fix(docker-jans): handle failure on running API requests to Kubernetes API server in Google Cloud Run (#3893) --- .../scripts/auth_handler.py | 117 +++++++++--------- .../jans/pycloudlib/meta/kubernetes_meta.py | 34 ++++- 2 files changed, 86 insertions(+), 65 deletions(-) diff --git a/docker-jans-certmanager/scripts/auth_handler.py b/docker-jans-certmanager/scripts/auth_handler.py index 56b571ee49d..9702baa2b3c 100644 --- a/docker-jans-certmanager/scripts/auth_handler.py +++ b/docker-jans-certmanager/scripts/auth_handler.py @@ -197,7 +197,7 @@ def __init__(self, manager, dry_run, **opts): self.rotation_interval = opts.get("interval", 48) self.push_keys = as_boolean(opts.get("push-to-container", True)) self.key_strategy = opts.get("key-strategy", "OLDER") - self.privkey_push_delay = opts.get("privkey-push-delay", 0) + self.privkey_push_delay = int(opts.get("privkey-push-delay", 0)) self.privkey_push_strategy = opts.get("privkey-push-strategy", "OLDER") self.sig_keys = resolve_sig_keys(opts.get("sig-keys", SIG_KEYS)) self.enc_keys = resolve_enc_keys(opts.get("enc-keys", ENC_KEYS)) @@ -290,7 +290,7 @@ def patch(self): push_delay_invalid = False try: - if int(self.privkey_push_delay) < 0: + if self.privkey_push_delay < 0: push_delay_invalid = True except ValueError: push_delay_invalid = True @@ -354,10 +354,9 @@ def patch(self): logger.warning( "Unable to find any jans-auth container; make sure " "to deploy jans-auth and set APP_NAME=auth-server " - "label on container level" + "label on container level; Note that pushing keys to " + "containers will be skipped!" ) - # exit immediately to avoid persistence/secrets being modified - return for container in auth_containers: name = self.meta_client.get_container_name(container) @@ -367,7 +366,7 @@ def patch(self): logger.info(f"creating new {name}:{jwks_fn}") self.meta_client.copy_to_container(container, jwks_fn) - if int(self.privkey_push_delay) > 0: + if self.privkey_push_delay > 0: # delayed jks push continue @@ -397,54 +396,54 @@ def patch(self): logger.info(f"restoring backup of {name}:{jwks_fn}") self.meta_client.exec_cmd(container, f"cp {jwks_fn}.backup {jwks_fn}") - if int(self.privkey_push_delay) > 0: + if self.privkey_push_delay > 0: # delayed jks revert continue name = self.meta_client.get_container_name(container) logger.info(f"restoring backup of {name}:{jks_fn}") self.meta_client.exec_cmd(container, f"cp {jks_fn}.backup {jks_fn}") - return - self.manager.secret.set("auth_jks_base64", encode_jks(self.manager)) - self.manager.config.set("auth_key_rotated_at", int(time.time())) - self.manager.secret.set("auth_openid_jks_pass", jks_pass) - self.manager.config.set("auth_sig_keys", self.sig_keys) - self.manager.config.set("auth_enc_keys", self.enc_keys) - # jwks - self.manager.secret.set( - "auth_openid_key_base64", - generate_base64_contents(json.dumps(keys)), - ) - - # publish delayed jks - if int(self.privkey_push_delay) > 0: - logger.info(f"Waiting for private key push delay ({int(self.privkey_push_delay)} seconds) ...") - time.sleep(int(self.privkey_push_delay)) + else: + self.manager.secret.set("auth_jks_base64", encode_jks(self.manager)) + self.manager.config.set("auth_key_rotated_at", int(time.time())) + self.manager.secret.set("auth_openid_jks_pass", jks_pass) + self.manager.config.set("auth_sig_keys", self.sig_keys) + self.manager.config.set("auth_enc_keys", self.enc_keys) + # jwks + self.manager.secret.set( + "auth_openid_key_base64", + generate_base64_contents(json.dumps(keys)), + ) - for container in auth_containers: - logger.info(f"creating backup of {name}:{jks_fn}") - self.meta_client.exec_cmd(container, f"cp {jks_fn} {jks_fn}.backup") - logger.info(f"creating new {name}:{jks_fn}") - self.meta_client.copy_to_container(container, jks_fn) - - # as new JKS pushed to container, we need to tell auth-server to reload the private keys - # by increasing jansRevision again; note that as jansRevision may have been modified externally - # we need to ensure we have fresh jansRevision value to increase to - config = self.backend.get_auth_config() - rev = int(config["jansRevision"]) + 1 - conf_dynamic.update({ - "keySelectionStrategy": self.privkey_push_strategy, - }) - - logger.info(f"using keySelectionStrategy {self.privkey_push_strategy}") - - self.backend.modify_auth_config( - config["id"], - rev, - conf_dynamic, - keys, - ) + # publish delayed jks + if self.push_keys and self.privkey_push_delay > 0 and auth_containers: + logger.info(f"Waiting for private key push delay ({self.privkey_push_delay} seconds) ...") + time.sleep(self.privkey_push_delay) + + for container in auth_containers: + logger.info(f"creating backup of {name}:{jks_fn}") + self.meta_client.exec_cmd(container, f"cp {jks_fn} {jks_fn}.backup") + logger.info(f"creating new {name}:{jks_fn}") + self.meta_client.copy_to_container(container, jks_fn) + + # as new JKS pushed to container, we need to tell auth-server to reload the private keys + # by increasing jansRevision again; note that as jansRevision may have been modified externally + # we need to ensure we have fresh jansRevision value to increase to + config = self.backend.get_auth_config() + rev = int(config["jansRevision"]) + 1 + conf_dynamic.update({ + "keySelectionStrategy": self.privkey_push_strategy, + }) + + logger.info(f"using keySelectionStrategy {self.privkey_push_strategy}") + + self.backend.modify_auth_config( + config["id"], + rev, + conf_dynamic, + keys, + ) except (TypeError, ValueError,) as exc: logger.warning(f"Unable to get public keys; reason={exc}") @@ -539,7 +538,8 @@ def prune(self): logger.warning( "Unable to find any jans-auth container; make sure " "to deploy jans-auth and set APP_NAME=auth-server " - "label on container level" + "label on container level. Note that pushing keys to " + "containers will be skipped!" ) # exit immediately to avoid persistence/secrets being modified return @@ -579,18 +579,17 @@ def prune(self): self.meta_client.exec_cmd(container, f"cp {jks_fn}.backup {jks_fn}") logger.info(f"restoring backup of {name}:{jwks_fn}") self.meta_client.exec_cmd(container, f"cp {jwks_fn}.backup {jwks_fn}") - return - - self.manager.secret.set("auth_jks_base64", encode_jks(self.manager)) - self.manager.config.set("auth_key_rotated_at", int(time.time())) - self.manager.secret.set("auth_openid_jks_pass", jks_pass) - self.manager.config.set("auth_sig_keys", self.sig_keys) - self.manager.config.set("auth_enc_keys", self.enc_keys) - # jwks - self.manager.secret.set( - "auth_openid_key_base64", - generate_base64_contents(json.dumps(keys)), - ) + else: + self.manager.secret.set("auth_jks_base64", encode_jks(self.manager)) + self.manager.config.set("auth_key_rotated_at", int(time.time())) + self.manager.secret.set("auth_openid_jks_pass", jks_pass) + self.manager.config.set("auth_sig_keys", self.sig_keys) + self.manager.config.set("auth_enc_keys", self.enc_keys) + # jwks + self.manager.secret.set( + "auth_openid_key_base64", + generate_base64_contents(json.dumps(keys)), + ) except (TypeError, ValueError,) as exc: logger.warning(f"Unable to get public keys; reason={exc}") diff --git a/jans-pycloudlib/jans/pycloudlib/meta/kubernetes_meta.py b/jans-pycloudlib/jans/pycloudlib/meta/kubernetes_meta.py index c312c5eb3f7..f1e8d9d5789 100644 --- a/jans-pycloudlib/jans/pycloudlib/meta/kubernetes_meta.py +++ b/jans-pycloudlib/jans/pycloudlib/meta/kubernetes_meta.py @@ -27,7 +27,7 @@ class KubernetesMeta(BaseMeta): """A class to interact with a subset of Kubernetes APIs.""" - def __init__(self) -> None: + def __init__(self) -> None: # noqa: D107 self._client: _t.Union[kubernetes.client.CoreV1Api, None] = None self.kubeconfig_file = os.path.expanduser("~/.kube/config") @@ -38,10 +38,28 @@ def client(self) -> kubernetes.client.CoreV1Api: # config loading priority try: kubernetes.config.load_incluster_config() - except kubernetes.config.config_exception.ConfigException: - kubernetes.config.load_kube_config(self.kubeconfig_file) - self._client = kubernetes.client.CoreV1Api() - self._client.api_client.configuration.assert_hostname = False + config_loaded = True + except kubernetes.config.config_exception.ConfigException as exc: + # some cluster running restricted env (like Google Cloud Run) doesn't have + # required env vars `KUBERNETES_SERVICE_HOST` and `KUBERNETES_SERVICE_PORT_HTTPS` + logger.warning(f"Unable to load Kubernetes in-cluster config; reason={exc}") + config_loaded = False + + # try loading config from kubeconfig file + if not config_loaded: + try: + kubernetes.config.load_kube_config(self.kubeconfig_file) + config_loaded = True + except kubernetes.config.config_exception.ConfigException as exc: + logger.warning(f"Unable to load Kubernetes config from {self.kubeconfig_file}; reason={exc}") + config_loaded = False + + # set client only if config is loaded properly + if config_loaded: + self._client = kubernetes.client.CoreV1Api() + self._client.api_client.configuration.assert_hostname = False + else: + logger.warning("Kubernetes client config are not loaded properly, thus client will be disabled!") return self._client def get_containers(self, label: str) -> list[V1Pod]: @@ -57,7 +75,11 @@ def get_containers(self, label: str) -> list[V1Pod]: List of pod objects. """ namespace = os.environ.get("CN_CONTAINER_METADATA_NAMESPACE", "default") - pods: list[V1Pod] = self.client.list_namespaced_pod(namespace, label_selector=label).items + try: + pods: list[V1Pod] = self.client.list_namespaced_pod(namespace, label_selector=label).items + except AttributeError: + # client is not set due to missing k8s config + pods = [] return pods def get_container_ip(self, container: V1Pod) -> str: