Skip to content

Commit

Permalink
fix(docker-jans): handle failure on running API requests to Kubernete…
Browse files Browse the repository at this point in the history 
…s API server in Google Cloud Run (#3893)
  • Loading branch information
@iromli
iromli authored Feb 21, 2023
1 parent a8ded4a commit a31dee3
Show file tree
Hide file tree
Showing 2 changed files with 86 additions and 65 deletions.
117 changes: 58 additions & 59 deletions docker-jans-certmanager/scripts/auth_handler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ def __init__(self, manager, dry_run, **opts):
self.rotation_interval = opts.get("interval", 48)
self.push_keys = as_boolean(opts.get("push-to-container", True))
self.key_strategy = opts.get("key-strategy", "OLDER")
self.privkey_push_delay = opts.get("privkey-push-delay", 0)
self.privkey_push_delay = int(opts.get("privkey-push-delay", 0))
self.privkey_push_strategy = opts.get("privkey-push-strategy", "OLDER")
self.sig_keys = resolve_sig_keys(opts.get("sig-keys", SIG_KEYS))
self.enc_keys = resolve_enc_keys(opts.get("enc-keys", ENC_KEYS))
Expand Down Expand Up @@ -290,7 +290,7 @@ def patch(self):

push_delay_invalid = False
try:
if int(self.privkey_push_delay) < 0:
if self.privkey_push_delay < 0:
push_delay_invalid = True
except ValueError:
push_delay_invalid = True
Expand Down Expand Up @@ -354,10 +354,9 @@ def patch(self):
logger.warning(
"Unable to find any jans-auth container; make sure "
"to deploy jans-auth and set APP_NAME=auth-server "
"label on container level"
"label on container level; Note that pushing keys to "
"containers will be skipped!"
)
# exit immediately to avoid persistence/secrets being modified
return

for container in auth_containers:
name = self.meta_client.get_container_name(container)
Expand All @@ -367,7 +366,7 @@ def patch(self):
logger.info(f"creating new {name}:{jwks_fn}")
self.meta_client.copy_to_container(container, jwks_fn)

if int(self.privkey_push_delay) > 0:
if self.privkey_push_delay > 0:
# delayed jks push
continue

Expand Down Expand Up @@ -397,54 +396,54 @@ def patch(self):
logger.info(f"restoring backup of {name}:{jwks_fn}")
self.meta_client.exec_cmd(container, f"cp {jwks_fn}.backup {jwks_fn}")

if int(self.privkey_push_delay) > 0:
if self.privkey_push_delay > 0:
# delayed jks revert
continue

name = self.meta_client.get_container_name(container)
logger.info(f"restoring backup of {name}:{jks_fn}")
self.meta_client.exec_cmd(container, f"cp {jks_fn}.backup {jks_fn}")
return

self.manager.secret.set("auth_jks_base64", encode_jks(self.manager))
self.manager.config.set("auth_key_rotated_at", int(time.time()))
self.manager.secret.set("auth_openid_jks_pass", jks_pass)
self.manager.config.set("auth_sig_keys", self.sig_keys)
self.manager.config.set("auth_enc_keys", self.enc_keys)
# jwks
self.manager.secret.set(
"auth_openid_key_base64",
generate_base64_contents(json.dumps(keys)),
)

# publish delayed jks
if int(self.privkey_push_delay) > 0:
logger.info(f"Waiting for private key push delay ({int(self.privkey_push_delay)} seconds) ...")
time.sleep(int(self.privkey_push_delay))
else:
self.manager.secret.set("auth_jks_base64", encode_jks(self.manager))
self.manager.config.set("auth_key_rotated_at", int(time.time()))
self.manager.secret.set("auth_openid_jks_pass", jks_pass)
self.manager.config.set("auth_sig_keys", self.sig_keys)
self.manager.config.set("auth_enc_keys", self.enc_keys)
# jwks
self.manager.secret.set(
"auth_openid_key_base64",
generate_base64_contents(json.dumps(keys)),
)

for container in auth_containers:
logger.info(f"creating backup of {name}:{jks_fn}")
self.meta_client.exec_cmd(container, f"cp {jks_fn} {jks_fn}.backup")
logger.info(f"creating new {name}:{jks_fn}")
self.meta_client.copy_to_container(container, jks_fn)

# as new JKS pushed to container, we need to tell auth-server to reload the private keys
# by increasing jansRevision again; note that as jansRevision may have been modified externally
# we need to ensure we have fresh jansRevision value to increase to
config = self.backend.get_auth_config()
rev = int(config["jansRevision"]) + 1
conf_dynamic.update({
"keySelectionStrategy": self.privkey_push_strategy,
})

logger.info(f"using keySelectionStrategy {self.privkey_push_strategy}")

self.backend.modify_auth_config(
config["id"],
rev,
conf_dynamic,
keys,
)
# publish delayed jks
if self.push_keys and self.privkey_push_delay > 0 and auth_containers:
logger.info(f"Waiting for private key push delay ({self.privkey_push_delay} seconds) ...")
time.sleep(self.privkey_push_delay)

for container in auth_containers:
logger.info(f"creating backup of {name}:{jks_fn}")
self.meta_client.exec_cmd(container, f"cp {jks_fn} {jks_fn}.backup")
logger.info(f"creating new {name}:{jks_fn}")
self.meta_client.copy_to_container(container, jks_fn)

# as new JKS pushed to container, we need to tell auth-server to reload the private keys
# by increasing jansRevision again; note that as jansRevision may have been modified externally
# we need to ensure we have fresh jansRevision value to increase to
config = self.backend.get_auth_config()
rev = int(config["jansRevision"]) + 1
conf_dynamic.update({
"keySelectionStrategy": self.privkey_push_strategy,
})

logger.info(f"using keySelectionStrategy {self.privkey_push_strategy}")

self.backend.modify_auth_config(
config["id"],
rev,
conf_dynamic,
keys,
)
except (TypeError, ValueError,) as exc:
logger.warning(f"Unable to get public keys; reason={exc}")

Expand Down Expand Up @@ -539,7 +538,8 @@ def prune(self):
logger.warning(
"Unable to find any jans-auth container; make sure "
"to deploy jans-auth and set APP_NAME=auth-server "
"label on container level"
"label on container level. Note that pushing keys to "
"containers will be skipped!"
)
# exit immediately to avoid persistence/secrets being modified
return
Expand Down Expand Up @@ -579,18 +579,17 @@ def prune(self):
self.meta_client.exec_cmd(container, f"cp {jks_fn}.backup {jks_fn}")
logger.info(f"restoring backup of {name}:{jwks_fn}")
self.meta_client.exec_cmd(container, f"cp {jwks_fn}.backup {jwks_fn}")
return

self.manager.secret.set("auth_jks_base64", encode_jks(self.manager))
self.manager.config.set("auth_key_rotated_at", int(time.time()))
self.manager.secret.set("auth_openid_jks_pass", jks_pass)
self.manager.config.set("auth_sig_keys", self.sig_keys)
self.manager.config.set("auth_enc_keys", self.enc_keys)
# jwks
self.manager.secret.set(
"auth_openid_key_base64",
generate_base64_contents(json.dumps(keys)),
)
else:
self.manager.secret.set("auth_jks_base64", encode_jks(self.manager))
self.manager.config.set("auth_key_rotated_at", int(time.time()))
self.manager.secret.set("auth_openid_jks_pass", jks_pass)
self.manager.config.set("auth_sig_keys", self.sig_keys)
self.manager.config.set("auth_enc_keys", self.enc_keys)
# jwks
self.manager.secret.set(
"auth_openid_key_base64",
generate_base64_contents(json.dumps(keys)),
)
except (TypeError, ValueError,) as exc:
logger.warning(f"Unable to get public keys; reason={exc}")

Expand Down
34 changes: 28 additions & 6 deletions jans-pycloudlib/jans/pycloudlib/meta/kubernetes_meta.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
class KubernetesMeta(BaseMeta):
"""A class to interact with a subset of Kubernetes APIs."""

def __init__(self) -> None:
def __init__(self) -> None: # noqa: D107
self._client: _t.Union[kubernetes.client.CoreV1Api, None] = None
self.kubeconfig_file = os.path.expanduser("~/.kube/config")

Expand All @@ -38,10 +38,28 @@ def client(self) -> kubernetes.client.CoreV1Api:
# config loading priority
try:
kubernetes.config.load_incluster_config()
except kubernetes.config.config_exception.ConfigException:
kubernetes.config.load_kube_config(self.kubeconfig_file)
self._client = kubernetes.client.CoreV1Api()
self._client.api_client.configuration.assert_hostname = False
config_loaded = True
except kubernetes.config.config_exception.ConfigException as exc:
# some cluster running restricted env (like Google Cloud Run) doesn't have
# required env vars `KUBERNETES_SERVICE_HOST` and `KUBERNETES_SERVICE_PORT_HTTPS`
logger.warning(f"Unable to load Kubernetes in-cluster config; reason={exc}")
config_loaded = False

# try loading config from kubeconfig file
if not config_loaded:
try:
kubernetes.config.load_kube_config(self.kubeconfig_file)
config_loaded = True
except kubernetes.config.config_exception.ConfigException as exc:
logger.warning(f"Unable to load Kubernetes config from {self.kubeconfig_file}; reason={exc}")
config_loaded = False

# set client only if config is loaded properly
if config_loaded:
self._client = kubernetes.client.CoreV1Api()
self._client.api_client.configuration.assert_hostname = False
else:
logger.warning("Kubernetes client config are not loaded properly, thus client will be disabled!")
return self._client

def get_containers(self, label: str) -> list[V1Pod]:
Expand All @@ -57,7 +75,11 @@ def get_containers(self, label: str) -> list[V1Pod]:
List of pod objects.
"""
namespace = os.environ.get("CN_CONTAINER_METADATA_NAMESPACE", "default")
pods: list[V1Pod] = self.client.list_namespaced_pod(namespace, label_selector=label).items
try:
pods: list[V1Pod] = self.client.list_namespaced_pod(namespace, label_selector=label).items
except AttributeError:
# client is not set due to missing k8s config
pods = []
return pods

def get_container_ip(self, container: V1Pod) -> str:
Expand Down

0 comments on commit a31dee3

Please sign in to comment.