From 0522e61809b9052adce4fdb0db77e2d71558144e Mon Sep 17 00:00:00 2001 From: Isman Firmansyah Date: Wed, 30 Nov 2022 20:47:16 +0700 Subject: [PATCH] feat(jans-pycloudlib): add AWS Secrets Manager support for configuration layers (#3112) * feat(jans-pycloudlib): add AWS Secrets Manager support for configuration layers Ref: https://github.com/JanssenProject/jans/issues/3026 * chore(jans-pycloudlib): updated build (#3113) Signed-off-by: mo-auto <54212639+mo-auto@users.noreply.github.com> Signed-off-by: mo-auto <54212639+mo-auto@users.noreply.github.com> * feat: add aws secret setup to helm chart * ci: add update of pycloud exception Signed-off-by: mo-auto <54212639+mo-auto@users.noreply.github.com> Co-authored-by: mo-auto <54212639+mo-auto@users.noreply.github.com> Co-authored-by: moabu <47318409+moabu@users.noreply.github.com> --- .github/workflows/pr-ref-issue.yml | 1 + .../templates/cronjobs.yaml | 31 +++ .../auth-server/templates/deployment.yml | 31 +++ .../config-api/templates/deployment.yaml | 31 +++ .../config/templates/load-init-config.yml | 31 +++ .../charts/config/templates/secrets.yaml | 27 ++ .../charts/fido2/templates/deployment.yml | 31 +++ .../charts/persistence/templates/jobs.yml | 31 +++ .../charts/scim/templates/deployment.yml | 31 +++ charts/janssen/values.schema.json | 8 +- charts/janssen/values.yaml | 30 +- docker-jans-auth-server/Dockerfile | 5 +- docker-jans-auth-server/README.md | 11 +- docker-jans-auth-server/requirements.txt | 2 +- docker-jans-certmanager/Dockerfile | 5 +- docker-jans-certmanager/README.md | 11 +- docker-jans-certmanager/requirements.txt | 2 +- docker-jans-config-api/Dockerfile | 5 +- docker-jans-config-api/README.md | 11 +- docker-jans-config-api/requirements.txt | 2 +- docker-jans-configurator/Dockerfile | 5 +- docker-jans-configurator/README.md | 11 +- docker-jans-configurator/requirements.txt | 2 +- docker-jans-fido2/Dockerfile | 5 +- docker-jans-fido2/README.md | 7 + docker-jans-fido2/requirements.txt | 2 +- docker-jans-persistence-loader/Dockerfile | 5 +- docker-jans-persistence-loader/README.md | 7 + .../requirements.txt | 2 +- docker-jans-scim/Dockerfile | 5 +- docker-jans-scim/README.md | 7 + docker-jans-scim/requirements.txt | 2 +- .../jans/pycloudlib/config/__init__.py | 2 + .../jans/pycloudlib/config/aws_config.py | 261 +++++++++++++++++ jans-pycloudlib/jans/pycloudlib/manager.py | 16 +- .../jans/pycloudlib/secret/__init__.py | 2 + .../jans/pycloudlib/secret/aws_secret.py | 262 ++++++++++++++++++ jans-pycloudlib/pyproject.toml | 2 + jans-pycloudlib/setup.py | 5 +- 39 files changed, 914 insertions(+), 33 deletions(-) create mode 100644 jans-pycloudlib/jans/pycloudlib/config/aws_config.py create mode 100644 jans-pycloudlib/jans/pycloudlib/secret/aws_secret.py diff --git a/.github/workflows/pr-ref-issue.yml b/.github/workflows/pr-ref-issue.yml index ea48016494d..6312cdebc91 100644 --- a/.github/workflows/pr-ref-issue.yml +++ b/.github/workflows/pr-ref-issue.yml @@ -15,6 +15,7 @@ on: - "release-please-**" - "dependabot/**" - "snyk-**" + - "update-pycloud-in-**" workflow_dispatch: jobs: check-prs-issue: diff --git a/charts/janssen/charts/auth-server-key-rotation/templates/cronjobs.yaml b/charts/janssen/charts/auth-server-key-rotation/templates/cronjobs.yaml index 33be90bbed8..3736a8d2c65 100644 --- a/charts/janssen/charts/auth-server-key-rotation/templates/cronjobs.yaml +++ b/charts/janssen/charts/auth-server-key-rotation/templates/cronjobs.yaml @@ -41,6 +41,17 @@ spec: {{- include "auth-server-key-rotation.usr-secret-envs" . | indent 16 }} imagePullPolicy: {{ .Values.image.pullPolicy }} volumeMounts: + {{ if or (eq .Values.global.configSecretAdapter "aws") (eq .Values.global.configAdapterName "aws") }} + - mountPath: {{ .Values.global.cnAwsSharedCredentialsFile }} + name: aws-shared-credential-file + subPath: aws_shared_credential_file + - mountPath: {{ .Values.global.cnAwsConfigFile }} + name: aws-config-file + subPath: aws_config_file + - mountPath: {{ .Values.global.cnAwsSecretsReplicaRegionsFile }} + name: aws-secrets-replica-regions + subPath: aws_secrets_replica_regions + {{- end }} {{ if or (eq .Values.global.configSecretAdapter "google") (eq .Values.global.cnPersistenceType "spanner") }} - mountPath: {{ .Values.global.cnGoogleApplicationCredentials }} name: google-sa @@ -80,6 +91,26 @@ spec: {{- with .Values.volumes }} {{- toYaml . | nindent 12 }} {{- end }} + {{ if or (eq .Values.global.configSecretAdapter "aws") (eq .Values.global.configAdapterName "aws") }} + - name: aws-shared-credential-file + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_shared_credential_file + path: aws_shared_credential_file + - name: aws-config-file + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_config_file + path: aws_config_file + - name: aws-secrets-replica-regions + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_secrets_replica_regions + path: aws_secrets_replica_regions + {{- end }} {{ if or (eq .Values.global.configSecretAdapter "google") (eq .Values.global.cnPersistenceType "spanner") }} - name: google-sa secret: diff --git a/charts/janssen/charts/auth-server/templates/deployment.yml b/charts/janssen/charts/auth-server/templates/deployment.yml index d8604e06427..5dc6710f5c5 100644 --- a/charts/janssen/charts/auth-server/templates/deployment.yml +++ b/charts/janssen/charts/auth-server/templates/deployment.yml @@ -81,6 +81,17 @@ spec: {{- with .Values.volumeMounts }} {{- toYaml . | nindent 10 }} {{- end }} + {{ if or (eq .Values.global.configSecretAdapter "aws") (eq .Values.global.configAdapterName "aws") }} + - mountPath: {{ .Values.global.cnAwsSharedCredentialsFile }} + name: aws-shared-credential-file + subPath: aws_shared_credential_file + - mountPath: {{ .Values.global.cnAwsConfigFile }} + name: aws-config-file + subPath: aws_config_file + - mountPath: {{ .Values.global.cnAwsSecretsReplicaRegionsFile }} + name: aws-secrets-replica-regions + subPath: aws_secrets_replica_regions + {{- end }} {{ if or (eq .Values.global.configSecretAdapter "google") (eq .Values.global.cnPersistenceType "spanner") }} - mountPath: {{ .Values.global.cnGoogleApplicationCredentials }} name: google-sa @@ -121,6 +132,26 @@ spec: {{- with .Values.volumes }} {{- toYaml . | nindent 8 }} {{- end }} + {{ if or (eq .Values.global.configSecretAdapter "aws") (eq .Values.global.configAdapterName "aws") }} + - name: aws-shared-credential-file + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_shared_credential_file + path: aws_shared_credential_file + - name: aws-config-file + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_config_file + path: aws_config_file + - name: aws-secrets-replica-regions + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_secrets_replica_regions + path: aws_secrets_replica_regions + {{- end }} {{ if or (eq .Values.global.configSecretAdapter "google") (eq .Values.global.cnPersistenceType "spanner") }} - name: google-sa secret: diff --git a/charts/janssen/charts/config-api/templates/deployment.yaml b/charts/janssen/charts/config-api/templates/deployment.yaml index eb3a15c3e70..e36f7b26b92 100644 --- a/charts/janssen/charts/config-api/templates/deployment.yaml +++ b/charts/janssen/charts/config-api/templates/deployment.yaml @@ -77,6 +77,17 @@ spec: {{- with .Values.volumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} + {{ if or (eq .Values.global.configSecretAdapter "aws") (eq .Values.global.configAdapterName "aws") }} + - mountPath: {{ .Values.global.cnAwsSharedCredentialsFile }} + name: aws-shared-credential-file + subPath: aws_shared_credential_file + - mountPath: {{ .Values.global.cnAwsConfigFile }} + name: aws-config-file + subPath: aws_config_file + - mountPath: {{ .Values.global.cnAwsSecretsReplicaRegionsFile }} + name: aws-secrets-replica-regions + subPath: aws_secrets_replica_regions + {{- end }} {{ if or (eq .Values.global.configSecretAdapter "google") (eq .Values.global.cnPersistenceType "spanner") }} - mountPath: {{ .Values.global.cnGoogleApplicationCredentials }} name: google-sa @@ -106,6 +117,26 @@ spec: {{- with .Values.volumes }} {{- toYaml . | nindent 8 }} {{- end }} + {{ if or (eq .Values.global.configSecretAdapter "aws") (eq .Values.global.configAdapterName "aws") }} + - name: aws-shared-credential-file + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_shared_credential_file + path: aws_shared_credential_file + - name: aws-config-file + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_config_file + path: aws_config_file + - name: aws-secrets-replica-regions + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_secrets_replica_regions + path: aws_secrets_replica_regions + {{- end }} {{ if or (eq .Values.global.configSecretAdapter "google") (eq .Values.global.cnPersistenceType "spanner") }} - name: google-sa secret: diff --git a/charts/janssen/charts/config/templates/load-init-config.yml b/charts/janssen/charts/config/templates/load-init-config.yml index 19a5196c8aa..43ef9a817ad 100644 --- a/charts/janssen/charts/config/templates/load-init-config.yml +++ b/charts/janssen/charts/config/templates/load-init-config.yml @@ -34,6 +34,26 @@ spec: volumes: {{- with .Values.volumes }} {{- toYaml . | nindent 8 }} + {{- end }} + {{ if or (eq .Values.global.configSecretAdapter "aws") (eq .Values.global.configAdapterName "aws") }} + - name: aws-shared-credential-file + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_shared_credential_file + path: aws_shared_credential_file + - name: aws-config-file + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_config_file + path: aws_config_file + - name: aws-secrets-replica-regions + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_secrets_replica_regions + path: aws_secrets_replica_regions {{- end }} - name: {{ include "config.fullname" . }}-mount-gen-file secret: @@ -59,6 +79,17 @@ spec: {{- with .Values.volumeMounts }} {{- toYaml . | nindent 10 }} {{- end }} + {{ if or (eq .Values.global.configSecretAdapter "aws") (eq .Values.global.configAdapterName "aws") }} + - mountPath: {{ .Values.global.cnAwsSharedCredentialsFile }} + name: aws-shared-credential-file + subPath: aws_shared_credential_file + - mountPath: {{ .Values.global.cnAwsConfigFile }} + name: aws-config-file + subPath: aws_config_file + - mountPath: {{ .Values.global.cnAwsSecretsReplicaRegionsFile }} + name: aws-secrets-replica-regions + subPath: aws_secrets_replica_regions + {{- end }} - mountPath: /app/db/generate.json name: {{ include "config.fullname" . }}-mount-gen-file subPath: generate.json diff --git a/charts/janssen/charts/config/templates/secrets.yaml b/charts/janssen/charts/config/templates/secrets.yaml index de53aec0a2c..78e9093cfa6 100644 --- a/charts/janssen/charts/config/templates/secrets.yaml +++ b/charts/janssen/charts/config/templates/secrets.yaml @@ -59,6 +59,33 @@ data: couchbase.crt: {{ .Values.configmap.cnCouchbaseCrt }} {{- end }} {{- end }} +{{ if or (eq .Values.global.configSecretAdapter "aws") (eq .Values.global.configAdapterName "aws") }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-aws-config-creds + labels: +{{ include "config.labels" . | indent 4 }} +{{- if .Values.additionalLabels }} +{{ toYaml .Values.additionalLabels | indent 4 }} +{{- end }} +{{- if .Values.additionalAnnotations }} + annotations: +{{ toYaml .Values.additionalAnnotations | indent 4 }} +{{- end }} +type: Opaque +stringData: + aws_shared_credential_file: |- + [{{ .Values.configmap.cnAwsProfile | quote }}] + aws_access_key_id = {{ .Values.configmap.cnAwsAccessKeyId }} + aws_secret_access_key = {{ .Values.configmap.cnAwsSecretAccessKey }} + aws_config_file: |- + [{{ .Values.configmap.cnAwsProfile | quote }}] + region = {{ .Values.configmap.cnAwsDefaultRegion | quote }} + aws_secrets_replica_regions: |- + {{ .Values.configmap.cnAwsSecretsReplicaRegions | toJson }} +{{- end }} {{ if or (eq .Values.global.configSecretAdapter "google") (eq .Values.global.cnPersistenceType "spanner") }} --- apiVersion: v1 diff --git a/charts/janssen/charts/fido2/templates/deployment.yml b/charts/janssen/charts/fido2/templates/deployment.yml index 9a2d36673e5..56fbf2ca16e 100644 --- a/charts/janssen/charts/fido2/templates/deployment.yml +++ b/charts/janssen/charts/fido2/templates/deployment.yml @@ -81,6 +81,17 @@ spec: {{- with .Values.volumeMounts }} {{- toYaml . | nindent 10 }} {{- end }} + {{ if or (eq .Values.global.configSecretAdapter "aws") (eq .Values.global.configAdapterName "aws") }} + - mountPath: {{ .Values.global.cnAwsSharedCredentialsFile }} + name: aws-shared-credential-file + subPath: aws_shared_credential_file + - mountPath: {{ .Values.global.cnAwsConfigFile }} + name: aws-config-file + subPath: aws_config_file + - mountPath: {{ .Values.global.cnAwsSecretsReplicaRegionsFile }} + name: aws-secrets-replica-regions + subPath: aws_secrets_replica_regions + {{- end }} {{ if or (eq .Values.global.configSecretAdapter "google") (eq .Values.global.cnPersistenceType "spanner") }} - mountPath: {{ .Values.global.cnGoogleApplicationCredentials }} name: google-sa @@ -121,6 +132,26 @@ spec: {{- with .Values.volumes }} {{- toYaml . | nindent 8 }} {{- end }} + {{ if or (eq .Values.global.configSecretAdapter "aws") (eq .Values.global.configAdapterName "aws") }} + - name: aws-shared-credential-file + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_shared_credential_file + path: aws_shared_credential_file + - name: aws-config-file + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_config_file + path: aws_config_file + - name: aws-secrets-replica-regions + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_secrets_replica_regions + path: aws_secrets_replica_regions + {{- end }} {{ if or (eq .Values.global.configSecretAdapter "google") (eq .Values.global.cnPersistenceType "spanner") }} - name: google-sa secret: diff --git a/charts/janssen/charts/persistence/templates/jobs.yml b/charts/janssen/charts/persistence/templates/jobs.yml index 3ee53af10f2..56b6aebb6c4 100644 --- a/charts/janssen/charts/persistence/templates/jobs.yml +++ b/charts/janssen/charts/persistence/templates/jobs.yml @@ -67,6 +67,17 @@ spec: {{- with .Values.volumeMounts }} {{- toYaml . | nindent 10 }} {{- end }} + {{ if or (eq .Values.global.configSecretAdapter "aws") (eq .Values.global.configAdapterName "aws") }} + - mountPath: {{ .Values.global.cnAwsSharedCredentialsFile }} + name: aws-shared-credential-file + subPath: aws_shared_credential_file + - mountPath: {{ .Values.global.cnAwsConfigFile }} + name: aws-config-file + subPath: aws_config_file + - mountPath: {{ .Values.global.cnAwsSecretsReplicaRegionsFile }} + name: aws-secrets-replica-regions + subPath: aws_secrets_replica_regions + {{- end }} {{ if or (eq .Values.global.configSecretAdapter "google") (eq .Values.global.cnPersistenceType "spanner") }} - mountPath: {{ .Values.global.cnGoogleApplicationCredentials }} name: google-sa @@ -83,6 +94,26 @@ spec: {{- with .Values.volumes }} {{- toYaml . | nindent 8 }} {{- end }} + {{ if or (eq .Values.global.configSecretAdapter "aws") (eq .Values.global.configAdapterName "aws") }} + - name: aws-shared-credential-file + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_shared_credential_file + path: aws_shared_credential_file + - name: aws-config-file + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_config_file + path: aws_config_file + - name: aws-secrets-replica-regions + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_secrets_replica_regions + path: aws_secrets_replica_regions + {{- end }} {{ if or (eq .Values.global.configSecretAdapter "google") (eq .Values.global.cnPersistenceType "spanner") }} - name: google-sa secret: diff --git a/charts/janssen/charts/scim/templates/deployment.yml b/charts/janssen/charts/scim/templates/deployment.yml index eab5946f88a..ac7b9f603fd 100644 --- a/charts/janssen/charts/scim/templates/deployment.yml +++ b/charts/janssen/charts/scim/templates/deployment.yml @@ -89,6 +89,17 @@ spec: {{- with .Values.volumeMounts }} {{- toYaml . | nindent 10 }} {{- end }} + {{ if or (eq .Values.global.configSecretAdapter "aws") (eq .Values.global.configAdapterName "aws") }} + - mountPath: {{ .Values.global.cnAwsSharedCredentialsFile }} + name: aws-shared-credential-file + subPath: aws_shared_credential_file + - mountPath: {{ .Values.global.cnAwsConfigFile }} + name: aws-config-file + subPath: aws_config_file + - mountPath: {{ .Values.global.cnAwsSecretsReplicaRegionsFile }} + name: aws-secrets-replica-regions + subPath: aws_secrets_replica_regions + {{- end }} {{ if or (eq .Values.global.configSecretAdapter "google") (eq .Values.global.cnPersistenceType "spanner") }} - mountPath: {{ .Values.global.cnGoogleApplicationCredentials }} name: google-sa @@ -119,6 +130,26 @@ spec: {{- with .Values.volumes }} {{- toYaml . | nindent 8 }} {{- end }} + {{ if or (eq .Values.global.configSecretAdapter "aws") (eq .Values.global.configAdapterName "aws") }} + - name: aws-shared-credential-file + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_shared_credential_file + path: aws_shared_credential_file + - name: aws-config-file + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_config_file + path: aws_config_file + - name: aws-secrets-replica-regions + secret: + secretName: {{ .Release.Name }}-aws-config-creds + items: + - key: aws_secrets_replica_regions + path: aws_secrets_replica_regions + {{- end }} {{ if or (eq .Values.global.configSecretAdapter "google") (eq .Values.global.cnPersistenceType "spanner") }} - name: google-sa secret: diff --git a/charts/janssen/values.schema.json b/charts/janssen/values.schema.json index c6c5737459f..ed3e23b3958 100644 --- a/charts/janssen/values.schema.json +++ b/charts/janssen/values.schema.json @@ -458,14 +458,14 @@ } }, "configAdapterName": { - "description": "The config backend adapter that will hold Janssen configuration layer. google|kubernetes", + "description": "The config backend adapter that will hold Janssen configuration layer. aws|google|kubernetes", "type": "string", - "pattern": "^(kubernetes|google)$" + "pattern": "^(kubernetes|google|aws)$" }, "configSecretAdapter": { - "description": "The config backend adapter that will hold Janssen secret layer. google|kubernetes", + "description": "The config backend adapter that will hold Janssen secret layer. aws|google|kubernetes", "type": "string", - "pattern": "^(kubernetes|google)$" + "pattern": "^(kubernetes|google|aws)$" }, "cnGoogleApplicationCredentials": { "description": "Base64 encoded service account. The sa must have roles/secretmanager.admin to use Google secrets and roles/spanner.databaseUser to use Spanner.", diff --git a/charts/janssen/values.yaml b/charts/janssen/values.yaml index 9239959e0df..2d744ea2df8 100644 --- a/charts/janssen/values.yaml +++ b/charts/janssen/values.yaml @@ -217,6 +217,24 @@ config: cnConfigGoogleSecretNamePrefix: janssen # [google_secret_manager_envs] END # [google_envs] END + # [aws_envs] Envs related to using AWS + # [aws_secret_manager_envs] + # AWS Access key id that belong to a user/id with SecretsManagerReadWrite policy + cnAwsAccessKeyId: "" + # AWS Secret Access key that belong to a user/id with SecretsManagerReadWrite policy + cnAwsSecretAccessKey: "" + # The URL of AWS secretsmanager service (if omitted, will use the one in specified region). Used only when global.configAdapterName and global.configSecretAdapter is set to aws. + cnAwsSecretsEndpointUrl: "" + # The prefix name of the secrets. Used only when global.configAdapterName and global.configSecretAdapter is set to aws. + cnAwsSecretsNamePrefix: janssen + # The default AWS Region to use, for example, `us-west-1` or `us-west-2`. + cnAwsDefaultRegion: us-west-1 + # The default profile to use. + cnAwsProfile: "janssen" + # Example replicated region [{"Region": "us-west-1"}, {"Region": "us-west-2"}] + cnAwsSecretsReplicaRegions: [] + # [aws_secret_manager_envs] END + # [aws_envs] END # -- OpenDJ internal address. Leave as default. Used when `global.cnPersistenceType` is set to `ldap`. cnLdapUrl: "opendj:1636" # -- Value passed to Java option -XX:MaxRAMPercentage @@ -574,12 +592,18 @@ global: enabled: true # -- https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/ jobTtlSecondsAfterFinished: 300 - # -- The config backend adapter that will hold Janssen configuration layer. google|kubernetes + # -- The config backend adapter that will hold Janssen configuration layer. aws|google|kubernetes configAdapterName: kubernetes - # -- The config backend adapter that will hold Janssen secret layer. google|kubernetes + # -- The config backend adapter that will hold Janssen secret layer. aws|google|kubernetes configSecretAdapter: kubernetes - # -- Base64 encoded service account. The sa must have roles/secretmanager.admin to use Google secrets and roles/spanner.databaseUser to use Spanner. + # -- Base64 encoded service account. The sa must have roles/secretmanager.admin to use Google secrets and roles/spanner.databaseUser to use Spanner. Leave as this is a sensible default. cnGoogleApplicationCredentials: /etc/jans/conf/google-credentials.json + # The location of the shared credentials file used by the client (see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html).Leave as this is a sensible default. + cnAwsSharedCredentialsFile: /etc/jans/conf/aws_shared_credential_file + # The location of the config file used by the client (see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). Leave as this is a sensible default. + cnAwsConfigFile: /etc/jans/conf/aws_config_file + # The location of file contains replica regions definition (if any). This file is mostly used in primary region. Example of contents of the file: `[{"Region": "us-west-1"}]`. Used only when global.configAdapterName and global.configSecretAdapter is set to aws. Leave as this is a sensible default. + cnAwsSecretsReplicaRegionsFile: /etc/jans/conf/aws_secrets_replica_regions config-api: # -- Name of the config-api service. Please keep it as default. configApiServerServiceName: config-api diff --git a/docker-jans-auth-server/Dockerfile b/docker-jans-auth-server/Dockerfile index a92b6988a6e..1b07a38bf2f 100644 --- a/docker-jans-auth-server/Dockerfile +++ b/docker-jans-auth-server/Dockerfile @@ -248,7 +248,10 @@ ENV CN_MAX_RAM_PERCENTAGE=75.0 \ GOOGLE_APPLICATION_CREDENTIALS=/etc/jans/conf/google-credentials.json \ ADMIN_UI_JWKS=http://0.0.0.0:8080/jans-auth/restv1/jwks \ CN_JETTY_REQUEST_HEADER_SIZE=8192 \ - CN_PROMETHEUS_PORT="" + CN_PROMETHEUS_PORT="" \ + CN_AWS_SECRETS_ENDPOINT_URL="" \ + CN_AWS_SECRETS_PREFIX=jans \ + CN_AWS_SECRETS_REPLICA_FILE="" # ========== # misc stuff diff --git a/docker-jans-auth-server/README.md b/docker-jans-auth-server/README.md index ab9efbd2143..3fc256ae09e 100644 --- a/docker-jans-auth-server/README.md +++ b/docker-jans-auth-server/README.md @@ -11,7 +11,7 @@ For bleeding-edge/unstable version, use `janssenproject/auth-server:1.0.1_dev`. The following environment variables are supported by the container: -- `CN_CONFIG_ADAPTER`: The config backend adapter, can be `consul` (default), `kubernetes`, or `google`. +- `CN_CONFIG_ADAPTER`: The config backend adapter, can be `consul` (default), `kubernetes`, `google`, or `aws`. - `CN_CONFIG_CONSUL_HOST`: hostname or IP of Consul (default to `localhost`). - `CN_CONFIG_CONSUL_PORT`: port of Consul (default to `8500`). - `CN_CONFIG_CONSUL_CONSISTENCY`: Consul consistency mode (choose one of `default`, `consistent`, or `stale`). Default to `stale` mode. @@ -26,7 +26,7 @@ The following environment variables are supported by the container: - `CN_CONFIG_KUBERNETES_USE_KUBE_CONFIG`: Load credentials from `$HOME/.kube/config`, only useful for non-container environment (default to `false`). - `CN_CONFIG_GOOGLE_SECRET_VERSION_ID`: Janssen configuration secret version ID in Google Secret Manager. Defaults to `latest`, which is recommended. - `CN_CONFIG_GOOGLE_SECRET_NAME_PREFIX`: Prefix for Janssen configuration secret in Google Secret Manager. Defaults to `jans`. If left intact `jans-configuration` secret will be created. -- `CN_SECRET_ADAPTER`: The secrets' adapter, can be `vault` (default), `kubernetes`, or `google`. +- `CN_SECRET_ADAPTER`: The secrets' adapter, can be `vault` (default), `kubernetes`, `google`, or `aws`. - `CN_SECRET_VAULT_SCHEME`: supported Vault scheme (`http` or `https`). - `CN_SECRET_VAULT_HOST`: hostname or IP of Vault (default to `localhost`). - `CN_SECRET_VAULT_PORT`: port of Vault (default to `8200`). @@ -83,6 +83,13 @@ The following environment variables are supported by the container: - `CN_SQL_DB_DIALECT`: Dialect name of the SQL (`mysql` for MySQL or `pgsql` for PostgreSQL; default to `mysql`). - `CN_SQL_DB_TIMEZONE`: Timezone used by the SQL database (default to `UTC`). - `CN_SQL_DB_SCHEMA`: Schema name used by SQL database (default to empty-string; if using MySQL, the schema name will be resolved as the database name, whereas in PostgreSQL the schema name will be resolved as `"public"`). +- `CN_AWS_SECRETS_ENDPOINT_URL`: The URL of AWS secretsmanager service (if omitted, will use the one in specified region). +- `CN_AWS_SECRETS_PREFIX`: The prefix name of the secrets (default to `jans`). +- `CN_AWS_SECRETS_REPLICA_FILE`: The location of file contains replica regions definition (if any). This file is mostly used in primary region. Example of contents of the file: `[{"Region": "us-west-1"}]`. +- `AWS_DEFAULT_REGION`: The default AWS Region to use, for example, `us-west-1` or `us-west-2`. +- `AWS_SHARED_CREDENTIALS_FILE`: The location of the shared credentials file used by the client (see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). +- `AWS_CONFIG_FILE`: The location of the config file used by the client (see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). +- `AWS_PROFILE`: The default profile to use, if any. ### Configure app loggers diff --git a/docker-jans-auth-server/requirements.txt b/docker-jans-auth-server/requirements.txt index a57125ce21d..a7a3f2d0618 100644 --- a/docker-jans-auth-server/requirements.txt +++ b/docker-jans-auth-server/requirements.txt @@ -1,4 +1,4 @@ # pinned to py3-grpcio version to avoid failure on native extension build grpcio==1.41.0 libcst<0.4 -git+https://github.com/JanssenProject/jans@c23a2e505b7eb325a293975d60bbc65d5e367c7d#egg=jans-pycloudlib&subdirectory=jans-pycloudlib +git+https://github.com/JanssenProject/jans@0e84a4a0ba80477281f16aaf4abd7fee25faae26#egg=jans-pycloudlib&subdirectory=jans-pycloudlib diff --git a/docker-jans-certmanager/Dockerfile b/docker-jans-certmanager/Dockerfile index 1d4d576ceba..a7664377533 100644 --- a/docker-jans-certmanager/Dockerfile +++ b/docker-jans-certmanager/Dockerfile @@ -129,7 +129,10 @@ ENV CN_WAIT_MAX_TIME=300 \ CN_WAIT_SLEEP_DURATION=10 \ CN_CONTAINER_METADATA=docker \ GOOGLE_PROJECT_ID="" \ - GOOGLE_APPLICATION_CREDENTIALS=/etc/jans/conf/google-credentials.json + GOOGLE_APPLICATION_CREDENTIALS=/etc/jans/conf/google-credentials.json \ + CN_AWS_SECRETS_ENDPOINT_URL="" \ + CN_AWS_SECRETS_PREFIX=jans \ + CN_AWS_SECRETS_REPLICA_FILE="" # ==== # misc diff --git a/docker-jans-certmanager/README.md b/docker-jans-certmanager/README.md index ce605a209dd..27016f9766a 100644 --- a/docker-jans-certmanager/README.md +++ b/docker-jans-certmanager/README.md @@ -12,7 +12,7 @@ For bleeding-edge/unstable version, use `janssenproject/certmanager:1.0.1_dev`. The following environment variables are supported by the container: -- `CN_CONFIG_ADAPTER`: The config backend adapter, can be `consul` (default), `kubernetes`, or `google`. +- `CN_CONFIG_ADAPTER`: The config backend adapter, can be `consul` (default), `kubernetes`, `google`, or `aws`. - `CN_CONFIG_CONSUL_HOST`: hostname or IP of Consul (default to `localhost`). - `CN_CONFIG_CONSUL_PORT`: port of Consul (default to `8500`). - `CN_CONFIG_CONSUL_CONSISTENCY`: Consul consistency mode (choose one of `default`, `consistent`, or `stale`). Default to `stale` mode. @@ -30,7 +30,7 @@ The following environment variables are supported by the container: - `CN_SECRET_GOOGLE_SECRET_VERSION_ID`: Janssen secret version ID in Google Secret Manager. Defaults to `latest`, which is recommended. - `CN_SECRET_GOOGLE_SECRET_MANAGER_PASSPHRASE`: Passphrase for Janssen secret in Google Secret Manager. This is recommended to be changed and defaults to `secret`. - `CN_SECRET_GOOGLE_SECRET_NAME_PREFIX`: Prefix for Janssen secret in Google Secret Manager. Defaults to `jans`. If left `jans-secret` secret will be created. -- `CN_SECRET_ADAPTER`: The secrets' adapter, can be `vault` (default), `kubernetes`, or `google`. +- `CN_SECRET_ADAPTER`: The secrets' adapter, can be `vault` (default), `kubernetes`, `google`, or `aws`. - `CN_SECRET_VAULT_SCHEME`: supported Vault scheme (`http` or `https`). - `CN_SECRET_VAULT_HOST`: hostname or IP of Vault (default to `localhost`). - `CN_SECRET_VAULT_PORT`: port of Vault (default to `8200`). @@ -73,6 +73,13 @@ The following environment variables are supported by the container: - `CN_SQL_DB_DIALECT`: Dialect name of the SQL (`mysql` for MySQL or `pgsql` for PostgreSQL; default to `mysql`). - `CN_SQL_DB_TIMEZONE`: Timezone used by the SQL database (default to `UTC`). - `CN_SQL_DB_SCHEMA`: Schema name used by SQL database (default to empty-string; if using MySQL, the schema name will be resolved as the database name, whereas in PostgreSQL the schema name will be resolved as `"public"`). +- `CN_AWS_SECRETS_ENDPOINT_URL`: The URL of AWS secretsmanager service (if omitted, will use the one in specified region). +- `CN_AWS_SECRETS_PREFIX`: The prefix name of the secrets (default to `jans`). +- `CN_AWS_SECRETS_REPLICA_FILE`: The location of file contains replica regions definition (if any). This file is mostly used in primary region. Example of contents of the file: `[{"Region": "us-west-1"}]`. +- `AWS_DEFAULT_REGION`: The default AWS Region to use, for example, `us-west-1` or `us-west-2`. +- `AWS_SHARED_CREDENTIALS_FILE`: The location of the shared credentials file used by the client (see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). +- `AWS_CONFIG_FILE`: The location of the config file used by the client (see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). +- `AWS_PROFILE`: The default profile to use, if any. ## Usage diff --git a/docker-jans-certmanager/requirements.txt b/docker-jans-certmanager/requirements.txt index 5d7768eb212..1e35e9c67ab 100644 --- a/docker-jans-certmanager/requirements.txt +++ b/docker-jans-certmanager/requirements.txt @@ -2,4 +2,4 @@ grpcio==1.41.0 click==6.7 libcst<0.4 -git+https://github.com/JanssenProject/jans@c23a2e505b7eb325a293975d60bbc65d5e367c7d#egg=jans-pycloudlib&subdirectory=jans-pycloudlib +git+https://github.com/JanssenProject/jans@0e84a4a0ba80477281f16aaf4abd7fee25faae26#egg=jans-pycloudlib&subdirectory=jans-pycloudlib diff --git a/docker-jans-config-api/Dockerfile b/docker-jans-config-api/Dockerfile index 6c9120ace3c..12c9865c7fd 100644 --- a/docker-jans-config-api/Dockerfile +++ b/docker-jans-config-api/Dockerfile @@ -233,7 +233,10 @@ ENV CN_MAX_RAM_PERCENTAGE=75.0 \ CN_JAVA_OPTIONS="" \ GOOGLE_PROJECT_ID="" \ GOOGLE_APPLICATION_CREDENTIALS=/etc/jans/conf/google-credentials.json \ - CN_PROMETHEUS_PORT="" + CN_PROMETHEUS_PORT="" \ + CN_AWS_SECRETS_ENDPOINT_URL="" \ + CN_AWS_SECRETS_PREFIX=jans \ + CN_AWS_SECRETS_REPLICA_FILE="" # ==== # misc diff --git a/docker-jans-config-api/README.md b/docker-jans-config-api/README.md index 90ea441bacf..da3fbf7ade1 100644 --- a/docker-jans-config-api/README.md +++ b/docker-jans-config-api/README.md @@ -11,7 +11,7 @@ For bleeding-edge/unstable version, use `janssenproject/config-api:1.0.1_dev`. The following environment variables are supported by the container: -- `CN_CONFIG_ADAPTER`: The config backend adapter, can be `consul` (default) or `kubernetes`. +- `CN_CONFIG_ADAPTER`: The config backend adapter, can be `consul` (default), `kubernetes`, `google`, or `aws`. - `CN_CONFIG_CONSUL_HOST`: hostname or IP of Consul (default to `localhost`). - `CN_CONFIG_CONSUL_PORT`: port of Consul (default to `8500`). - `CN_CONFIG_CONSUL_CONSISTENCY`: Consul consistency mode (choose one of `default`, `consistent`, or `stale`). Default to `stale` mode. @@ -26,7 +26,7 @@ The following environment variables are supported by the container: - `CN_CONFIG_KUBERNETES_USE_KUBE_CONFIG`: Load credentials from `$HOME/.kube/config`, only useful for non-container environment (default to `false`). - `CN_CONFIG_GOOGLE_SECRET_VERSION_ID`: Janssen configuration secret version ID in Google Secret Manager. Defaults to `latest`, which is recommended. - `CN_CONFIG_GOOGLE_SECRET_NAME_PREFIX`: Prefix for Janssen configuration secret in Google Secret Manager. Defaults to `jans`. If left intact `jans-configuration` secret will be created. -- `CN_SECRET_ADAPTER`: The secrets' adapter, can be `vault` or `kubernetes`. +- `CN_SECRET_ADAPTER`: The secrets' adapter, can be `vault`, `kubernetes`, `google`, or `aws`. - `CN_SECRET_VAULT_SCHEME`: supported Vault scheme (`http` or `https`). - `CN_SECRET_VAULT_HOST`: hostname or IP of Vault (default to `localhost`). - `CN_SECRET_VAULT_PORT`: port of Vault (default to `8200`). @@ -80,6 +80,13 @@ The following environment variables are supported by the container: - `CN_SQL_DB_DIALECT`: Dialect name of the SQL (`mysql` for MySQL or `pgsql` for PostgreSQL; default to `mysql`). - `CN_SQL_DB_TIMEZONE`: Timezone used by the SQL database (default to `UTC`). - `CN_SQL_DB_SCHEMA`: Schema name used by SQL database (default to empty-string; if using MySQL, the schema name will be resolved as the database name, whereas in PostgreSQL the schema name will be resolved as `"public"`). +- `CN_AWS_SECRETS_ENDPOINT_URL`: The URL of AWS secretsmanager service (if omitted, will use the one in specified region). +- `CN_AWS_SECRETS_PREFIX`: The prefix name of the secrets (default to `jans`). +- `CN_AWS_SECRETS_REPLICA_FILE`: The location of file contains replica regions definition (if any). This file is mostly used in primary region. Example of contents of the file: `[{"Region": "us-west-1"}]`. +- `AWS_DEFAULT_REGION`: The default AWS Region to use, for example, `us-west-1` or `us-west-2`. +- `AWS_SHARED_CREDENTIALS_FILE`: The location of the shared credentials file used by the client (see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). +- `AWS_CONFIG_FILE`: The location of the config file used by the client (see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). +- `AWS_PROFILE`: The default profile to use, if any. ### Configure app loggers diff --git a/docker-jans-config-api/requirements.txt b/docker-jans-config-api/requirements.txt index 8fe9871f854..1a855925087 100644 --- a/docker-jans-config-api/requirements.txt +++ b/docker-jans-config-api/requirements.txt @@ -2,4 +2,4 @@ grpcio==1.41.0 libcst<0.4 ruamel.yaml==0.16.10 -git+https://github.com/JanssenProject/jans@c23a2e505b7eb325a293975d60bbc65d5e367c7d#egg=jans-pycloudlib&subdirectory=jans-pycloudlib +git+https://github.com/JanssenProject/jans@0e84a4a0ba80477281f16aaf4abd7fee25faae26#egg=jans-pycloudlib&subdirectory=jans-pycloudlib diff --git a/docker-jans-configurator/Dockerfile b/docker-jans-configurator/Dockerfile index cdd7658bc90..a44a098f8f0 100644 --- a/docker-jans-configurator/Dockerfile +++ b/docker-jans-configurator/Dockerfile @@ -106,7 +106,10 @@ ENV CN_WAIT_MAX_TIME=300 \ CN_WAIT_SLEEP_DURATION=10 \ GOOGLE_PROJECT_ID="" \ GOOGLE_APPLICATION_CREDENTIALS=/etc/jans/conf/google-credentials.json \ - CN_CONFIGURATION_SKIP_INITIALIZED=false + CN_CONFIGURATION_SKIP_INITIALIZED=false \ + CN_AWS_SECRETS_ENDPOINT_URL="" \ + CN_AWS_SECRETS_PREFIX=jans \ + CN_AWS_SECRETS_REPLICA_FILE="" # ==== # misc diff --git a/docker-jans-configurator/README.md b/docker-jans-configurator/README.md index 563374be81d..3e413e7554e 100644 --- a/docker-jans-configurator/README.md +++ b/docker-jans-configurator/README.md @@ -11,7 +11,7 @@ For bleeding-edge/unstable version, use `janssenproject/configurator:1.0.1_dev`. The following environment variables are supported by the container: -- `CN_CONFIG_ADAPTER`: The config backend adapter, can be `consul` (default), `kubernetes`, or `google`. +- `CN_CONFIG_ADAPTER`: The config backend adapter, can be `consul` (default), `kubernetes`, `google`, or `aws`. - `CN_CONFIG_CONSUL_HOST`: hostname or IP of Consul (default to `localhost`). - `CN_CONFIG_CONSUL_PORT`: port of Consul (default to `8500`). - `CN_CONFIG_CONSUL_CONSISTENCY`: Consul consistency mode (choose one of `default`, `consistent`, or `stale`). Default to `stale` mode. @@ -26,7 +26,7 @@ The following environment variables are supported by the container: - `CN_CONFIG_KUBERNETES_USE_KUBE_CONFIG`: Load credentials from `$HOME/.kube/config`, only useful for non-container environment (default to `false`). - `CN_CONFIG_GOOGLE_SECRET_VERSION_ID`: Janssen configuration secret version ID in Google Secret Manager. Defaults to `latest`, which is recommended. - `CN_CONFIG_GOOGLE_SECRET_NAME_PREFIX`: Prefix for Janssen configuration secret in Google Secret Manager. Defaults to `jans`. If left intact `jans-configuration` secret will be created. -- `CN_SECRET_ADAPTER`: The secrets' adapter, can be `vault` (default), `kubernetes`, or `google`. +- `CN_SECRET_ADAPTER`: The secrets' adapter, can be `vault` (default), `kubernetes`, `google`, or `aws`. - `CN_SECRET_VAULT_SCHEME`: supported Vault scheme (`http` or `https`). - `CN_SECRET_VAULT_HOST`: hostname or IP of Vault (default to `localhost`). - `CN_SECRET_VAULT_PORT`: port of Vault (default to `8200`). @@ -47,6 +47,13 @@ The following environment variables are supported by the container: - `GOOGLE_PROJECT_ID`: Google Project ID (default to empty string). Used when `CN_CONFIG_ADAPTER` or `CN_SECRET_ADAPTER` set to `google`. - `GOOGLE_APPLICATION_CREDENTIALS`: Path to Google credentials JSON file (default to `/etc/jans/conf/google-credentials.json`). Used when `CN_CONFIG_ADAPTER` or `CN_SECRET_ADAPTER` set to `google`. - `CN_CONFIGURATION_SKIP_INITIALIZED`: skip initialization if backend already initialized (default to `false`). +- `CN_AWS_SECRETS_ENDPOINT_URL`: The URL of AWS secretsmanager service (if omitted, will use the one in specified region). +- `CN_AWS_SECRETS_PREFIX`: The prefix name of the secrets (default to `jans`). +- `CN_AWS_SECRETS_REPLICA_FILE`: The location of file contains replica regions definition (if any). This file is mostly used in primary region. Example of contents of the file: `[{"Region": "us-west-1"}]`. +- `AWS_DEFAULT_REGION`: The default AWS Region to use, for example, `us-west-1` or `us-west-2`. +- `AWS_SHARED_CREDENTIALS_FILE`: The location of the shared credentials file used by the client (see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). +- `AWS_CONFIG_FILE`: The location of the config file used by the client (see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). +- `AWS_PROFILE`: The default profile to use, if any. ## Commands diff --git a/docker-jans-configurator/requirements.txt b/docker-jans-configurator/requirements.txt index 728fd198839..7a91e097a25 100644 --- a/docker-jans-configurator/requirements.txt +++ b/docker-jans-configurator/requirements.txt @@ -4,4 +4,4 @@ click==6.7 marshmallow==3.10.0 fqdn==1.4.0 libcst<0.4 -git+https://github.com/JanssenProject/jans@c23a2e505b7eb325a293975d60bbc65d5e367c7d#egg=jans-pycloudlib&subdirectory=jans-pycloudlib +git+https://github.com/JanssenProject/jans@0e84a4a0ba80477281f16aaf4abd7fee25faae26#egg=jans-pycloudlib&subdirectory=jans-pycloudlib diff --git a/docker-jans-fido2/Dockerfile b/docker-jans-fido2/Dockerfile index 8436b437723..3716cff7b60 100644 --- a/docker-jans-fido2/Dockerfile +++ b/docker-jans-fido2/Dockerfile @@ -202,7 +202,10 @@ ENV CN_MAX_RAM_PERCENTAGE=75.0 \ CN_JAVA_OPTIONS="" \ GOOGLE_PROJECT_ID="" \ GOOGLE_APPLICATION_CREDENTIALS=/etc/jans/conf/google-credentials.json \ - CN_PROMETHEUS_PORT="" + CN_PROMETHEUS_PORT="" \ + CN_AWS_SECRETS_ENDPOINT_URL="" \ + CN_AWS_SECRETS_PREFIX=jans \ + CN_AWS_SECRETS_REPLICA_FILE="" # ========== # misc stuff diff --git a/docker-jans-fido2/README.md b/docker-jans-fido2/README.md index bc77bff790e..6caa8b7ce20 100644 --- a/docker-jans-fido2/README.md +++ b/docker-jans-fido2/README.md @@ -72,6 +72,13 @@ The following environment variables are supported by the container: - `CN_SQL_DB_DIALECT`: Dialect name of the SQL (`mysql` for MySQL or `pgsql` for PostgreSQL; default to `mysql`). - `CN_SQL_DB_TIMEZONE`: Timezone used by the SQL database (default to `UTC`). - `CN_SQL_DB_SCHEMA`: Schema name used by SQL database (default to empty-string; if using MySQL, the schema name will be resolved as the database name, whereas in PostgreSQL the schema name will be resolved as `"public"`). +- `CN_AWS_SECRETS_ENDPOINT_URL`: The URL of AWS secretsmanager service (if omitted, will use the one in specified region). +- `CN_AWS_SECRETS_PREFIX`: The prefix name of the secrets (default to `jans`). +- `CN_AWS_SECRETS_REPLICA_FILE`: The location of file contains replica regions definition (if any). This file is mostly used in primary region. Example of contents of the file: `[{"Region": "us-west-1"}]`. +- `AWS_DEFAULT_REGION`: The default AWS Region to use, for example, `us-west-1` or `us-west-2`. +- `AWS_SHARED_CREDENTIALS_FILE`: The location of the shared credentials file used by the client (see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). +- `AWS_CONFIG_FILE`: The location of the config file used by the client (see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). +- `AWS_PROFILE`: The default profile to use, if any. ### Configure app loggers diff --git a/docker-jans-fido2/requirements.txt b/docker-jans-fido2/requirements.txt index a57125ce21d..a7a3f2d0618 100644 --- a/docker-jans-fido2/requirements.txt +++ b/docker-jans-fido2/requirements.txt @@ -1,4 +1,4 @@ # pinned to py3-grpcio version to avoid failure on native extension build grpcio==1.41.0 libcst<0.4 -git+https://github.com/JanssenProject/jans@c23a2e505b7eb325a293975d60bbc65d5e367c7d#egg=jans-pycloudlib&subdirectory=jans-pycloudlib +git+https://github.com/JanssenProject/jans@0e84a4a0ba80477281f16aaf4abd7fee25faae26#egg=jans-pycloudlib&subdirectory=jans-pycloudlib diff --git a/docker-jans-persistence-loader/Dockerfile b/docker-jans-persistence-loader/Dockerfile index fd38abc9903..4a482aa540c 100644 --- a/docker-jans-persistence-loader/Dockerfile +++ b/docker-jans-persistence-loader/Dockerfile @@ -160,7 +160,10 @@ ENV CN_CACHE_TYPE=NATIVE_PERSISTENCE \ CN_JACKRABBIT_ADMIN_ID_FILE=/etc/jans/conf/jackrabbit_admin_id \ CN_JACKRABBIT_ADMIN_PASSWORD_FILE=/etc/jans/conf/jackrabbit_admin_password \ GOOGLE_PROJECT_ID="" \ - GOOGLE_APPLICATION_CREDENTIALS=/etc/jans/conf/google-credentials.json + GOOGLE_APPLICATION_CREDENTIALS=/etc/jans/conf/google-credentials.json \ + CN_AWS_SECRETS_ENDPOINT_URL="" \ + CN_AWS_SECRETS_PREFIX=jans \ + CN_AWS_SECRETS_REPLICA_FILE="" # ==== # misc diff --git a/docker-jans-persistence-loader/README.md b/docker-jans-persistence-loader/README.md index 88a32bddfbb..24b4a9e8552 100644 --- a/docker-jans-persistence-loader/README.md +++ b/docker-jans-persistence-loader/README.md @@ -78,6 +78,13 @@ The following environment variables are supported by the container: - `CN_SQL_DB_DIALECT`: Dialect name of the SQL (`mysql` for MySQL or `pgsql` for PostgreSQL; default to `mysql`). - `CN_SQL_DB_TIMEZONE`: Timezone used by the SQL database (default to `UTC`). - `CN_SQL_DB_SCHEMA`: Schema name used by SQL database (default to empty-string; if using MySQL, the schema name will be resolved as the database name, whereas in PostgreSQL the schema name will be resolved as `"public"`). +- `CN_AWS_SECRETS_ENDPOINT_URL`: The URL of AWS secretsmanager service (if omitted, will use the one in specified region). +- `CN_AWS_SECRETS_PREFIX`: The prefix name of the secrets (default to `jans`). +- `CN_AWS_SECRETS_REPLICA_FILE`: The location of file contains replica regions definition (if any). This file is mostly used in primary region. Example of contents of the file: `[{"Region": "us-west-1"}]`. +- `AWS_DEFAULT_REGION`: The default AWS Region to use, for example, `us-west-1` or `us-west-2`. +- `AWS_SHARED_CREDENTIALS_FILE`: The location of the shared credentials file used by the client (see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). +- `AWS_CONFIG_FILE`: The location of the config file used by the client (see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). +- `AWS_PROFILE`: The default profile to use, if any. ### Hybrid mapping diff --git a/docker-jans-persistence-loader/requirements.txt b/docker-jans-persistence-loader/requirements.txt index 8fe9871f854..1a855925087 100644 --- a/docker-jans-persistence-loader/requirements.txt +++ b/docker-jans-persistence-loader/requirements.txt @@ -2,4 +2,4 @@ grpcio==1.41.0 libcst<0.4 ruamel.yaml==0.16.10 -git+https://github.com/JanssenProject/jans@c23a2e505b7eb325a293975d60bbc65d5e367c7d#egg=jans-pycloudlib&subdirectory=jans-pycloudlib +git+https://github.com/JanssenProject/jans@0e84a4a0ba80477281f16aaf4abd7fee25faae26#egg=jans-pycloudlib&subdirectory=jans-pycloudlib diff --git a/docker-jans-scim/Dockerfile b/docker-jans-scim/Dockerfile index 931c1e20ef1..bca5e05378e 100644 --- a/docker-jans-scim/Dockerfile +++ b/docker-jans-scim/Dockerfile @@ -204,7 +204,10 @@ ENV CN_MAX_RAM_PERCENTAGE=75.0 \ CN_JAVA_OPTIONS="" \ GOOGLE_PROJECT_ID="" \ GOOGLE_APPLICATION_CREDENTIALS=/etc/jans/conf/google-credentials.json \ - CN_PROMETHEUS_PORT="" + CN_PROMETHEUS_PORT="" \ + CN_AWS_SECRETS_ENDPOINT_URL="" \ + CN_AWS_SECRETS_PREFIX=jans \ + CN_AWS_SECRETS_REPLICA_FILE="" # ========== # misc stuff diff --git a/docker-jans-scim/README.md b/docker-jans-scim/README.md index defc2b17e5b..809666403f3 100644 --- a/docker-jans-scim/README.md +++ b/docker-jans-scim/README.md @@ -72,6 +72,13 @@ The following environment variables are supported by the container: - `CN_SQL_DB_DIALECT`: Dialect name of the SQL (`mysql` for MySQL or `pgsql` for PostgreSQL; default to `mysql`). - `CN_SQL_DB_TIMEZONE`: Timezone used by the SQL database (default to `UTC`). - `CN_SQL_DB_SCHEMA`: Schema name used by SQL database (default to empty-string; if using MySQL, the schema name will be resolved as the database name, whereas in PostgreSQL the schema name will be resolved as `"public"`). +- `CN_AWS_SECRETS_ENDPOINT_URL`: The URL of AWS secretsmanager service (if omitted, will use the one in specified region). +- `CN_AWS_SECRETS_PREFIX`: The prefix name of the secrets (default to `jans`). +- `CN_AWS_SECRETS_REPLICA_FILE`: The location of file contains replica regions definition (if any). This file is mostly used in primary region. Example of contents of the file: `[{"Region": "us-west-1"}]`. +- `AWS_DEFAULT_REGION`: The default AWS Region to use, for example, `us-west-1` or `us-west-2`. +- `AWS_SHARED_CREDENTIALS_FILE`: The location of the shared credentials file used by the client (see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). +- `AWS_CONFIG_FILE`: The location of the config file used by the client (see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). +- `AWS_PROFILE`: The default profile to use, if any. ### Configure app loggers diff --git a/docker-jans-scim/requirements.txt b/docker-jans-scim/requirements.txt index 8fe9871f854..1a855925087 100644 --- a/docker-jans-scim/requirements.txt +++ b/docker-jans-scim/requirements.txt @@ -2,4 +2,4 @@ grpcio==1.41.0 libcst<0.4 ruamel.yaml==0.16.10 -git+https://github.com/JanssenProject/jans@c23a2e505b7eb325a293975d60bbc65d5e367c7d#egg=jans-pycloudlib&subdirectory=jans-pycloudlib +git+https://github.com/JanssenProject/jans@0e84a4a0ba80477281f16aaf4abd7fee25faae26#egg=jans-pycloudlib&subdirectory=jans-pycloudlib diff --git a/jans-pycloudlib/jans/pycloudlib/config/__init__.py b/jans-pycloudlib/jans/pycloudlib/config/__init__.py index 492aa6e8d27..2662a4d7888 100644 --- a/jans-pycloudlib/jans/pycloudlib/config/__init__.py +++ b/jans-pycloudlib/jans/pycloudlib/config/__init__.py @@ -2,10 +2,12 @@ from jans.pycloudlib.config.consul_config import ConsulConfig # noqa: F401 from jans.pycloudlib.config.kubernetes_config import KubernetesConfig # noqa: F401 from jans.pycloudlib.config.google_config import GoogleConfig # noqa: F401 +from jans.pycloudlib.config.aws_config import AwsConfig # noqa: F401 # avoid implicit reexport disabled error __all__ = [ "ConsulConfig", "KubernetesConfig", "GoogleConfig", + "AwsConfig", ] diff --git a/jans-pycloudlib/jans/pycloudlib/config/aws_config.py b/jans-pycloudlib/jans/pycloudlib/config/aws_config.py new file mode 100644 index 00000000000..4dd4c12be59 --- /dev/null +++ b/jans-pycloudlib/jans/pycloudlib/config/aws_config.py @@ -0,0 +1,261 @@ +"""This module contains config adapter class to interact with AWS Secrets Manager.""" + +import json +import logging +import os +import typing as _t +from contextlib import suppress +from functools import cached_property +from functools import partial +from pathlib import Path + +import boto3 +from botocore.exceptions import ClientError +from botocore.exceptions import NoCredentialsError +from botocore.exceptions import NoRegionError + +from jans.pycloudlib.config.base_config import BaseConfig +from jans.pycloudlib.utils import safe_value + +logger = logging.getLogger(__name__) + + +def _dump_value(value: _t.Any) -> str: + """Dump string from any Python data type. + + Args: + value: Any given value. + + Returns: + Compressed bytes contains the value. + """ + return json.dumps(value) + + +def _load_value(value: str) -> _t.Any: + """Load string into any Python data type. + + Args: + value: Any given value + + Returns: + Any Python data type. + """ + return json.loads(value) + + +class AwsConfig(BaseConfig): + """This class interacts with AWS Secrets Manager backend. + + The instance of this class is configured via environment variables. + + Supported environment variables: + + - `CN_AWS_SECRETS_ENDPOINT_URL`: The URL of AWS secretsmanager service (if omitted, will use the one in specified region). + - `CN_AWS_SECRETS_PREFIX`: The prefix name of the secrets (default to `jans`). + - `CN_AWS_SECRETS_REPLICA_FILE`: The location of file contains replica regions definition (if any). This file is mostly used in primary region. + + The following environment variables are used by the underlying AWS SDK: + + - `AWS_DEFAULT_REGION`: The default AWS Region to use, for example, `us-west-1` or `us-west-2`. + - `AWS_SHARED_CREDENTIALS_FILE`: The location of the shared credentials file used by the client. + - `AWS_CONFIG_FILE`: The location of the config file used by the client. + - `AWS_PROFILE`: The default profile to use, if any. + + Example of AWS credentials file: + + ``` + [default] + aws_access_key_id = DEFAULT_ACCESS_KEY_ID + aws_secret_access_key = DEFAULT_SECRET_ACCESS_KEY + + [jans] + aws_access_key_id = JANS_ACCESS_KEY_ID + aws_secret_access_key = JANS_SECRET_ACCESS_KEY + ``` + + Example of AWS config file: + + ``` + [default] + region = us-east-1 + + [profile jans] + region = us-west-1 + ``` + + Example of replica regions: + + ``` + [ + {"Region": "us-west-1"} + ] + ``` + """ + + def __init__(self) -> None: + # unique name used as prefix to distinguish with other secrets + # a typical usage is to use vendor/organization name + prefix = os.environ.get("CN_AWS_SECRETS_PREFIX", "jans") + + # the secrets name will use `_` instead of `-` char to avoid clashing with generated suffix by AWS; + # see https://docs.aws.amazon.com/cli/latest/reference/secretsmanager/create-secret.html#options + self.basepath = f"{prefix}_configs" + + # flag to determine whether AWS secrets already created + self.basepath_exists = False + + @cached_property + def client(self) -> boto3.session.Session.client: + """Create the Secret Manager client.""" + try: + client = boto3.client( + "secretsmanager", + endpoint_url=os.environ.get("CN_AWS_SECRETS_ENDPOINT_URL") or None, + ) + return client + except NoRegionError: + raise RuntimeError( + "AWS region is not specified. Please specify the region in a file " + "pointed by AWS_CONFIG_FILE environment variable, or specify profile " + "name via AWS_PROFILE environment variable, or set the region " + "via AWS_DEFAULT_REGION environment variable." + ) + + def get_all(self) -> dict[str, _t.Any]: + """Get all key-value pairs. + + Returns: + A mapping of configs (if any). + """ + self._prepare_secret() + resp = self.client.get_secret_value(SecretId=self.basepath) + + # SecretString is a `dict` data type + data: dict[str, _t.Any] = _load_value(resp["SecretString"]) + return data + + def get(self, key: str, default: _t.Any = "") -> _t.Any: + """Get value based on given key. + + Args: + key: Key name. + default: Default value if key is not exist. + + Returns: + Value based on given key or default one. + """ + data = self.get_all() + return data.get(key) or default + + def set(self, key: str, value: _t.Any) -> bool: + """Set key with given value. + + Args: + key: Key name. + value: Value of the key. + + Returns: + A boolean to mark whether config is set or not. + """ + data = self.get_all() + data[key] = safe_value(value) + + resp = self.client.update_secret( + SecretId=self.basepath, + SecretString=_dump_value(data), + ) + return bool(resp) + + def set_all(self, data: dict[str, _t.Any]) -> bool: + """Set all key-value pairs. + + Args: + data: key-value pairs of configs. + + Returns: + A boolean indicating operation is succeed or not. + """ + self._prepare_secret() + + resp = self.client.update_secret( + SecretId=self.basepath, + SecretString=_dump_value( + # ensure key-value that has bytes is converted to text + {k: safe_value(v) for k, v in data.items()} + ), + ) + return bool(resp) + + def _prepare_secret(self) -> None: + """Prepare (create if missing) secrets with empty value.""" + # check whether secrets already exists + if self.basepath_exists: + return + + try: + # get the secret + self.client.get_secret_value(SecretId=self.basepath) + + # mark the secret as exists so subsequent checks made by + # client instance won't need to make requests to AWS service + self.basepath_exists = True + + except ClientError as exc: + # raise exception if not related to missing secrets; + # note that missing secrets will be created + if exc.response["Error"]["Code"] != "ResourceNotFoundException": + raise RuntimeError(f"Unable to access AWS Secrets Manager service; reason={exc}") + + create_secret = partial( + self.client.create_secret, + Name=self.basepath, + SecretString=_dump_value({}), + Description="Non-sensitive secrets (configs) for Janssen cluster", + ) + + if self.replica_regions: + # if there's replica regions, pass `AddReplicaRegions` argument; + # this will create replica in the specified regions, but note that + # a replica secret can't be updated independently from its primary secret, + # except for its encryption key. + create_secret.keywords["AddReplicaRegions"] = self.replica_regions + create_secret.keywords["ForceOverwriteReplicaSecret"] = True + + # run the actual secrets creation + create_secret() + + # mark the secrets as exists so subsequent checks made by + # client instance won't need to make requests to AWS service + self.basepath_exists = True + + except NoCredentialsError: + raise RuntimeError( + "AWS credentials are not specified. Please specify the credentials " + "(contains AWS access key ID and secret access key) in a file pointed " + "by AWS_SHARED_CREDENTIALS_FILE environment variable, or specify profile " + "name via AWS_PROFILE environment variable." + ) + + @cached_property + def replica_regions(self) -> list[dict[str, _t.Any]]: + """Get replica regions specified in a file. + + The location of the file is pointed by `CN_AWS_SECRETS_REPLICA_FILE` environment variable. + """ + regions = [] + + with suppress(FileNotFoundError, TypeError, IsADirectoryError): + file_ = os.environ.get("CN_AWS_SECRETS_REPLICA_FILE", "") + try: + txt = Path(file_).read_text().strip() + regions = json.loads(txt) + except json.decoder.JSONDecodeError as exc: + raise ValueError(f"Unable to load replica regions from {file_}; reason={exc}") + else: + # ensure regions does not include current client's region + regions = [ + region for region in regions + if region["Region"] != self.client.meta.region_name + ] + return regions diff --git a/jans-pycloudlib/jans/pycloudlib/manager.py b/jans-pycloudlib/jans/pycloudlib/manager.py index 4611bdc6ae5..dca2664cafe 100644 --- a/jans-pycloudlib/jans/pycloudlib/manager.py +++ b/jans-pycloudlib/jans/pycloudlib/manager.py @@ -10,13 +10,15 @@ from jans.pycloudlib.config import ConsulConfig from jans.pycloudlib.config import KubernetesConfig from jans.pycloudlib.config import GoogleConfig +from jans.pycloudlib.config import AwsConfig from jans.pycloudlib.secret import KubernetesSecret from jans.pycloudlib.secret import VaultSecret from jans.pycloudlib.secret import GoogleSecret +from jans.pycloudlib.secret import AwsSecret from jans.pycloudlib.utils import decode_text from jans.pycloudlib.utils import encode_text -ConfigAdapter = _t.Union[ConsulConfig, KubernetesConfig, GoogleConfig] +ConfigAdapter = _t.Union[ConsulConfig, KubernetesConfig, GoogleConfig, AwsConfig] """Configs adapter type. Currently supports the following classes: @@ -24,9 +26,10 @@ * [ConsulConfig][jans.pycloudlib.config.consul_config.ConsulConfig] * [KubernetesConfig][jans.pycloudlib.config.kubernetes_config.KubernetesConfig] * [GoogleConfig][jans.pycloudlib.config.google_config.GoogleConfig] +* [AwsConfig][jans.pycloudlib.config.aws_config.AwsConfig] """ -SecretAdapter = _t.Union[VaultSecret, KubernetesSecret, GoogleSecret] +SecretAdapter = _t.Union[VaultSecret, KubernetesSecret, GoogleSecret, AwsSecret] """Secrets adapter type. Currently supports the following classes: @@ -34,6 +37,7 @@ * [VaultSecret][jans.pycloudlib.secret.vault_secret.VaultSecret] * [KubernetesSecret][jans.pycloudlib.secret.kubernetes_secret.KubernetesSecret] * [GoogleSecret][jans.pycloudlib.secret.google_secret.GoogleSecret] +* [AwsSecret][jans.pycloudlib.secret.aws_secret.AwsSecret] """ @@ -153,6 +157,7 @@ def adapter(self) -> ConfigAdapter: # noqa: D412 - `consul`: returns an instance of [ConsulConfig][jans.pycloudlib.config.consul_config.ConsulConfig] - `kubernetes`: returns an instance of [KubernetesConfig][jans.pycloudlib.config.kubernetes_config.KubernetesConfig] - `google`: returns an instance of [GoogleConfig][jans.pycloudlib.config.google_config.GoogleConfig] + - `aws`: returns an instance of [AwsConfig][jans.pycloudlib.config.aws_config.AwsConfig] """ adapter = os.environ.get("CN_CONFIG_ADAPTER", "consul") @@ -165,6 +170,9 @@ def adapter(self) -> ConfigAdapter: # noqa: D412 if adapter == "google": return GoogleConfig() + if adapter == "aws": + return AwsConfig() + raise ValueError(f"Unsupported config adapter {adapter!r}") @@ -195,6 +203,7 @@ def adapter(self) -> SecretAdapter: # noqa: D412 - `vault`: returns an instance of [VaultSecret][jans.pycloudlib.secret.vault_secret.VaultSecret] - `kubernetes`: returns an instance of [KubernetesSecret][jans.pycloudlib.secret.kubernetes_secret.KubernetesSecret] - `google`: returns an instance of [GoogleSecret][jans.pycloudlib.secret.google_secret.GoogleSecret] + - `aws`: returns an instance of [AwsSecret][jans.pycloudlib.secret.aws_secret.AwsSecret] """ adapter = os.environ.get("CN_SECRET_ADAPTER", "vault") @@ -207,6 +216,9 @@ def adapter(self) -> SecretAdapter: # noqa: D412 if adapter == "google": return GoogleSecret() + if adapter == "aws": + return AwsSecret() + raise ValueError(f"Unsupported secret adapter {adapter!r}") def to_file( diff --git a/jans-pycloudlib/jans/pycloudlib/secret/__init__.py b/jans-pycloudlib/jans/pycloudlib/secret/__init__.py index 7332463666f..68730d01fdd 100644 --- a/jans-pycloudlib/jans/pycloudlib/secret/__init__.py +++ b/jans-pycloudlib/jans/pycloudlib/secret/__init__.py @@ -2,10 +2,12 @@ from jans.pycloudlib.secret.kubernetes_secret import KubernetesSecret # noqa: F401 from jans.pycloudlib.secret.vault_secret import VaultSecret # noqa: F401 from jans.pycloudlib.secret.google_secret import GoogleSecret # noqa: F401 +from jans.pycloudlib.secret.aws_secret import AwsSecret # noqa: F401 # avoid implicit reexport disabled error __all__ = [ "KubernetesSecret", "VaultSecret", "GoogleSecret", + "AwsSecret", ] diff --git a/jans-pycloudlib/jans/pycloudlib/secret/aws_secret.py b/jans-pycloudlib/jans/pycloudlib/secret/aws_secret.py new file mode 100644 index 00000000000..95beda31e04 --- /dev/null +++ b/jans-pycloudlib/jans/pycloudlib/secret/aws_secret.py @@ -0,0 +1,262 @@ +"""This module contains secret adapter class to interact with AWS Secrets Manager.""" + +import json +import logging +import lzma +import os +import typing as _t +from contextlib import suppress +from functools import cached_property +from functools import partial +from pathlib import Path + +import boto3 +from botocore.exceptions import ClientError +from botocore.exceptions import NoCredentialsError +from botocore.exceptions import NoRegionError + +from jans.pycloudlib.secret.base_secret import BaseSecret +from jans.pycloudlib.utils import safe_value + +logger = logging.getLogger(__name__) + + +def _dump_value(value: _t.Any) -> bytes: + """Dump compressed bytes from any Python data type. + + Args: + value: Any given value. + + Returns: + Compressed bytes contains the value. + """ + return lzma.compress(json.dumps(value).encode()) + + +def _load_value(value: bytes) -> _t.Any: + """Load compressed bytes into any Python data type. + + Args: + value: Any given value + + Returns: + Any Python data type. + """ + return json.loads(lzma.decompress(value).decode()) + + +class AwsSecret(BaseSecret): + """This class interacts with AWS Secrets Manager backend. + + The instance of this class is configured via environment variables. + + Supported environment variables: + + - `CN_AWS_SECRETS_ENDPOINT_URL`: The URL of AWS secretsmanager service (if omitted, will use the one in specified region). + - `CN_AWS_SECRETS_PREFIX`: The prefix name of the secrets (default to `jans`). + - `CN_AWS_SECRETS_REPLICA_FILE`: The location of file contains replica regions definition (if any). This file is mostly used in primary region. + + The following environment variables are used by the underlying AWS SDK: + + - `AWS_DEFAULT_REGION`: The default AWS Region to use, for example, `us-west-1` or `us-west-2`. + - `AWS_SHARED_CREDENTIALS_FILE`: The location of the shared credentials file used by the client. + - `AWS_CONFIG_FILE`: The location of the config file used by the client. + - `AWS_PROFILE`: The default profile to use, if any. + + Example of AWS credentials file: + + ``` + [default] + aws_access_key_id = DEFAULT_ACCESS_KEY_ID + aws_secret_access_key = DEFAULT_SECRET_ACCESS_KEY + + [jans] + aws_access_key_id = JANS_ACCESS_KEY_ID + aws_secret_access_key = JANS_SECRET_ACCESS_KEY + ``` + + Example of AWS config file: + + ``` + [default] + region = us-east-1 + + [profile jans] + region = us-west-1 + ``` + + Example of replica regions: + + ``` + [ + {"Region": "us-west-1"} + ] + ``` + """ + + def __init__(self) -> None: + # unique name used as prefix to distinguish with other secrets + # a typical usage is to use vendor/organization name + prefix = os.environ.get("CN_AWS_SECRETS_PREFIX", "jans") + + # the secrets name will use `_` instead of `-` char to avoid clashing with generated suffix by AWS; + # see https://docs.aws.amazon.com/cli/latest/reference/secretsmanager/create-secret.html#options + self.basepath = f"{prefix}_secrets" + + # flag to determine whether AWS secrets already created + self.basepath_exists = False + + @cached_property + def client(self) -> boto3.session.Session.client: + """Create the Secret Manager client.""" + try: + client = boto3.client( + "secretsmanager", + endpoint_url=os.environ.get("CN_AWS_SECRETS_ENDPOINT_URL") or None, + ) + return client + except NoRegionError: + raise RuntimeError( + "AWS region is not specified. Please specify the region in a file " + "pointed by AWS_CONFIG_FILE environment variable, or specify profile " + "name via AWS_PROFILE environment variable, or set the region " + "via AWS_DEFAULT_REGION environment variable." + ) + + def get_all(self) -> dict[str, _t.Any]: + """Get all key-value pairs. + + Returns: + A mapping of secrets (if any). + """ + self._prepare_secret() + resp = self.client.get_secret_value(SecretId=self.basepath) + + # SecretBinary is a `dict` data type + data: dict[str, _t.Any] = _load_value(resp["SecretBinary"]) + return data + + def get(self, key: str, default: _t.Any = "") -> _t.Any: + """Get value based on given key. + + Args: + key: Key name. + default: Default value if key is not exist. + + Returns: + Value based on given key or default one. + """ + data = self.get_all() + return data.get(key) or default + + def set(self, key: str, value: _t.Any) -> bool: + """Set key with given value. + + Args: + key: Key name. + value: Value of the key. + + Returns: + A boolean to mark whether config is set or not. + """ + data = self.get_all() + data[key] = safe_value(value) + + resp = self.client.update_secret( + SecretId=self.basepath, + SecretBinary=_dump_value(data), + ) + return bool(resp) + + def set_all(self, data: dict[str, _t.Any]) -> bool: + """Set all key-value pairs. + + Args: + data: key-value pairs of secrets. + + Returns: + A boolean indicating operation is succeed or not. + """ + self._prepare_secret() + + resp = self.client.update_secret( + SecretId=self.basepath, + SecretBinary=_dump_value( + # ensure key-value that has bytes is converted to text + {k: safe_value(v) for k, v in data.items()} + ), + ) + return bool(resp) + + def _prepare_secret(self) -> None: + """Prepare (create if missing) secrets with empty value.""" + # check whether secrets already exists + if self.basepath_exists: + return + + try: + # get the secret + self.client.get_secret_value(SecretId=self.basepath) + + # mark the secret as exists so subsequent checks made by + # client instance won't need to make requests to AWS service + self.basepath_exists = True + + except ClientError as exc: + # raise exception if not related to missing secrets; + # note that missing secrets will be created + if exc.response["Error"]["Code"] != "ResourceNotFoundException": + raise RuntimeError(f"Unable to access AWS Secrets Manager service; reason={exc}") + + create_secret = partial( + self.client.create_secret, + Name=self.basepath, + SecretBinary=_dump_value({}), + Description="Secrets for Janssen cluster", + ) + + if self.replica_regions: + # if there's replica regions, pass `AddReplicaRegions` argument; + # this will create replica in the specified regions, but note that + # a replica secret can't be updated independently from its primary secret, + # except for its encryption key. + create_secret.keywords["AddReplicaRegions"] = self.replica_regions + create_secret.keywords["ForceOverwriteReplicaSecret"] = True + + # run the actual secrets creation + create_secret() + + # mark the secrets as exists so subsequent checks made by + # client instance won't need to make requests to AWS service + self.basepath_exists = True + + except NoCredentialsError: + raise RuntimeError( + "AWS credentials are not specified. Please specify the credentials " + "(contains AWS access key ID and secret access key) in a file pointed " + "by AWS_SHARED_CREDENTIALS_FILE environment variable, or specify profile " + "name via AWS_PROFILE environment variable." + ) + + @cached_property + def replica_regions(self) -> list[dict[str, _t.Any]]: + """Get replica regions specified in a file. + + The location of the file is pointed by `CN_AWS_SECRETS_REPLICA_FILE` environment variable. + """ + regions = [] + + with suppress(FileNotFoundError, TypeError, IsADirectoryError): + file_ = os.environ.get("CN_AWS_SECRETS_REPLICA_FILE", "") + try: + txt = Path(file_).read_text().strip() + regions = json.loads(txt) + except json.decoder.JSONDecodeError as exc: + raise ValueError(f"Unable to load replica regions from {file_}; reason={exc}") + else: + # ensure regions does not include current client's region + regions = [ + region for region in regions + if region["Region"] != self.client.meta.region_name + ] + return regions diff --git a/jans-pycloudlib/pyproject.toml b/jans-pycloudlib/pyproject.toml index 88bd60b5410..e84b430f00b 100644 --- a/jans-pycloudlib/pyproject.toml +++ b/jans-pycloudlib/pyproject.toml @@ -26,5 +26,7 @@ module = [ "ldif", "sqlalchemy.*", "requests_toolbelt.*", + "boto3", + "botocore.*", ] ignore_missing_imports = true diff --git a/jans-pycloudlib/setup.py b/jans-pycloudlib/setup.py index ad7ec223e18..b2c06838b2c 100644 --- a/jans-pycloudlib/setup.py +++ b/jans-pycloudlib/setup.py @@ -32,8 +32,6 @@ def find_version(*file_paths): "python-consul>=1.0.1", "hvac>=0.7.0", "kubernetes>=11.0", - # "urllib3<1.25,>=1.21.1", - # "urllib3>=1.25.2", "ldap3>=2.5", "backoff>=2.1.2", "docker>=3.7.2", @@ -48,6 +46,7 @@ def find_version(*file_paths): "ldif>=4.1.1", # handle CVE-2022-36087 "oauthlib>=3.2.1", + "boto3", ], classifiers=[ "Intended Audience :: Developers", @@ -58,6 +57,8 @@ def find_version(*file_paths): "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], include_package_data=True, entry_points={