feat: deprecate password file in favor of secrets

.github/workflows/docker_build_image.yml

@@ -7,6 +7,8 @@ on:
     branches:
       # below branch is triggered by jenkins which triggers dev image
       - update-source-build-date
+      # below protected branche prefix only allows internal teams to push to it
+      - cn-*
     paths:
       - "docker-jans-**/**"
       - "!**.md"

.github/workflows/jans_pycloud_build_package.yml

@@ -23,8 +23,7 @@ jobs:
   docker:
     runs-on: ubuntu-latest
     env:
-      PR_DOCKER_DEV_BRANCH_NAME: update-dev-jans-pycloudlib
-      PR_DOCKER_STABLE_BRANCH_NAME: update-stable-jans-pycloudlib
+      PR_DOCKER_BRANCH_NAME: update-jans-pycloudlib
       GITHUB_TOKEN: ${{ secrets.MOWORKFLOWTOKEN }}
     steps:
       - name: Checkout
@@ -39,79 +38,40 @@ jobs:
           git_user_signingkey: true
           git_commit_gpgsign: true
 
-      - name: Update dev requriments in docker images
-        if: github.event_name == 'pull_request'
-        id: build_dev_reqs
-        run: |
-          dockerimages="auth-server certmanager client-api config-api configurator fido2 persistence-loader scim"
-          for image in $dockerimages; do
-            sed -i '/git+https/c\git+https://github.com/${{ github.repository }}@${{ env.PR_DOCKER_DEV_BRANCH_NAME }}#egg=jans-pycloudlib&subdirectory=jans-pycloudlib' ./docker-jans-$image/requirements.txt
-          done
-
-      - name: Configure Git and open dev PR
-        if: github.event_name == 'pull_request'
-        run: |
-          git config user.name "mo-auto"
-          git config user.email "54212639+mo-auto@users.noreply.github.com"
-          git config --global user.signingkey "${{ steps.import_gpg.outputs.keyid }}"
-          git add -A
-          git commit -S -s -m "chore(jans-pycloudlib): updated dev build"
-
-
       - name: Update stable requriments in docker images
-        if: github.event_name != 'pull_request'
         id: build_stable_reqs
         run: |
           dockerimages="auth-server certmanager client-api config-api configurator fido2 persistence-loader scim"
           for image in $dockerimages; do
             sed -i '/git+https/c\git+https://github.com/${{ github.repository }}@${{ github.sha }}#egg=jans-pycloudlib&subdirectory=jans-pycloudlib' ./docker-jans-$image/requirements.txt
           done
 
-      - name: Configure Git and open stable PR
-        if: github.event_name != 'pull_request'
+      - name: Configure Git
         run: |
           git config user.name "mo-auto"
           git config user.email "54212639+mo-auto@users.noreply.github.com"
           git config --global user.signingkey "${{ steps.import_gpg.outputs.keyid }}"
           git add -A
-          git commit -S -s -m "chore(jans-pycloudlib): updated stable build"
-
-      # Buggy behaviour with gh pr command. Will use the following action until bugs have been fixed.
-      #PR=$(gh pr create --head $PR_DOCKER_DEV_BRANCH_NAME --assignee "moabu" --base "master" --body "Updated build date. Auto-generated." --label "enhancement,bot" --reviewer "moabu" --title "chore(Dockerfile): updated build dates" || echo "PR Branch is already open")
-      - name: Open PR
-        if: github.event_name != 'pull_request'
-        uses: peter-evans/create-pull-request@v3
-        with:
-          token: ${{ secrets.MOWORKFLOWTOKEN }}
-          committer: mo-auto <54212639+mo-auto@users.noreply.github.com>
-          author: mo-auto <54212639+mo-auto@users.noreply.github.com>
-          branch: ${{ env.PR_DOCKER_STABLE_BRANCH_NAME }}
-          title: 'chore(Dockerfiles): updated janspycloud build'
-          body: |
-            - Merge PR. Do not leave open
-            - Updated stable build
-            - Auto-generated.
-          labels: |
-            enhancement
-            bot
-          assignees: moabu
-          reviewers: moabu
-          delete-branch: true
+          git commit -S -s -m "chore(jans-pycloudlib): updated build"
+          if [[ "${{ github.event_name }}" == 'pull_request' ]]; then
+            echo "pushing to feature branch"
+            git push origin HEAD:${{ github.head_ref }} --force
+          fi
 
       # Buggy behaviour with gh pr command. Will use the following action until bugs have been fixed.
       #PR=$(gh pr create --head $PR_DOCKER_DEV_BRANCH_NAME --assignee "moabu" --base "master" --body "Updated build date. Auto-generated." --label "enhancement,bot" --reviewer "moabu" --title "chore(Dockerfile): updated build dates" || echo "PR Branch is already open")
       - name: Open PR
-        if: github.event_name == 'pull_request'
         uses: peter-evans/create-pull-request@v3
+        if: github.event_name != 'pull_request'
         with:
           token: ${{ secrets.MOWORKFLOWTOKEN }}
           committer: mo-auto <54212639+mo-auto@users.noreply.github.com>
           author: mo-auto <54212639+mo-auto@users.noreply.github.com>
-          branch: ${{ env.PR_DOCKER_DEV_BRANCH_NAME }}
+          branch: ${{ env.PR_DOCKER_BRANCH_NAME }}
           title: 'chore(Dockerfiles): updated janspycloud build'
           body: |
             - Always leave open
-            - Updated stable build
+            - Updated unstable build
             - Auto-generated.
           labels: |
             enhancement

docker-jans-auth-server/requirements.txt

@@ -1 +1 @@
-git+https://github.com/JanssenProject/jans-cloud-native@eed35d1118df9137989294237623ab4b8e7fac52#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
+git+https://github.com/JanssenProject/jans-cloud-native@3e226fe36da47353ed3c90b330f23bcb9a36342a#egg=jans-pycloudlib&subdirectory=jans-pycloudlib

docker-jans-certmanager/requirements.txt

@@ -1,2 +1,2 @@
 click==6.7
-git+https://github.com/JanssenProject/jans-cloud-native@eed35d1118df9137989294237623ab4b8e7fac52#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
+git+https://github.com/JanssenProject/jans-cloud-native@3e226fe36da47353ed3c90b330f23bcb9a36342a#egg=jans-pycloudlib&subdirectory=jans-pycloudlib

docker-jans-client-api/requirements.txt

@@ -1,2 +1,2 @@
 ruamel.yaml==0.16.10
-git+https://github.com/JanssenProject/jans-cloud-native@eed35d1118df9137989294237623ab4b8e7fac52#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
+git+https://github.com/JanssenProject/jans-cloud-native@3e226fe36da47353ed3c90b330f23bcb9a36342a#egg=jans-pycloudlib&subdirectory=jans-pycloudlib

docker-jans-config-api/requirements.txt

@@ -1 +1 @@
-git+https://github.com/JanssenProject/jans-cloud-native@eed35d1118df9137989294237623ab4b8e7fac52#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
+git+https://github.com/JanssenProject/jans-cloud-native@3e226fe36da47353ed3c90b330f23bcb9a36342a#egg=jans-pycloudlib&subdirectory=jans-pycloudlib

docker-jans-configurator/README.md

@@ -82,6 +82,11 @@ The load command can be used either to generate or restore config and secret for
 
     - `auth_sig_keys`: space-separated key algorithm for signing (default to `RS256 RS384 RS512 ES256 ES384 ES512 PS256 PS384 PS512`)
     - `auth_enc_keys`: space-separated key algorithm for encryption (default to `RSA1_5 RSA-OAEP`)
+    - `optional_scopes`: list of scopes that will be used (supported scopes are `ldap`, `scim`, `fido2`, `client-api`, `couchbase`, `redis`, `sql`; default to empty list)
+    - `ldap_pw`: user's password to access LDAP database (only used if `optional_scopes` list contains `ldap` scope)
+    - `sql_pw`: user's password to access SQL database (only used if `optional_scopes` list contains `sql` scope)
+    - `couchbase_pw`: user's password to access Couchbase database (only used if `optional_scopes` list contains `couchbase` scope)
+    - `couchbase_superuser_pw`: superuser's password to access Couchbase database (only used if `optional_scopes` list contains `couchbase` scope)
 
 1. Mount the volume into container:
 

docker-jans-configurator/requirements.txt

@@ -1,4 +1,4 @@
 click==6.7
 marshmallow==3.10.0
 fqdn==1.4.0
-git+https://github.com/JanssenProject/jans-cloud-native@eed35d1118df9137989294237623ab4b8e7fac52#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
+git+https://github.com/JanssenProject/jans-cloud-native@3e226fe36da47353ed3c90b330f23bcb9a36342a#egg=jans-pycloudlib&subdirectory=jans-pycloudlib

docker-jans-configurator/scripts/bootstrap.py

@@ -747,6 +747,8 @@ def couchbase_ctx(self):
         # TODO: move this to persistence-loader?
         self.set_config("couchbaseTrustStoreFn", "/etc/certs/couchbase.pkcs12")
         self.set_secret("couchbase_shib_user_password", get_random_chars)
+        self.set_secret("couchbase_password", self.params["couchbase_pw"])
+        self.set_secret("couchbase_superuser_password", self.params["couchbase_superuser_pw"])
 
     def jackrabbit_ctx(self):
         # self.set_secret("jca_pw", get_random_chars())
@@ -757,6 +759,9 @@ def fido2_ctx(self):
         # TODO: hardcoded in persistence-loader?
         self.set_config("fido2ConfigFolder", "/etc/jans/conf/fido2")
 
+    def sql_ctx(self):
+        self.set_secret("sql_password", self.params["sql_pw"])
+
     def generate(self):
         opt_scopes = self.params["optional_scopes"]
 
@@ -788,6 +793,9 @@ def generate(self):
         if "fido2" in opt_scopes:
             self.fido2_ctx()
 
+        if "sql" in opt_scopes:
+            self.sql_ctx()
+
         # populated config
         return self.ctx
 

docker-jans-configurator/scripts/parameter.py

@@ -30,6 +30,7 @@
     "client-api",
     "couchbase",
     "redis",
+    "sql",
 )
 
 
@@ -54,7 +55,6 @@ class Meta:
 
     email = Email(required=True)
 
-    # see validate_fqdn for validation
     hostname = Str(required=True)
 
     org_name = Str(required=True)
@@ -67,9 +67,18 @@ class Meta:
         missing=[],
     )
 
-    # see validate_ldap_pw for validation
     ldap_pw = Str(missing="", default="")
 
+    sql_pw = Str(missing="", default="")
+
+    couchbase_pw = Str(missing="", default="")
+
+    couchbase_superuser_pw = Str(missing="", default="")
+
+    auth_sig_keys = Str(missing="")
+
+    auth_enc_keys = Str(missing="")
+
     @validates("hostname")
     def validate_fqdn(self, value):
         fqdn = FQDN(value)
@@ -82,7 +91,7 @@ def validate_password(self, value, **kwargs):
             raise ValidationError(
                 "Must be at least 6 characters and include "
                 "one uppercase letter, one lowercase letter, one digit, "
-                " and one special character."
+                "and one special character."
             )
 
     @validates_schema
@@ -93,6 +102,24 @@ def validate_ldap_pw(self, data, **kwargs):
             except ValidationError as exc:
                 raise ValidationError({"ldap_pw": exc.messages})
 
+    @validates_schema
+    def validate_ext_persistence_pw(self, data, **kwargs):
+        err = {}
+        scope_attr_map = [
+            ("sql", "sql_pw"),
+            ("couchbase", "couchbase_pw"),
+        ]
+
+        for scope, attr in scope_attr_map:
+            # note we don't enforce custom password validation as cloud-based
+            # databases may use password that not conform to our policy
+            # hence we simply check for empty password only
+            if scope in data["optional_scopes"] and data[attr] == "":
+                err[attr] = ["Empty password isn't allowed"]
+
+        if err:
+            raise ValidationError(err)
+
 
 def params_from_file(path):
     out = {}

docker-jans-fido2/requirements.txt

@@ -1 +1 @@
-git+https://github.com/JanssenProject/jans-cloud-native@eed35d1118df9137989294237623ab4b8e7fac52#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
+git+https://github.com/JanssenProject/jans-cloud-native@3e226fe36da47353ed3c90b330f23bcb9a36342a#egg=jans-pycloudlib&subdirectory=jans-pycloudlib

docker-jans-persistence-loader/requirements.txt

@@ -1,2 +1,2 @@
 ldif==4.1.1
-git+https://github.com/JanssenProject/jans-cloud-native@eed35d1118df9137989294237623ab4b8e7fac52#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
+git+https://github.com/JanssenProject/jans-cloud-native@3e226fe36da47353ed3c90b330f23bcb9a36342a#egg=jans-pycloudlib&subdirectory=jans-pycloudlib

docker-jans-scim/requirements.txt

@@ -1 +1 @@
-git+https://github.com/JanssenProject/jans-cloud-native@eed35d1118df9137989294237623ab4b8e7fac52#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
+git+https://github.com/JanssenProject/jans-cloud-native@3e226fe36da47353ed3c90b330f23bcb9a36342a#egg=jans-pycloudlib&subdirectory=jans-pycloudlib

jans-pycloudlib/jans/pycloudlib/persistence/couchbase.py

@@ -17,13 +17,40 @@
 from jans.pycloudlib.utils import encode_text
 from jans.pycloudlib.utils import cert_to_truststore
 from jans.pycloudlib.utils import as_boolean
-from jans.pycloudlib.utils import secure_password_file
 
 CN_COUCHBASE_TRUSTSTORE_PASSWORD = "newsecret"
 
 logger = logging.getLogger(__name__)
 
 
+def _get_cb_password(manager, password_file, secret_name):
+    """Get Couchbase user's password.
+
+    Priority:
+
+    1. get from password file (for backward-compat)
+    2. get from secrets
+
+    :params manager: An instance of :class:`~jans.pycloudlib.manager._Manager`.
+    :params password_file: Path to file contains password.
+    :params secret_name: Name of the secrets to pull/push the password.
+    :returns: Plaintext password.
+    """
+
+    if os.path.isfile(password_file):
+        with open(password_file) as f:
+            password = f.read().strip()
+            manager.secret.set(secret_name, password)
+            logger.warning(
+                f"Loading password from {password_file} file is deprecated and will be removed in future releases. "
+                f"Note, the password has been saved to secrets with key {secret_name} for later usage."
+            )
+    else:
+        # get from secrets (if any)
+        password = manager.secret.get(secret_name)
+    return password
+
+
 def get_couchbase_user(manager=None) -> str:
     """Get Couchbase username from ``CN_COUCHBASE_USER``
     environment variable (default to ``admin``).
@@ -43,9 +70,9 @@ def get_couchbase_password(manager) -> str:
     :params manager: An instance of :class:`~jans.pycloudlib.manager._Manager`.
     :returns: Plaintext password.
     """
+    secret_name = "couchbase_password"
     password_file = os.environ.get("CN_COUCHBASE_PASSWORD_FILE", "/etc/jans/conf/couchbase_password")
-    salt = manager.secret.get("encoded_salt")
-    return secure_password_file(password_file, salt)
+    return _get_cb_password(manager, password_file, secret_name)
 
 
 def get_couchbase_superuser(manager=None) -> str:
@@ -67,9 +94,9 @@ def get_couchbase_superuser_password(manager) -> str:
     :params manager: An instance of :class:`~jans.pycloudlib.manager._Manager`.
     :returns: Plaintext password.
     """
+    secret_name = "couchbase_superuser_password"
     password_file = os.environ.get("CN_COUCHBASE_SUPERUSER_PASSWORD_FILE", "/etc/jans/conf/couchbase_superuser_password")
-    salt = manager.secret.get("encoded_salt")
-    return secure_password_file(password_file, salt)
+    return _get_cb_password(manager, password_file, secret_name)
 
 
 def prefixed_couchbase_mappings():

jans-pycloudlib/jans/pycloudlib/persistence/sql.py

@@ -15,19 +15,35 @@
 
 from jans.pycloudlib import get_manager
 from jans.pycloudlib.utils import encode_text
-from jans.pycloudlib.utils import secure_password_file
 
 logger = logging.getLogger(__name__)
 
 
 def get_sql_password(manager) -> str:
     """Get password used for SQL database user.
 
+    Priority:
+
+    1. get from password file
+    2. get from secrets
+
     :returns: Plaintext password.
     """
+    secret_name = "sql_password"
     password_file = os.environ.get("CN_SQL_PASSWORD_FILE", "/etc/jans/conf/sql_password")
-    salt = manager.secret.get("encoded_salt")
-    return secure_password_file(password_file, salt)
+
+    if os.path.isfile(password_file):
+        with open(password_file) as f:
+            password = f.read().strip()
+            manager.secret.set(secret_name, password)
+            logger.warning(
+                f"Loading password from {password_file} file is deprecated and will be removed in future releases. "
+                f"Note, the password has been saved to secrets with key {secret_name} for later usage."
+            )
+    else:
+        # get from secrets (if any)
+        password = manager.secret.get(secret_name)
+    return password
 
 
 class BaseClient:

jans-pycloudlib/jans/pycloudlib/utils.py

@@ -437,43 +437,3 @@ def generate_signed_ssl_certkey(suffix, ca_key_fn, ca_cert_fn, email, hostname,
 
     sign_csr(cert_fn, csr, ca_key, ca_cert)
     return cert_fn, key_fn
-
-
-def secure_password_file(password_file, salt):
-    """Secure password file by encoding the contents (if required).
-
-    :param password_file: Path to password file.
-    :param salt: Salt string.
-    :returns: Contents of password file (in plaintext format).
-    """
-    password = ""
-
-    # get password
-    with open(password_file) as f:
-        password = f.read().strip()
-
-    # check if password is encoded; non-encoded and empty password will throw incorrect
-    # padding/bytes which will be handled by encoding the password;
-    # other errors will be thrown automatically by interpreter
-    should_encode = False
-
-    try:
-        password = decode_text(password, salt).decode()
-    except ValueError:
-        if not password:
-            msg = f"Got empty password in {password_file}"
-        else:
-            msg = f"Current password in {password_file} is not encoded"
-        logger.warning(msg)
-        should_encode = True
-
-    if should_encode:
-        logger.warning(f"Attempting to encode the password in {password_file}")
-        try:
-            with open(password_file, "w") as f:
-                f.write(encode_text(password, salt).decode())
-        except Exception as exc:  # noqa: B902
-            logger.warning(f"Unable to encode the password in {password_file}; reason={exc}")
-
-    # returns plain password for compatibility
-    return password