added using access token when pulling from private repo with stage wo…

…rkflow schema, resolves #247

server/kraken/server/bg/jobs.py

@@ -29,7 +29,7 @@
 
 from ..models import db, Run, Job, TestCaseResult, Branch, Flow, Stage, Project, get_setting
 from ..models import AgentsGroup, Agent, System, TestCaseComment, Tool
-from ..models import RepoChanges
+from ..models import RepoChanges, Secret
 from ..schema import prepare_new_planner_triggers
 from ..schema import check_and_correct_stage_schema
 from ..cloud import cloud
@@ -943,7 +943,7 @@ def refresh_schema_repo(stage_id, complete_starting_run_id=None):
             log.error('got unknown stage: %s', stage_id)
             return
 
-        log.info('refresh schema repo for stage: %d, run: %d',
+        log.info('refresh schema repo for stage: %d, run: %s',
                  stage_id, complete_starting_run_id)
 
         planner_url = os.environ.get('KRAKEN_PLANNER_URL', consts.DEFAULT_PLANNER_URL)
@@ -957,7 +957,13 @@ def refresh_schema_repo(stage_id, complete_starting_run_id=None):
 
         try:
             # get schema from repo
-            schema_code, version = gitops.get_schema_from_repo(stage.repo_url, stage.repo_branch, stage.repo_access_token,
+            if stage.repo_access_token:
+                secret = Secret.query.filter_by(project=stage.branch.project, name=stage.repo_access_token).one()
+                repo_access_token = secret.data['secret']
+            else:
+                repo_access_token = None
+
+            schema_code, version = gitops.get_schema_from_repo(stage.repo_url, stage.repo_branch, repo_access_token,
                                                                stage.schema_file, stage.git_clone_params)
 
             # check schema
@@ -966,7 +972,7 @@ def refresh_schema_repo(stage_id, complete_starting_run_id=None):
             stage.repo_error = str(e)
             stage.repo_state = consts.REPO_STATE_ERROR
             db.session.commit()
-            log.exception('problem with schema, stage: %d, run: %d',
+            log.exception('problem with schema, stage: %d, run: %s',
                           stage_id, complete_starting_run_id)
             return
 

server/kraken/server/gitops.py

@@ -18,13 +18,19 @@
 import tempfile
 import subprocess
 
+import furl
+
 from . import minioops
 
 log = logging.getLogger(__name__)
 
 
-def _run(cmd, check=True, cwd=None, capture_output=False, text=None):
-    log.info("execute '%s' in '%s'", cmd, cwd)
+def _run(cmd, check=True, cwd=None, capture_output=False, text=None, secret=None):
+    if secret:
+        cmd2 = cmd.replace(secret, '******')
+    else:
+        cmd2 = cmd
+    log.info("execute '%s' in '%s'", cmd2, cwd)
     p = subprocess.run(cmd, shell=True, check=check, cwd=cwd, capture_output=capture_output, text=text)
     return p
 
@@ -186,9 +192,17 @@ def get_schema_from_repo(repo_url, repo_branch, repo_access_token, schema_file,
         # clone repo
         if not git_clone_params:
             git_clone_params = ''
+
+        if repo_access_token:
+            url = furl.furl(repo_url)
+            url.username = repo_access_token
+            repo_url = url.tostr()
+
         cmd = "git clone --depth 1 --single-branch --branch %s %s '%s' repo" % (repo_branch, git_clone_params, repo_url)
-        p = _run(cmd, check=False, cwd=tmpdir, capture_output=True, text=True)
+        p = _run(cmd, check=False, cwd=tmpdir, capture_output=True, text=True, secret=repo_access_token)
         if p.returncode != 0:
+            if repo_access_token:
+                cmd = cmd.replace(repo_access_token, '******')
             err = "command '%s' returned non-zero exit status %d\n" % (cmd, p.returncode)
             err += p.stdout.strip()[:140] + '\n'
             err += p.stderr.strip()[:140]

server/kraken/server/management.py

@@ -533,7 +533,14 @@ def update_stage(stage_id, body, token_info=None):
         if 'repo_branch' in body:
             stage.repo_branch = body['repo_branch']
         if 'repo_access_token' in body:
-            stage.repo_access_token = body['repo_access_token']
+            repo_access_token = body['repo_access_token']
+            secret = Secret.query.filter_by(project=stage.branch.project, name=repo_access_token).one_or_none()
+            if secret is None:
+                abort(400, "Secret '%s' for access token does not exist" % repo_access_token)
+            if secret.kind != consts.SECRET_KIND_SIMPLE:
+                abort(400, "Type of '%s' access token secret should be Simple Secret" % repo_access_token)
+            stage.repo_access_token = repo_access_token
+
         if 'schema_file' in body:
             stage.schema_file = body['schema_file']
         if 'repo_refresh_interval' in body:

server/pyproject.toml

@@ -62,6 +62,7 @@ casbin = "^1.17.1"
 python-ldap = "^3.4.3"
 Authlib = "^1.1.0"
 setuptools = "^66.0.0"  # required by apscheduler
+furl = "^2.1.3"
 
 [tool.poetry.dev-dependencies]
 pytest = "^7.1.2"