pawsey_s3 to test #973
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Check tool destinations and job configuration | |
| # # Controls when the action will run. Triggers the workflow on push or pull request | |
| # # events but only for the master branch | |
| on: | |
| pull_request: | |
| branches: [ master ] | |
| paths: | |
| - templates/galaxy/config/galaxy_job_conf.yml.j2 | |
| - files/galaxy/dynamic_job_rules/production/total_perspective_vortex/* # check | |
| # # A workflow run is made up of one or more jobs that can run sequentially or in parallel | |
| jobs: | |
| # This workflow contains a single job called "build" | |
| build: | |
| # The type of runner that the job will run on | |
| runs-on: ubuntu-latest | |
| # Steps represent a sequence of tasks that will be executed as part of the job | |
| steps: | |
| # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 # number of commits to fetch. 0 indicates all history for all branches and tags | |
| # Set up python environment | |
| - name: Set up Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.8 | |
| # Install python dependencies for check_files script | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install total-perspective-vortex==$TPV_VERSION pyyaml galaxy-app==23.1.4 galaxy-auth==23.1.4 galaxy-config==23.1.4 galaxy-data==23.1.4 galaxy-files==23.1.4 galaxy-job-execution==23.1.4 galaxy-job-metrics==23.1.4 galaxy-navigation==23.1.4 galaxy-objectstore==23.1.4 galaxy-tool-util==23.1.4 galaxy-tours==23.1.4 galaxy-util==23.1.4 galaxy-web-framework==23.1.4 galaxy-web-stack==23.1.4 | |
| env: | |
| TPV_VERSION: '2.3.2' | |
| - name: Check TPV files and job conf | |
| env: | |
| SETTING: github | |
| shell: python | |
| run: | | |
| """This code uses ansible to template out the jinja templates for tpv configs and job config, | |
| then runs the tpv linter on the resultant files.""" | |
| import re | |
| import subprocess | |
| import os | |
| import shutil | |
| working_dir = '.github/workflows/check_tpv_config' | |
| hosts_file = os.path.join(working_dir, 'hosts') | |
| playbook_file = os.path.join(working_dir, 'template_playbook.yml') | |
| files_dir = os.path.join(working_dir, 'files') | |
| templates_dir = os.path.join(working_dir, 'templates') | |
| os.mkdir(files_dir) | |
| os.mkdir(templates_dir) | |
| tpv_dir = 'files/galaxy/dynamic_job_rules/production/total_perspective_vortex' | |
| tpv_config_files = [os.path.join(tpv_dir, f) for f in os.listdir(tpv_dir)] | |
| job_conf_template = 'templates/galaxy/config/galaxy_job_conf.yml.j2' | |
| tpv_files = [] | |
| replace_patterns = ['\{\{\s?vault_\w+\s?\}\}', '\{\{\s?hostname\s?\}\}'] | |
| with open('.vault_pass.txt', 'w') as password_file: | |
| password_file.write('FalsePassword') | |
| def template_file(fname): | |
| output_fname = fname.replace(templates_dir, files_dir).replace('.j2', '') | |
| extra_vars = f'src={fname.replace(templates_dir+"/", "")} dest={os.path.realpath(output_fname)}' | |
| ansible_command = f'ansible-playbook -i {hosts_file} {playbook_file} --extra-vars \'{extra_vars}\'' | |
| try: | |
| subprocess.check_output(ansible_command, shell=True) | |
| except subprocess.CalledProcessError as e: | |
| print(e.output) | |
| raise Exception('Failed to template {os.path.basename(fname)}') | |
| return output_fname | |
| def sanitize_text(fname): | |
| """Replaces any references to vault variables with a constant string as the | |
| vault is not decrypted during tests""" | |
| with open(fname) as handle: | |
| text = handle.read() | |
| for rp in replace_patterns: | |
| text = re.sub(rp, 'text_replaced', text) | |
| with open(fname, 'w') as handle: | |
| handle.write(text) | |
| for t in tpv_config_files + [job_conf_template]: | |
| if t.endswith('.yml'): | |
| test_fname = os.path.join(files_dir, os.path.basename(t)) | |
| shutil.copy(t, test_fname) | |
| tpv_files.append(test_fname) | |
| elif t.endswith('.yml.j2'): | |
| test_fname = os.path.join(templates_dir, os.path.basename(t)) | |
| shutil.copy(t, test_fname) | |
| sanitize_text(test_fname) | |
| templated_file = template_file(test_fname) | |
| if 'tpv_dir' in t: | |
| tpv_files.append(templated_file) | |
| for t in tpv_files: | |
| subprocess.check_output(f'tpv lint {t}', shell=True) |