Remove pg-post-tasks and add common bash variable and functions

README.md

@@ -75,7 +75,6 @@ There are a few custom roles associated with these playbooks. They are:
 | common    | Performs common tasks on our virtual machines including setting up users, installing some required packages amongst other tasks |
 | mariadb | Sets up MariaDB for Slurm accounting on various clusters |
 | mounts | Sets up NFS mounts for various clusters |
-| pg-post-tasks | Adds some users and extra permissions to the Galaxy Database |
 | slg.db-backup | Role to setup database backups for Galaxy Australia |
 | slg.galaxy_stats | Sets up collection of statistics from Galaxy for Grafana |
 

dev-handler_playbook.yml

@@ -48,7 +48,6 @@
     - galaxyproject.slurm
     - galaxyproject.cvmfs
     - galaxyproject.gxadmin
-    - pg-post-tasks
     - postfix-mail-relay
     - geerlingguy.docker
     - usegalaxy_eu.gie_proxy

dev_playbook.yml

@@ -56,7 +56,6 @@
     - galaxyproject.slurm
     - galaxyproject.cvmfs
     - galaxyproject.gxadmin
-    - pg-post-tasks
     - remote-pulsar-cron
     - galaxy-pg-cleanup
     - galaxyproject.tiaas2

galaxy-handlers_playbook.yml

@@ -69,7 +69,6 @@
         - galaxyproject.slurm
         - galaxyproject.cvmfs
         - galaxyproject.gxadmin
-        - pg-post-tasks
         - postfix-mail-relay
         - geerlingguy.docker
         - dj-wasabi.telegraf

galaxy_playbook.yml

@@ -56,7 +56,6 @@
     - dj-wasabi.telegraf
     - geerlingguy.nfs
     - galaxyproject.gxadmin
-    - pg-post-tasks
     - postfix-mail-relay
     - remote-pulsar-cron
     # - delete-tmp-jwds-cron # commented out until galaxy etca is production galaxy, also needs to be enabled

group_vars/all.yml

@@ -201,12 +201,6 @@ galaxy_cvmfs_server_urls:
       - "http://cvmfs1-tacc0.galaxyproject.org/cvmfs/@fqrn@"
       - "http://cvmfs1-ufr0.galaxyproject.eu/cvmfs/@fqrn@"
 
-#sinfo and squeue configs
-bashrc_env: |
-  export SINFO_FORMAT="%24n %.14C %.8t"
-  export SQUEUE_FORMAT="%8i %9P %.35j %.9T %8r %19S %.10M %.6m %.3C %.3N %.55Z"
-  export SACCT_FORMAT="jobid%8,partition%9,jobname%30,alloccpus,elapsed,totalcpu,END,state,MaxRSS%12,ReqMem,NodeList%24"
-
 # miniconda
 miniconda_version: '4.12.0'
 miniconda_mamba_version: '0.23.3'

group_vars/galaxyservers.yml

@@ -117,6 +117,8 @@ galaxy_venv_dir: "{{ galaxy_root }}/venv" # set this explicitly as it can fail w
 galaxy_mutable_config_dir: "{{ galaxy_root }}/var"
 galaxy_mutable_data_dir: "{{ galaxy_root }}"
 galaxy_config_dir: "{{ galaxy_root }}/config"
+galaxy_job_working_directory: "{{ galaxy_tmp_dir }}/job_working_directory"
+galaxy_pulsar_app_config_file: "{{ galaxy_config_dir }}/pulsar_app.yml"
 galaxy_conda_prefix: "{{ galaxy_tools_indices_dir }}/tool_dependencies/_conda"
 galaxy_conda_exec: mamba
 
@@ -401,6 +403,7 @@ lsyncd_source: /mnt/galaxy
 lsyncd_target: /mnt/ghost-galaxy-app
 lsyncd_exclude:
   - cache
+  - log
 lsyncd_max_user_watches: 524288
 
 # debugging settings

hosts

@@ -143,18 +143,33 @@ pulsar_qld_himems
 pulsar_high_mem2
 pulsar_high_mem1
 pulsar_qld_blast
+pulsar_qld_gpus
 pulsar_nci_training_head
 pulsar_azure_0_head
 
+# All pulsar head nodes
+[pulsar_head_nodes:children]
+production_pulsars
+dev_pulsar_head
+staging_pulsar_head
+pulsar_nci_test_head
+
+# All pulsar workers
+[pulsar_workers:children]
+pulsar_mel3_workers
+pulsar_mel2_workers
+pulsar_QLD_workers
+pulsar_nci_training_workers
+pulsar_nci_test_workers
+
 # Dev cluster (GenomicsVL_Devt)
-[dev_group]
-dev-pulsar ansible_ssh_host=45.113.233.82
 [dev_group:children]
 dev_db_server
 dev_galaxy_server
 dev_handlers_server
 dev_slurm_head
 dev_workers
+dev_pulsar_head
 
 [dev_db_server]
 dev-db ansible_ssh_host=45.113.232.252
@@ -172,14 +187,16 @@ dev-queue ansible_ssh_host=45.113.232.149
 dev-w1 ansible_ssh_host=45.113.233.251
 dev-w2 ansible_ssh_host=45.113.233.7
 
+[dev_pulsar_head]
+dev-pulsar ansible_ssh_host=45.113.233.82
+
 # Staging cluster (GenomicsVL)
-[staging_group]
-staging-pulsar ansible_ssh_host=115.146.87.193
 [staging_group:children]
 staging_db_server
 staging_galaxy_server
 staging_slurm_head
 staging_workers
+staging_pulsar_head
 
 [staging_db_server]
 staging-db ansible_ssh_host=115.146.87.93
@@ -193,6 +210,9 @@ staging-queue ansible_ssh_host=115.146.84.121
 [staging_workers]
 staging-w1 ansible_ssh_host=115.146.87.130
 
+[staging_pulsar_head]
+staging-pulsar ansible_ssh_host=115.146.87.193
+
 # CVMFS
 [cvmfsstratum1servers]
 cvmfs1-mel0.gvl.org.au ansible_user=ec2-user ansible_ssh_host=115.146.85.207

roles/common/defaults/main.yml

@@ -8,3 +8,11 @@ htpass_username: "ubuntu"
 internal_ssh_keys: {}
 
 extra_keys: []
+
+common_sinfo_format: "%24n %.14C %.8t"
+common_squeue_format: "%8i %9P %.35j %.9T %8r %19S %.10M %.6m %.3C %.3N %.55Z"
+common_sacct_format: "jobid%8,partition%9,jobname%30,alloccpus,elapsed,totalcpu,END,state,MaxRSS%12,ReqMem,NodeList%24"
+
+common_bashrc_dir: /etc/bashrc_common
+common_bashrc_vars_file: "{{ common_bashrc_dir }}/common_variables"
+common_bashrc_functions_file: "{{ common_bashrc_dir }}/common_functions"

roles/common/tasks/bashrc.yml

@@ -0,0 +1,117 @@
+---
+- name: Ensure directory for bashrc common files
+  file:
+    state: directory
+    path: "{{ common_bashrc_dir }}"
+- name: Template bashrc_common variables path
+  copy:
+    dest: "{{ common_bashrc_vars_file }}"
+    mode: 0755
+    content: |
+        # Common variables for ubuntu and human users
+
+        # slurm
+        export SINFO_FORMAT="{{ common_sinfo_format }}"
+        export SQUEUE_FORMAT="{{ common_squeue_format }}"
+        export SINFO_FORMAT="{{ common_sacct_format }}"
+
+        # helpful settings
+        export EDITOR=vim
+        HISTSIZE=
+        HISTFILESIZE=
+
+        {% if is_galaxy_head_node %}
+        # galaxy
+        export PGHOST='{{ db_address }}'
+        export PGDATABASE='galaxy'
+        export PGPORT='5432'
+        export GALAXY_CONFIG_DIR={{ galaxy_config_dir }}
+        export GALAXY_MUTABLE_CONFIG_DIR={{ galaxy_mutable_config_dir }}
+        export GALAXY_ROOT={{ galaxy_server_dir }}
+        export GALAXY_CONFIG_FILE={{ galaxy_config_file }}
+        export GALAXY_JWD_PATH="{{ galaxy_job_working_directory }}"
+        export GALAXY_LOG_DIR="{{ galaxy_log_dir }}"
+        export GALAXY_PULSAR_APP_CONF="{{ galaxy_pulsar_app_config_file }}"
+        {% endif %}
+- name: Template common_bashrc functions path
+  copy:
+    dest: "{{ common_bashrc_functions_file }}"
+    mode: 0755
+    content: |
+        # Common functions for ubuntu and human users
+
+        {% if is_galaxy_head_node %}
+        # galaxy
+        jwd() { # print path of job working directory for job id
+            python "{{ common_bashrc_dir }}/get_jwd_path_galaxy.py" $1
+        }
+
+        jwd-size() {
+          du -sh $(jwd $1)
+        }
+
+        go-jwd() { # go to job working directory for job id
+          cd $(jwd $1)
+        }
+
+        tail-stderr() {
+          stderr_filename=$(jwd $1)/outputs/tool_stderr
+          if [ ! -f $stderr_filename ]; then
+            echo "No stderr file for job $1";
+          else
+            tail $stderr_filename;
+          fi
+        }
+        {% endif %}
+
+        {% if ansible_hostname in groups['pulsar_head_nodes'] %}
+        # pulsar
+        jwd() { # print path of job working directory for job id
+            echo {{ pulsar_staging_dir }}/${1}
+        }
+
+        jwd-size() {
+          du -sh $(jwd $1)
+        }
+
+        go-jwd() { # go to job working directory for job id
+          cd $(jwd $1)
+        }
+
+        tail-stderr() {
+          stderr_filename=$(jwd $1)/metadata/tool_stderr
+          if [ ! -f $stderr_filename ]; then
+            echo "No stderr file for job $1";
+          else
+            tail $stderr_filename;
+          fi
+        }
+        {% endif %}
+
+- name: Template helper scripts
+  template:
+    src: "bashrc/{{ item }}.j2"
+    dest: "{{ common_bashrc_dir }}/{{ item }}"
+  with_items:
+    - get_jwd_path_galaxy.py
+
+- name: Add to .bashrc files
+  blockinfile:
+    marker: "# {mark} ANSIBLE MANAGED BLOCK (common role: source common bash env for users)"
+    dest: "/home/{{ item.name }}/.bashrc"
+    block: |
+      source {{ common_bashrc_vars_file }}
+      source {{ common_bashrc_functions_file }}
+
+      {% if is_galaxy_head_node %}
+      # galaxy (per user)
+      {% if item == 'ubuntu' %}
+      export PGUSER='galaxy'
+      {% elif 'tiaas_admin' in item.roles|d([]) %}
+      export PGUSER='tiaasadmin'
+      {% else %}
+      export PGUSER='reader'
+      {% endif %}
+      {% endif %}
+
+  with_items: "{{ machine_users + [{'name': 'ubuntu'}] }}"

roles/common/tasks/machine_users.yml

@@ -4,13 +4,6 @@
   become_user: root
   block:
 
-  # FIX for deprecated code that added user files in /etc/sudoers.d/ - can be removed in future
-  - name: delete /etc/sudoers.d/username files for machine users
-    file:
-      path: "/etc/sudoers.d/{{ item.name }}"
-      state: absent
-    with_items: "{{ remove_machine_users + machine_users }}"
-
   - name: Ensure group "devs" exists
     group:
       name: devs
@@ -65,18 +58,41 @@
       force: yes 
     with_items: "{{ remove_machine_users }}"
 
-  - name: Add some env_vars to .bashrcs
-    blockinfile:
-      path: "/home/{{ item.name }}/.bashrc"
-      block: |
-        {{ bashrc_env }}
-      marker: "# {mark} MANAGED BY ANSIBLE - DO NOT MODIFY"
-    with_items: "{{ machine_users + [{'name': 'ubuntu'}] }}"
-    when: bashrc_env is defined
-
   - name: Add SSH keys for users
     authorized_key:
       user: "{{ item.name }}"
       key: "{{ lookup('file', item.key) }}"
       state: present
     with_items: "{{ machine_users }}"
+
+  # former pg-post-tasks tasks
+  - name: Pgpass creation and gxadmin user tasks
+    when: is_galaxy_head_node|d(False)
+    block:
+    - name: Create .pgpass files for users
+      copy:
+        dest: "/home/{{ item.name }}/.pgpass"
+        content: "{{ db_address }}:5432:*:{{ db_role }}:{{ db_password }}"
+        mode: "600"
+        group: "{{ item.name }}"
+        owner: "{{ item.name }}"
+      vars:
+        db_role: "{{ 'galaxy' if item.name == 'ubuntu' else ('reader' if not 'tiaas_admin' in item.roles|d([]) else 'tiaasadmin') }}"
+        db_password: "{{ pg_db_password[db_role] }}"
+      with_items: "{{ machine_users + [{'name': 'ubuntu'}] }}"
+    - name: Ensure gxadmin config directory exists for all users including ubuntu
+      file:
+        path: "/home/{{ item }}/.config"
+        state: directory
+        group: "{{ item }}"
+        owner: "{{ item }}"
+      with_items: "{{ (machine_users | map(attribute='name') | list) + ['ubuntu'] }}"
+    - name: Ensure that all users + ubuntu have a copy of gxadmin-local.sh
+      copy: 
+        src: files/galaxy/gxadmin/gxadmin-local.sh
+        dest: "/home/{{ item }}/.config/gxadmin-local.sh"
+        mode: "755"
+        group: "{{ item }}"
+        owner: "{{ item }}"
+        force: yes
+      with_items: "{{ (machine_users | map(attribute='name') | list) + ['ubuntu'] }}"

roles/common/tasks/main.yml

@@ -83,6 +83,13 @@
 
 - name: Add machine users
   import_tasks: machine_users.yml
+  tags:
+    - users
+
+- name: Add variables to bashrc files
+  import_tasks: bashrc.yml
+  tags:
+    - users
 
 - name: Add extra ssh keys
   include_tasks: extra_keys.yml
@@ -120,4 +127,4 @@
   lineinfile:
     path: /etc/environment
     create: true
-    line: export SACCT_FORMAT="jobid%8,partition%9,jobname%30,alloccpus,elapsed,totalcpu,END,state,MaxRSS%12,ReqMem,NodeList%24"
+    line: export SACCT_FORMAT="{{ common_sacct_format }}"

roles/common/templates/bashrc/get_jwd_path_galaxy.py.j2

@@ -0,0 +1,26 @@
+import argparse
+import os
+
+def main():
+    parser = argparse.ArgumentParser(description='Get job working directory from a job id')
+    parser.add_argument('job_id', type=int, help='Galaxy job id')
+    args = parser.parse_args()
+
+    prefix = os.environ.get("GALAXY_JWD_PATH")
+    if not prefix:
+        raise ValueError("Please set GALAXY_JWD_PATH environment variable")
+
+    jwd_path = get_jwd_path(args.job_id, prefix)
+    print(jwd_path)
+
+def get_jwd_path(job_id, prefix):
+    if len(str(job_id)) > 6: # on production this is the case
+        nine_digit_id = '0'*(9-len(str(job_id))) + str(job_id)
+        return os.path.join(prefix, nine_digit_id[:3], nine_digit_id[3:6], str(job_id))
+    elif len(str(job_id)) <= 6:
+        six_digit_id = '0'*(6-len(str(job_id))) + str(job_id)
+        return os.path.join(prefix, six_digit_id[:3], str(job_id))
+
+
+if __name__ == '__main__':
+    main()