Merge pull request #209 from dometto/prebuilt_client

Support prebuilt client

.github/workflows/molecule.yml

@@ -16,6 +16,7 @@ jobs:
       matrix:
         scenario:
           - default
+          - prebuilt_client
         from-image:
           - centos:7
           - rockylinux:8
@@ -34,6 +35,10 @@ jobs:
             galaxy-version: '21.01'
           - from-image: ubuntu:22.04
             galaxy-version: '22.01'
+          - scenario: prebuilt_client
+            galaxy-version: '22.01'
+          - scenario: prebuilt_client
+            galaxy-version: '22.05'       
     steps:
       - uses: actions/checkout@v3
         with:

defaults/main.yml

@@ -425,6 +425,12 @@ galaxy_uwsgi_config_default:
 # Client build settings
 #
 
+
+# If this is set to true, instead of building the client from source, a prebuilt version of the client will be installed from NPM.
+# This mimics the behavior of the GALAXY_INSTALL_PREBUILT_CLIENT=1 setting when running Galaxy with run.sh.
+# When setting this to true, none of the other client settings below will be used.
+galaxy_client_use_prebuilt: false
+
 # Options include: client / client-production / client-production-maps (default). Set to null to invoke each step of the
 # build separately, but this is not guaranteed to work for all version of Galaxy - using `make` is the safer choice.
 galaxy_client_make_target: client-production-maps

molecule/_common/_inc_pre_tasks.yml

@@ -0,0 +1,46 @@
+---
+- name: Install dependencies (yum)
+  yum:
+    # For completeness this should include git, but on EL7 we install a newer git from SCL in the Dockerfile to deal
+    # with automatic line ending conversion in the default git 1.8 breaking idempotence
+    #name: [sudo, git, make, bzip2]
+    name: [sudo, make, bzip2]
+  when: ansible_os_family == "RedHat" and ansible_distribution_major_version is version("8", "<")
+- name: Install dependencies (dnf)
+  dnf:
+    name: [sudo, git, make, bzip2]
+  when: ansible_os_family == "RedHat" and ansible_distribution_major_version is version("8", ">=")
+- name: Install dependencies (apt)
+  apt:
+    name:
+      - sudo
+      - git
+      - make
+      - python3-venv
+      - python3-setuptools
+      - python3-dev
+      - python3-psycopg2
+      - gcc
+      - acl
+      - gnutls-bin
+      - libmagic-dev
+  when: ansible_os_family == "Debian"
+
+# This is to cheat at idempotence, which will fail if a commit is merged between converge and idempotence.
+- name: Check whether server dir exists
+  stat:
+    path: "{{ galaxy_root }}/server"
+  register: __molecule_dir_check
+- name: Collect current commit id
+  git:
+    clone: false
+    depth: "{{ galaxy_clone_depth }}"
+    dest: "{{ galaxy_root }}/server"
+    repo: https://github.com/galaxyproject/galaxy.git
+  changed_when: false
+  become: "{{ __molecule_dir_check.stat.exists }}"
+  become_user: "{{ galaxy_privsep_user }}"
+  register: __molecule_git_check
+- name: Set galaxy_commit_id
+  set_fact:
+    galaxy_commit_id: "{{ __molecule_git_check.before or ((__galaxy_version == 'dev') | ternary('dev', 'release_' ~ __galaxy_version)) }}"

molecule/default/converge.yml

@@ -26,51 +26,6 @@
         conda_auto_init: false
     pip_virtualenv_command: /usr/bin/python3 -m venv
   pre_tasks:
-
-    - name: Install dependencies (yum)
-      yum:
-        # For completeness this should include git, but on EL7 we install a newer git from SCL in the Dockerfile to deal
-        # with automatic line ending conversion in the default git 1.8 breaking idempotence
-        #name: [sudo, git, make, bzip2]
-        name: [sudo, make, bzip2]
-      when: ansible_os_family == "RedHat" and ansible_distribution_major_version is version("8", "<")
-    - name: Install dependencies (dnf)
-      dnf:
-        name: [sudo, git, make, bzip2]
-      when: ansible_os_family == "RedHat" and ansible_distribution_major_version is version("8", ">=")
-    - name: Install dependencies (apt)
-      apt:
-        name:
-          - sudo
-          - git
-          - make
-          - python3-venv
-          - python3-setuptools
-          - python3-dev
-          - python3-psycopg2
-          - gcc
-          - acl
-          - gnutls-bin
-          - libmagic-dev
-      when: ansible_os_family == "Debian"
-
-    # This is to cheat at idempotence, which will fail if a commit is merged between converge and idempotence.
-    - name: Check whether server dir exists
-      stat:
-        path: "{{ galaxy_root }}/server"
-      register: __molecule_dir_check
-    - name: Collect current commit id
-      git:
-        clone: false
-        depth: "{{ galaxy_clone_depth }}"
-        dest: "{{ galaxy_root }}/server"
-        repo: https://github.com/galaxyproject/galaxy.git
-      changed_when: false
-      become: "{{ __molecule_dir_check.stat.exists }}"
-      become_user: "{{ galaxy_privsep_user }}"
-      register: __molecule_git_check
-    - name: Set galaxy_commit_id
-      set_fact:
-        galaxy_commit_id: "{{ __molecule_git_check.before or ((__galaxy_version == 'dev') | ternary('dev', 'release_' ~ __galaxy_version)) }}"
+    - include_tasks: "../_common/_inc_pre_tasks.yml"
   roles:
     - role: galaxyproject.galaxy

molecule/prebuilt_client/converge.yml

@@ -0,0 +1,32 @@
+---
+- name: Converge
+  hosts: all
+  vars:
+    galaxy_client_use_prebuilt: true # This scenario tests with the prebuilt client
+    __galaxy_version: "{{ lookup('env', 'GALAXY_VERSION') }}"
+    galaxy_create_user: yes
+    galaxy_manage_paths: yes
+    galaxy_manage_clone: yes
+    galaxy_manage_download: no
+    galaxy_manage_existing: no
+    galaxy_manage_systemd: yes
+    # role default is < 22.05
+    galaxy_manage_gravity: "{{ false if __galaxy_major_version is version('22.01', '<') else true }}"
+    galaxy_systemd_mode: "{{ 'mule' if __galaxy_major_version is version('22.01', '<') else 'gravity' }}"
+    galaxy_config_style: yaml
+    galaxy_layout: root-dir
+    galaxy_root: /srv/galaxy
+    galaxy_separate_privileges: yes
+    galaxy_user: galaxy
+    galaxy_group: galaxy
+    galaxy_privsep_user: gxpriv
+    galaxy_clone_depth: 1
+    galaxy_config:
+      galaxy:
+        database_connection: sqlite:///{{ galaxy_mutable_data_dir }}/universe.sqlite
+        conda_auto_init: false
+    pip_virtualenv_command: /usr/bin/python3 -m venv
+  pre_tasks:
+    - include_tasks: "../_common/_inc_pre_tasks.yml"
+  roles:
+    - role: galaxyproject.galaxy

molecule/prebuilt_client/molecule.yml

@@ -0,0 +1,26 @@
+---
+dependency:
+  name: galaxy
+  options:
+    force: false  # don't fetch every time
+    role-file: molecule/_common/requirements.yml
+    requirements-file: molecule/_common/requirements.yml
+driver:
+  name: docker
+platforms:
+  - name: galaxy-scenario-prebuilt_client
+    image: ${FROM_IMAGE:-centos:7}
+    platform: linux/amd64
+    # override default sleep loop, use Dockerfile CMD
+    command: ""
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:rw
+    privileged: true
+    dockerfile: ../_common/Dockerfile.j2
+    pre_build_image: false
+provisioner:
+  name: ansible
+  env:
+    GALAXY_VERSION: ${GALAXY_VERSION:-dev}
+verifier:
+  name: ansible

molecule/prebuilt_client/verify.yml

@@ -0,0 +1,16 @@
+---
+
+- name: Verify
+  hosts: all
+  vars:
+    __galaxy_version: "{{ lookup('env', 'GALAXY_VERSION') }}"
+  tasks:
+    - name: Check version
+      uri:
+        url: http://localhost:8080/api/version
+      register: response
+      failed_when: response.status != 200 or (__galaxy_version != 'dev' and response.json.version_major != lookup('env', 'GALAXY_VERSION'))
+      # this is just to wait until the server is up, we can stop retrying on any response
+      until: response.status > 0
+      retries: 60
+      delay: 1

tasks/_inc_client_install_tools.yml

@@ -0,0 +1,40 @@
+---
+
+- name: Install client build tools
+  block:
+    - name: Ensure galaxy_node_version is set
+      include_tasks: _inc_node_version.yml
+      when: galaxy_node_version is undefined
+
+    - name: Check if node is installed
+      stat:
+        path: "{{ galaxy_venv_dir }}/bin/node"
+      register: __galaxy_node_is_installed
+
+    - name: Collect installed node version
+      command: "{{ galaxy_venv_dir }}/bin/node --version"
+      when: __galaxy_node_is_installed.stat.exists
+      changed_when: false
+      register: __galaxy_installed_node_version
+
+    - name: Remove node_modules directory when upgrading node
+      file:
+        path: "{{ galaxy_venv_dir }}/lib/node_modules"
+        state: absent
+      when: (not __galaxy_node_is_installed.stat.exists) or (('v' ~ galaxy_node_version) != __galaxy_installed_node_version.stdout)
+
+    - name: Install or upgrade node
+      command: "nodeenv -n {{ galaxy_node_version }} -p"
+      environment:
+        PATH: "{{ galaxy_venv_dir }}/bin:{{ ansible_env.PATH }}"
+        VIRTUAL_ENV: "{{ galaxy_venv_dir }}"
+      when: (not __galaxy_node_is_installed.stat.exists) or (('v' ~ galaxy_node_version) != __galaxy_installed_node_version.stdout)
+
+    - name: Install yarn
+      npm:
+        executable: "{{ galaxy_venv_dir }}/bin/npm"
+        name: yarn
+        global: yes
+      environment:
+        PATH: "{{ galaxy_venv_dir }}/bin:{{ ansible_env.PATH }}"
+        VIRTUAL_ENV: "{{ galaxy_venv_dir }}"

tasks/client.yml

@@ -46,42 +46,8 @@
           changed_when: true
           when: __galaxy_from_git.stat.exists
 
-        - name: Ensure galaxy_node_version is set
-          include_tasks: _inc_node_version.yml
-          when: galaxy_node_version is undefined
-
-        - name: Check if node is installed
-          stat:
-            path: "{{ galaxy_venv_dir }}/bin/node"
-          register: __galaxy_node_is_installed
-
-        - name: Collect installed node version
-          command: "{{ galaxy_venv_dir }}/bin/node --version"
-          when: __galaxy_node_is_installed.stat.exists
-          changed_when: false
-          register: __galaxy_installed_node_version
-
-        - name: Remove node_modules directory when upgrading node
-          file:
-            path: "{{ galaxy_venv_dir }}/lib/node_modules"
-            state: absent
-          when: (not __galaxy_node_is_installed.stat.exists) or (('v' ~ galaxy_node_version) != __galaxy_installed_node_version.stdout)
-
-        - name: Install or upgrade node
-          command: "nodeenv -n {{ galaxy_node_version }} -p"
-          environment:
-            PATH: "{{ galaxy_venv_dir }}/bin:{{ ansible_env.PATH }}"
-            VIRTUAL_ENV: "{{ galaxy_venv_dir }}"
-          when: (not __galaxy_node_is_installed.stat.exists) or (('v' ~ galaxy_node_version) != __galaxy_installed_node_version.stdout)
-
-        - name: Install yarn
-          npm:
-            executable: "{{ galaxy_venv_dir }}/bin/npm"
-            name: yarn
-            global: yes
-          environment:
-            PATH: "{{ galaxy_venv_dir }}/bin:{{ ansible_env.PATH }}"
-            VIRTUAL_ENV: "{{ galaxy_venv_dir }}"
+        - name: Include client build tools process
+          include_tasks: "_inc_client_install_tools.yml"
 
         - name: Include client build process
           include_tasks: "_inc_client_build_{{ 'make' if galaxy_client_make_target is not none else 'steps' }}.yml"
@@ -91,3 +57,40 @@
   remote_user: "{{ galaxy_remote_users.privsep | default(__galaxy_remote_user) }}"
   become: "{{ true if galaxy_become_users.privsep is defined else __galaxy_become }}"
   become_user: "{{ galaxy_become_users.privsep | default(__galaxy_become_user) }}"
+  when: not galaxy_client_use_prebuilt
+
+- name: Install prebuilt client
+  block:
+    - name: Ensure prebuilt client is supported
+      assert:
+        that:
+          - __galaxy_major_version is version('23.0', '>=')
+        success_msg: "Prebuilt client is supported"
+        fail_msg: "Prebuilt client is not supported for Galaxy version {{ __galaxy_major_version }}, '>= 23.0' required."
+
+    - name: Include client install tools process
+      include_tasks: "_inc_client_install_tools.yml"
+
+    - name: Install prebuilt client with yarn
+      yarn:
+        executable: "yarn --check-files"
+        path: "{{ galaxy_server_dir }}"
+      environment:
+        PATH: "{{ galaxy_venv_dir }}/bin:{{ ansible_env.PATH }}"
+        VIRTUAL_ENV: "{{ galaxy_venv_dir }}"
+      register: __yarn_install
+      changed_when: "'already up-to-date' not in __yarn_install.out | lower"
+
+    - name: Stage prebuilt client # noqa no-changed-when
+      command: "yarn run stage"
+      args:
+        chdir: "{{ galaxy_server_dir }}"
+      environment:
+        PATH: "{{ galaxy_server_dir }}/client/node_modules/.bin:{{ galaxy_venv_dir }}/bin:{{ ansible_env.PATH }}"
+        VIRTUAL_ENV: "{{ galaxy_venv_dir }}"
+      when: __yarn_install.changed
+
+  remote_user: "{{ galaxy_remote_users.privsep | default(__galaxy_remote_user) }}"
+  become: "{{ true if galaxy_become_users.privsep is defined else __galaxy_become }}"
+  become_user: "{{ galaxy_become_users.privsep | default(__galaxy_become_user) }}"
+  when: galaxy_client_use_prebuilt