bgruening · gmauro · Nov 11, 2020 · Nov 11, 2020 · Nov 12, 2020 · Nov 12, 2020
diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml
@@ -125,6 +125,187 @@ jobs:
           done;
         shell: bash
         working-directory: ./compose
+  build_chronos:
+    needs: build_cluster_base
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        image:
+          - name: galaxy-server
+          - name: galaxy-configurator
+          - name: galaxy-bioblend-test
+            subdir: tests/
+          - name: galaxy-workflow-test
+            subdir: tests/
+      fail-fast: false
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set image tag
+        id: image_tag
+        run: |
+          if [ "${GITHUB_REF#refs/heads/}" = "master" ]; then
+            echo "::set-output name=image_tag::latest";
+          else
+            echo "::set-output name=image_tag::${GITHUB_REF#refs/heads/}"
+          fi
+      - name: Docker Login
+        run: echo "${{ secrets.docker_registry_password }}" | docker login -u ${{ secrets.docker_registry_username }} --password-stdin ${{ secrets.docker_registry }}
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: crazy-max/ghaction-docker-buildx@v1
+        with:
+          buildx-version: v0.3.1
+      - name: Run Buildx
+        run: |
+          for i in {1..4}; do
+            set +e
+            docker buildx build \
+            --output "type=image,name=${{ secrets.docker_registry }}/${{ secrets.docker_registry_username }}/${{ matrix.image.name }}:${{ steps.image_tag.outputs.image_tag }},push=true" \
+            --build-arg IMAGE_TAG=${{ steps.image_tag.outputs.image_tag }} \
+            --build-arg DOCKER_REGISTRY=${{ secrets.docker_registry }} \
+            --build-arg DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }} \
+            --build-arg GALAXY_REPO=https://github.com/gmauro/galaxy \
+            --build-arg GALAXY_RELEASE=mesos_runner \
+            ${{ matrix.image.subdir }}${{ matrix.image.name }} && break || echo "Fail.. Retrying"
+          done;
+        shell: bash
+        working-directory: ./compose
+  test_chronos:
+    needs: [build_chronos]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        infrastructure:
+          - name: galaxy-chronos
+            files: -f docker-compose.yml -f docker-compose.chronos.yml
+            exclude_test:
+              - workflow_example1
+        test:
+          - name: bioblend
+            files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.bioblend.yml
+            exit-from: galaxy-bioblend-test
+            timeout: 60
+            second_run: "true"
+          - name: workflow_ard
+            files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml
+            exit-from: galaxy-workflow-test
+            workflow: sklearn/ard/ard.ga
+            timeout: 60
+            second_run: "true"
+          - name: workflow_mapping_by_sequencing
+            files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml
+            exit-from: galaxy-workflow-test
+            workflow: training/variant-analysis/mapping-by-sequencing/mapping_by_sequencing.ga
+            timeout: 60
+          - name: workflow_example1
+            files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml
+            exit-from: galaxy-workflow-test
+            workflow: example1/wf3-shed-tools.ga
+            timeout: 60
+          - name: selenium
+            files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.selenium.yml
+            exit-from: galaxy-selenium-test
+            timeout: 60
+      fail-fast: false
+    steps:
+      # Self-made `exclude` as Github Actions currently does not support
+      # exclude/including of dicts in matrices
+      - name: Check if test should be run
+        id: run_check
+        if: contains(matrix.infrastructure.exclude_test, matrix.test.name) != true
+        run: echo ::set-output name=run::true
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set image tag in env
+        run: echo "::set-env name=IMAGE_TAG::${GITHUB_REF#refs/heads/}"
+      - name: Master branch - Set image to to 'latest'
+        if: github.ref == 'refs/heads/master'
+        run: echo "::set-env name=IMAGE_TAG::latest"
+      - name: Set WORKFLOWS env for worfklows-test
+        if: matrix.test.workflow
+        run: echo "::set-env name=WORKFLOWS::${{ matrix.test.workflow }}"
+      - name: Run tests for the first time
+        if: steps.run_check.outputs.run
+        run: |
+          export DOCKER_REGISTRY=${{ secrets.docker_registry }}
+          export DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }}
+          export ${{ matrix.infrastructure.env }}
+          export TIMEOUT=${{ matrix.test.timeout }}
+          docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} config
+          env
+          for i in {1..4}; do
+            echo "Running test - try \#$i"
+            echo "Removing export directory if existent";
+            sudo rm -rf export
+            docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} pull
+            set +e
+            docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} up ${{ matrix.infrastructure.options }} --exit-code-from ${{ matrix.test.exit-from }}
+            test_exit_code=$?
+            error_exit_codes_count=$(expr $(docker ps -a --filter exited=1 | wc -l) - 1)
+            docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} down
+            if [ $error_exit_codes_count != 0 ] || [ $test_exit_code != 0 ] ; then
+              echo "Test failed..";
+              continue;
+            else
+              exit $test_exit_code;
+            fi
+          done;
+          exit 1
+        shell: bash
+        working-directory: ./compose
+        continue-on-error: false
+      - name: Allow upload-artifact read access
+        if: failure()
+        run: sudo chmod -R +r ./compose/export/galaxy/database
+      - name: Save artifacts for debugging a failed test
+        uses: actions/upload-artifact@v1
+        if: failure()
+        with:
+          name: ${{ matrix.infrastructure.name }}_${{ matrix.test.name }}_first-run
+          path: ./compose/export/galaxy/database
+      - name: Clean up after first run
+        if: matrix.test.second_run == 'true'
+        run: |
+          sudo rm -rf export/postgres
+          sudo rm -rf export/galaxy/database
+        working-directory: ./compose
+      - name: Run tests a second time
+        if: matrix.test.second_run == 'true' && steps.run_check.run
+        run: |
+          export DOCKER_REGISTRY=${{ secrets.docker_registry }}
+          export DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }}
+          export ${{ matrix.infrastructure.env }}
+          export TIMEOUT=${{ matrix.test.timeout }}
+          for i in {1..4}; do
+            echo "Running test - try \#$i"
+            echo "Removing export directory if existent";
+            sudo rm -rf export
+            set +e
+            docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} up ${{ matrix.infrastructure.options }} --exit-code-from ${{ matrix.test.exit-from }}
+            test_exit_code=$?
+            error_exit_codes_count=$(expr $(docker ps -a --filter exited=1 | wc -l) - 1)
+            docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} down
+            if [ $error_exit_codes_count != 0 ] || [ $test_exit_code != 0 ] ; then
+              echo "Test failed..";
+              continue;
+            else
+              exit $test_exit_code;
+            fi
+          done;
+          exit 1
+        shell: bash
+        working-directory: ./compose
+        continue-on-error: false
+      - name: Allow upload-artifact read access
+        if: failure() && matrix.test.second_run == 'true'
+        run: sudo chmod -R +r ./compose/export/galaxy/database
+      - name: Save artifacts for debugging a failed test
+        uses: actions/upload-artifact@v1
+        if: failure() && matrix.test.second_run == 'true'
+        with:
+          name: ${{ matrix.infrastructure.name }}_${{ matrix.test.name }}_second-run
+          path: ./compose/export/galaxy/database
   test:
     needs: [build]
     runs-on: ubuntu-latest

diff --git a/compose/docker-compose.chronos.yml b/compose/docker-compose.chronos.yml
@@ -0,0 +1,74 @@
+# Extend Galaxy to run jobs on Mesos
+version: '3.7'
+services:
+  galaxy-configurator:
+    environment:
+      - GALAXY_JOB_RUNNER=chronos
+      - CHRONOS_CONFIG_CHRONOS_HOST=chronos
+      - CHRONOS_CONFIG_CHRONOS_HTTP_PORT=4400
+      - CHRONOS_CONFIG_VOLUMES=${EXPORT_DIR}/galaxy:/galaxy:RW
+  zookeeper:
+    image: indigodatacloud/zookeeper
+    environment:
+      - MYID=1
+      - SERVERS=localhost
+    ports:
+     - "2181:2181"
+     - "2888:2888"
+     - "3888:3888"
+    networks:
+      - galaxy
+  mesosmaster:
+    image: indigodatacloud/mesos-master:1.9.0
+    ports:
+      - "5050:5050"
+    environment:
+      - MESOS_HOSTNAME=mesosmaster
+      - MESOS_CLUSTER=Mesos
+      - MESOS_ZK=zk://zookeeper:2181/mesos
+      - MESOS_LOG_DIR=/var/log/mesos
+      - MESOS_QUORUM=1
+      - MESOS_WORK_DIR=/var/lib/mesos
+      - MESOS_OFFER_TIMEOUT=30secs
+    depends_on:
+      - zookeeper
+    networks:
+      - galaxy
+  mesosslave:
+    image: indigodatacloud/mesos-slave:1.9.0
+    ports:
+      - "5051:5051"
+    pid: host
+    privileged: true
+    environment:
+      MESOS_MASTER: zk://zookeeper:2181/mesos
+      MESOS_CONTAINERIZERS: docker,mesos
+      MESOS_PORT: 5051
+      MESOS_RESOURCES: ports(*):[11000-11999]
+      MESOS_HOSTNAME: mesosslave
+      MESOS_WORK_DIR: /tmp/mesos
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /tmp/mesos:/tmp/mesos
+      - ${EXPORT_DIR}/galaxy:${EXPORT_DIR}/galaxy
+    depends_on:
+      - zookeeper
+      - mesosmaster
+    networks:
+      - galaxy
+  chronos:
+    image: indigodatacloud/chronos:3.0.2_gpu
+    ports:
+      - "4400:4400"
+    environment:
+      - CHRONOS_HOSTNAME=chronos
+      - CHRONOS_HTTP_PORT=4400
+      - CHRONOS_MASTER=zk://zookeeper:2181/mesos
+      - CHRONOS_ZK_HOSTS=zk://zookeeper:2181
+      - CHRONOS_ZK_PATH=/chronos/state
+      - CHRONOS_MESOS_FRAMEWORK_NAME=chronos
+      - CHRONOS_HTTP_CREDENTIALS=admin:s3cret
+      - CHRONOS_DISABLE_AFTER_FAILURES=2
+    networks:
+      - galaxy
diff --git a/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 b/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2
@@ -25,6 +25,15 @@
           <param id="k8s_persistent_volume_claims">{{ GALAXY_K8S_PVC | default('galaxy-root:/galaxy,galaxy-database:/galaxy/database,galaxy-tool-deps:/tool_deps') }}</param>
         </plugin>
         {% endif -%}
+        {% if GALAXY_JOB_RUNNER == 'chronos' -%}
+        <plugin id="chronos" type="runner" load="galaxy.jobs.runners.chronos:ChronosJobRunner">
+          <param id="chronos">{{ CHRONOS_CONFIG_CHRONOS_HOST }}:{{ CHRONOS_CONFIG_CHRONOS_HTTP_PORT }}</param>
+          <param id="owner">foo@bar.com</param>
+          <param id="username">admin</param>
+          <param id="password">s3cret</param>
+          <param id="insecure">true</param>
+        </plugin>
+        {% endif -%}
     </plugins>
     <destinations default="{{ GALAXY_DEPENDENCY_RESOLUTION | default('conda') }}_{{ GALAXY_JOB_RUNNER | default('local') }}">
         <destination id="local" runner="local">
@@ -62,6 +71,14 @@
               <param id="docker_tag_default">{{ GALAXY_K8S_DOCKER_TAG_DEFAULT | default('18.04') }}</param>
               <param id="docker_enabled">true</param>
             {% endif -%}
+            {% if GALAXY_JOB_RUNNER == 'chronos' -%}
+              <param id="docker_enabled">true</param>
+              <param id="docker_memory">512</param>
+              <param id="docker_cpu">2</param>
+              <param id="volumes">{{ CHRONOS_CONFIG_VOLUMES }}</param>
+              <param id="max_retries">2</param>
+              <!-- Number of retries to attempt if a command returns a non-zero status -->
+            {% endif -%}
         </destination>
     </destinations>
     <tools>

diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile
@@ -4,9 +4,6 @@ ARG IMAGE_TAG=latest
 
 FROM buildpack-deps:18.04 as build_base
 
-ARG GALAXY_RELEASE=release_20.05
-ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy
-
 ENV EXPORT_DIR=/export \
     GALAXY_ROOT=/galaxy \
     HTCONDOR_ROOT=/opt/htcondor
@@ -52,6 +49,10 @@ RUN curl -s -L "https://repo.anaconda.com/miniconda/Miniconda2-${MINICONDA_VERSI
     && /usr/bin/common_cleanup.sh
 
 FROM build_base as build_galaxy
+
+ARG GALAXY_RELEASE=release_20.05
+ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy
+
 COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh
 # Install Galaxy
 RUN apt update && apt install --no-install-recommends libcurl4-openssl-dev libssl-dev python3-dev python3-pip -y \
@@ -60,7 +61,7 @@ RUN apt update && apt install --no-install-recommends libcurl4-openssl-dev libss
     && cd $GALAXY_ROOT \
     && ./scripts/common_startup.sh \
     && . $GALAXY_ROOT/.venv/bin/activate \
-    && pip3 install drmaa psycopg2 pycurl pykube \
+    && pip3 install drmaa psycopg2 pycurl pykube chronos-python \
     && deactivate \
     && rm -rf .ci .circleci .coveragerc .gitignore .travis.yml CITATION CODE_OF_CONDUCT.md CONTRIBUTING.md CONTRIBUTORS.md \
               LICENSE.txt Makefile README.rst SECURITY_POLICY.md pytest.ini tox.ini \
@@ -72,6 +73,9 @@ RUN apt update && apt install --no-install-recommends libcurl4-openssl-dev libss
 # --- Final image ---
 FROM $DOCKER_REGISTRY/$DOCKER_REGISTRY_USERNAME/galaxy-cluster-base:$IMAGE_TAG as final
 
+ARG GALAXY_RELEASE=release_20.05
+ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy
+
 COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh
 COPY ./files/create_galaxy_user.py /usr/local/bin/create_galaxy_user.py
 
@@ -80,9 +84,7 @@ ENV EXPORT_DIR=/export \
     GALAXY_PYTHON=/usr/bin/python3 \
     HTCONDOR_ROOT=/opt/htcondor
 
-ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_20.05} \
-    GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \
-    GALAXY_STATIC_DIR=$GALAXY_ROOT/static \
+ENV GALAXY_STATIC_DIR=$GALAXY_ROOT/static \
     GALAXY_EXPORT=$EXPORT_DIR/galaxy \
     GALAXY_CONFIG_DIR=$GALAXY_ROOT/config \
     GALAXY_CONFIG_TOOL_DEPENDENCY_DIR=/tool_deps \