Skip to content

Commit 626afc1

Browse files
sjanssen2sjanssen2
authored
Merge pull request #32 from jlab/add_multi_plugin
Add multi plugin
2 parents 670cd87 + bac91fa commit 626afc1

File tree

4 files changed

+102
-82
lines changed

4 files changed

+102
-82
lines changed

‎.github/workflows/buildContainer.yaml‎

Lines changed: 77 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -173,89 +173,91 @@ jobs:
173173
file: Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile
174174
tags: ghcr.io/${{ github.repository }}/${{ matrix.plugin }}:testcandidate
175175

176-
# build_mulit_plugins:
177-
# needs:
178-
# - build_plugins
179-
# strategy:
180-
# matrix:
181-
# plugin: ["qp-qiime2"]
182-
# runs-on: ubuntu-latest
183-
# steps:
184-
# - name: Checkout tinqiita repo
185-
# uses: actions/checkout@v4
186-
187-
# - name: Set up Docker Buildx
188-
# uses: docker/setup-buildx-action@v3
189-
190-
# - name: Read image version from dockerfile
191-
# id: vars
192-
# run: |
193-
# VERSION=$(head -n 1 Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile | cut -d ":" -f 2- | tr -d " ")
194-
# echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV
195-
# cp Images/${{ matrix.plugin }}/start_${{ matrix.plugin }}.sh Images/test_plugin.sh Images/${{ matrix.plugin }}/requirements.txt Images/trigger_noconda.py Images/trigger.py .
196-
197-
# - name: Download certificates from job build_main
198-
# uses: actions/download-artifact@v4
199-
# with:
200-
# name: certificates
201-
# path: ./
202-
# - name: Download fake references
203-
# uses: actions/download-artifact@v4
204-
# with:
205-
# name: fake_references
206-
# path: ./
207-
208-
# - name: Log in to GitHub Container Registry
209-
# uses: docker/login-action@v3
210-
# with:
211-
# registry: ghcr.io
212-
# username: ${{ github.actor }}
213-
# password: ${{ secrets.GITHUB_TOKEN }}
214-
215-
# - name: Build plugin images
216-
# uses: docker/build-push-action@v6
217-
# with:
218-
# context: .
219-
# load: true
220-
# file: Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile
221-
# tags: ghcr.io/${{ github.repository }}/${{ matrix.plugin }}:testcandidate
222-
# cache-from: type=gha,scope=tinqiita-${{ github.ref_name }}
223-
# cache-to: type=gha,scope=tinqiita-${{ github.ref_name }},mode=max
224-
225-
# - name: adapt compose file to select specific plugin
226-
# run: |
227-
# sed -i "s|||" compose_github.yaml
228-
# sed -i "s/MATRIXPLUGIN/${{ matrix.plugin }}/g" compose_github.yaml
229-
230-
# - name: Run docker compose
231-
# uses: hoverkraft-tech/compose-action@v2.0.1
232-
# with:
233-
# compose-file: "compose_github.yaml"
234-
# services: |
235-
# nginx
236-
237-
# - name: Execute tests in the running services
238-
# run: |
239-
# sleep 5
240-
# docker compose exec ${{ matrix.plugin }} /bin/bash -c "bash /test_plugin.sh"
241-
242-
# - name: Push image to ghcr (only if tests passed)
243-
# if: success()
244-
# uses: docker/build-push-action@v6
245-
# with:
246-
# context: .
247-
# push: true
248-
# file: Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile
249-
# tags: ghcr.io/${{ github.repository }}/${{ matrix.plugin }}:testcandidate
176+
# the qp-qiime2 plugin cannot be tested in isolation, it also needs qtp-diversity and qtp-visualization to be active in qiita
177+
build_mulit_plugins:
178+
needs:
179+
- build_plugins
180+
strategy:
181+
matrix:
182+
multiplugin: ["qp-qiime2"]
183+
runs-on: ubuntu-latest
184+
steps:
185+
- name: Checkout tinqiita repo
186+
uses: actions/checkout@v4
187+
188+
- name: Set up Docker Buildx
189+
uses: docker/setup-buildx-action@v3
190+
191+
- name: Read image version from dockerfile
192+
id: vars
193+
run: |
194+
VERSION=$(head -n 1 Images/${{ matrix.multiplugin }}/${{ matrix.multiplugin }}.dockerfile | cut -d ":" -f 2- | tr -d " ")
195+
echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV
196+
cp Images/${{ matrix.multiplugin }}/start_${{ matrix.multiplugin }}.sh Images/test_plugin.sh Images/${{ matrix.multiplugin }}/requirements.txt Images/trigger_noconda.py Images/trigger.py .
197+
198+
- name: Download certificates from job build_main
199+
uses: actions/download-artifact@v4
200+
with:
201+
name: certificates
202+
path: ./
203+
- name: Download fake references
204+
uses: actions/download-artifact@v4
205+
with:
206+
name: fake_references
207+
path: ./
208+
209+
- name: Log in to GitHub Container Registry
210+
uses: docker/login-action@v3
211+
with:
212+
registry: ghcr.io
213+
username: ${{ github.actor }}
214+
password: ${{ secrets.GITHUB_TOKEN }}
215+
216+
- name: Build plugin images
217+
uses: docker/build-push-action@v6
218+
with:
219+
context: .
220+
load: true
221+
file: Images/${{ matrix.multiplugin }}/${{ matrix.multiplugin }}.dockerfile
222+
tags: ghcr.io/${{ github.repository }}/${{ matrix.multiplugin }}:testcandidate
223+
cache-from: type=gha,scope=tinqiita-${{ github.ref_name }}
224+
cache-to: type=gha,scope=tinqiita-${{ github.ref_name }},mode=max
225+
226+
- name: adapt compose file to select specific plugin
227+
run: |
228+
if [[ "${{ matrix.multiplugin }}" == "qp-qiime2" ]]; then sed -i 's|- QIITA_PLUGINS="MATRIXPLUGIN:"|- QIITA_PLUGINS="${{ matrix.multiplugin }}:qtp-diversity:qtp-visualization:"|' compose_github.yaml; sed -i 's|MATRIXPLUGIN:|${{ matrix.multiplugin }}:\n condition: service_started\n qtp-diversity:\n condition: service_started\n qtp-visualization:|' compose_github.yaml; fi;
229+
230+
- name: Run docker compose
231+
uses: hoverkraft-tech/compose-action@v2.0.1
232+
with:
233+
compose-file: "compose_github.yaml"
234+
services: |
235+
nginx
236+
237+
- name: Execute tests in the running services
238+
run: |
239+
sleep 5
240+
docker compose exec ${{ matrix.multiplugin }} /bin/bash -c "bash /test_plugin.sh"
241+
242+
- name: Push image to ghcr (only if tests passed)
243+
if: success()
244+
uses: docker/build-push-action@v6
245+
with:
246+
context: .
247+
push: true
248+
file: Images/${{ matrix.multiplugin }}/${{ matrix.multiplugin }}.dockerfile
249+
tags: ghcr.io/${{ github.repository }}/${{ matrix.multiplugin }}:testcandidate
250250

251251
publish_images:
252252
needs:
253253
- build_plugins
254254
- build_main
255+
- build_mulit_plugins
255256
strategy:
256257
matrix:
257258
image: ["qp-deblur", "qtp-biom", "qtp-sequencing", "qtp-visualization", "qtp-diversity", "qp-target-gene", "qtp-job-output-folder",
258-
"nginx", "qiita", "plugin_collector"]
259+
"nginx", "qiita", "plugin_collector",
260+
"qp-qiime2"]
259261
runs-on: ubuntu-latest
260262
steps:
261263
- name: Checkout tinqiita repo

‎Images/qp-qiime2/qp-qiime2.dockerfile‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,16 @@ RUN wget https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_
2828

2929
# install tornado based trigger layer in base environment
3030
RUN pip install -U pip
31-
RUN conda install tornado
31+
RUN pip install tornado
3232
COPY trigger.py /trigger.py
3333

3434
# Download qiime2 yaml
3535
RUN wget --quiet https://data.qiime2.org/distro/core/qiime2-2023.5-py38-linux-conda.yml
3636

3737
# Create conda env
38-
RUN conda env create --name qiime2 -y --file qiime2-2023.5-py38-linux-conda.yml
38+
RUN conda env create --name qiime2 -y --file qiime2-2023.5-py38-linux-conda.yml \
39+
&& conda clean --all -y \
40+
&& rm -rf /opt/conda/pkgs
3941
# Make RUN commands use the new environment:
4042
# append --format docker to the build command, see https://github.com/containers/podman/issues/8477
4143
SHELL ["conda", "run", "-p", "/opt/conda/envs/qiime2", "/bin/bash", "-c"]
@@ -61,8 +63,9 @@ RUN export QP_QIIME2_DBS=/databases
6163

6264
# configuring the filtering QZAs available for QIIME 2
6365
RUN mkdir /filtering
64-
RUN wget -O /filtering/bloom-analyses.zip https://github.com/knightlab-analyses/bloom-analyses/archive/refs/heads/master.zip
65-
RUN unzip -j /filtering/bloom-analyses.zip bloom-analyses-master/data/qiime2-artifacts-for-qiita/*.qza -d /filtering/
66+
RUN wget -O /filtering/bloom-analyses.zip https://github.com/knightlab-analyses/bloom-analyses/archive/refs/heads/master.zip \
67+
&& unzip -j /filtering/bloom-analyses.zip bloom-analyses-master/data/qiime2-artifacts-for-qiita/*.qza -d /filtering/ \
68+
&& rm -f /filtering/bloom-analyses.zip
6669
RUN export QP_QIIME2_FILTER_QZA=/filtering/
6770

6871
# TODO: should the plugin get the server configuration?!

‎Images/qp-qiime2/requirements.txt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
tornado
2+
q2-umap
3+
q2-greengenes2
4+
certifi
5+
pip-system-certs
6+
7+
-e /qiita_client
8+
-e /qiita-files
9+
-e /qp-qiime2

‎Images/test_plugin.sh‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,10 @@ else
1111
REQUESTS_CA_BUNDLE="" pip install pytest;
1212
fi;
1313

14-
# clone plugin repository
15-
git clone https://github.com/qiita-spots/${PLUGIN}
14+
if [ "qp-qiime2" != "$PLUGIN" ]; then
15+
# clone plugin repository
16+
git clone https://github.com/qiita-spots/${PLUGIN}
17+
fi;
1618

1719
# NOTE: client api reset only works when communicating with Qitta Master,
1820
# thus, you need to directly address the port of the master container. Don't
@@ -42,4 +44,8 @@ export QIITA_PORT=21174
4244
export QIITA_ROOTCA_CERT=$SSL_CERT_FILE
4345

4446
# change into plugin source directory and execute actual tests
45-
cd ${PLUGIN} && pytest
47+
if [ "qp-qiime2" == "$PLUGIN" ]; then
48+
source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/conda/envs/qiime2; cd ${PLUGIN} && pytest;
49+
else
50+
cd ${PLUGIN} && pytest;
51+
fi;

0 commit comments

Comments
 (0)