Skip to content

Commit 879c468

Browse files
sjanssen2sjanssen2
committed
test all other plugins (that are small by now)
1 parent 6fff701 commit 879c468

File tree

5 files changed

+124
-34
lines changed

5 files changed

+124
-34
lines changed

‎.github/workflows/buildContainer.yaml‎

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ jobs:
99
docker:
1010
strategy:
1111
matrix:
12-
plugin: ["qtp-sequencing" # "qp-deblur", "qtp-biom", "qp-target-gene", "qtp-sequencing", "nginx", "qtp-visualization", #"qtp-diversity",
13-
# "qiita", "plugin_collector"
12+
plugin: ["qp-deblur", "qtp-biom", "qp-target-gene", "qtp-sequencing", "qtp-visualization",
13+
"nginx", "qiita", "plugin_collector"
1414
]
1515
runs-on: ubuntu-latest
1616
steps:
@@ -56,6 +56,7 @@ jobs:
5656
sed -i "s|image: janssenlab/${{ matrix.plugin }}:latest|image: tinqiita/${{ matrix.plugin }}:testcandidate|" compose_github.yaml
5757
5858
- name: Run docker compose
59+
if: ${{ !contains('nginx,qiita,plugin_collector', matrix.plugin) }}
5960
uses: hoverkraft-tech/compose-action@v2.0.1
6061
with:
6162
compose-file: "compose_github.yaml"
@@ -77,16 +78,3 @@ jobs:
7778
tags: |
7879
${{ vars.DOCKERHUB_USERNAME }}/${{ matrix.plugin }}:${{ env.IMAGE_TAG }}
7980
${{ vars.DOCKERHUB_USERNAME }}/${{ matrix.plugin }}:latest
80-
81-
82-
83-
# - name: Run integration tests with docker-compose
84-
# run: |
85-
# docker compose -f docker-compose.test.yml up -d
86-
# # kurze Wartezeit, bis Services bereit sind
87-
# sleep 10
88-
# # Beispiel: Tests im Container ausführen
89-
# docker exec my_service_container pytest -v
90-
# docker compose -f docker-compose.test.yml down
91-
92-

‎Images/qp-target-gene/qp-target-gene.dockerfile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,4 +118,7 @@ RUN sed -i "s|^#\!.*|#\!/usr/bin/python2|" /usr/local/bin/start_target_gene
118118
RUN configure_target_gene --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f`
119119
RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qp-target-gene/" /unshared_plugins/*.conf
120120

121+
# for testing
122+
COPY test_plugin.sh /test_plugin.sh
123+
121124
CMD ["./start_qp-target-gene.sh"]

‎Images/qtp-biom/qtp-biom.dockerfile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,9 @@ RUN sed -i "s/'display.max_colwidth', -1/'display.max_colwidth', None/" /usr/loc
169169
# remove conda command from tigger.py
170170
# RUN sed -i "s|source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/conda/envs/%s;||" /trigger.py && sed -i "s|conda_env_name, ||" /trigger.py
171171

172+
# for testing
173+
COPY test_plugin.sh /test_plugin.sh
174+
172175
CMD ["./start_qtp-biom.sh"]
173176

174177
# python -c "import qiime2.plugins.feature_table"

‎Images/qtp-diversity/qtp-diversity.dockerfile‎

Lines changed: 112 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
FROM ubuntu:24.04
1+
FROM ubuntu:24.04 AS builder
22

33
ARG MINIFORGE_VERSION=24.1.2-0
4+
ARG QIIME2RELEASE=2022.11
45

56
ENV CONDA_DIR=/opt/conda
67
ENV PATH=${CONDA_DIR}/bin:${PATH}
@@ -29,44 +30,136 @@ RUN conda install tornado
2930
COPY trigger.py /trigger.py
3031

3132
# Download qiime2 yaml
32-
RUN wget -q https://data.qiime2.org/distro/core/qiime2-2022.11-py38-linux-conda.yml
33+
RUN wget -q https://data.qiime2.org/distro/core/qiime2-${QIIME2RELEASE}-py38-linux-conda.yml
34+
35+
RUN sed -n '/channels/,/dependencies/p' qiime2-${QIIME2RELEASE}-py38-linux-conda.yml > tinyq2.yml && \
36+
echo " - q2-metadata=${QIIME2RELEASE}" >> tinyq2.yml && \
37+
echo " - q2-mystery-stew=${QIIME2RELEASE}" >> tinyq2.yml && \
38+
echo " - q2-types=${QIIME2RELEASE}" >> tinyq2.yml && \
39+
echo " - q2-diversity=${QIIME2RELEASE}" >> tinyq2.yml && \
40+
echo " - q2-diversity-lib=${QIIME2RELEASE}" >> tinyq2.yml && \
41+
echo " - q2cli=${QIIME2RELEASE}" >> tinyq2.yml && \
42+
echo " - q2templates=${QIIME2RELEASE}" >> tinyq2.yml && \
43+
echo " - q2-taxa=${QIIME2RELEASE}" >> tinyq2.yml && \
44+
echo " - typeguard=2.13.3" >> tinyq2.yml && \
45+
# echo " - unifrac-binaries=1.1.1" >> tinyq2.yml && \
46+
echo " - qiime2" >> tinyq2.yml
3347

3448
# Create conda env
35-
RUN conda env create --name qiime2 -y --file qiime2-2022.11-py38-linux-conda.yml
49+
RUN conda config --set channel_priority strict && conda env create --name qiime2 -y --file tinyq2.yml
3650
# Make RUN commands use the new environment:
3751
# append --format docker to the build command, see https://github.com/containers/podman/issues/8477
3852
SHELL ["conda", "run", "-p", "/opt/conda/envs/qiime2", "/bin/bash", "-c"]
3953

4054
RUN pip install -U pip
41-
RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip
42-
RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip
55+
# RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip
56+
RUN git clone -b master https://github.com/qiita-spots/qiita_client.git
57+
RUN cd qiita_client && pip install --no-cache-dir .
58+
59+
# RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip
60+
RUN git clone -b master https://github.com/qiita-spots/qiita-files.git
61+
RUN cd /qiita-files && pip install -e . -v
62+
4363
RUN pip install https://github.com/biocore/q2-mislabeled/archive/refs/heads/main.zip
4464
RUN git clone https://github.com/qiita-spots/qtp-diversity.git
45-
WORKDIR qtp-diversity
65+
WORKDIR /qtp-diversity
66+
RUN sed -i "s|'qiita-files @ https://github.com/'||" setup.py
67+
RUN sed -i "s|'qiita-spots/qiita-files/archive/master.zip',||" setup.py
68+
RUN sed -i "s|'qiita_client @ https://github.com/qiita-spots/'||" setup.py
69+
RUN sed -i "s|'qiita_client/archive/master.zip'||" setup.py
4670
RUN pip install -e .
4771
RUN pip install --upgrade certifi
4872
RUN pip install pip-system-certs
4973

50-
# TODO: should the plugin get the server configuration?!
51-
RUN export QIITA_CONFIG_FP=/qiita/config_qiita_oidc.cfg
52-
5374
WORKDIR /
5475

76+
RUN repo=q2-metadata; mkdir -p /$repo && wget -O- https://github.com/qiime2/$repo/archive/refs/tags/${QIIME2RELEASE}.0.tar.gz | tar -xz --strip-components=1 -C /$repo
77+
RUN repo=q2-mystery-stew; mkdir -p /$repo && wget -O- https://github.com/qiime2/$repo/archive/refs/tags/${QIIME2RELEASE}.0.tar.gz | tar -xz --strip-components=1 -C /$repo
78+
RUN repo=q2-types; mkdir -p /$repo && wget -O- https://github.com/qiime2/$repo/archive/refs/tags/${QIIME2RELEASE}.0.tar.gz | tar -xz --strip-components=1 -C /$repo
79+
RUN repo=q2cli; mkdir -p /$repo && wget -O- https://github.com/qiime2/$repo/archive/refs/tags/${QIIME2RELEASE}.1.tar.gz | tar -xz --strip-components=1 -C /$repo
80+
RUN repo=q2templates; mkdir -p /$repo && wget -O- https://github.com/qiime2/$repo/archive/refs/tags/${QIIME2RELEASE}.0.tar.gz | tar -xz --strip-components=1 -C /$repo
81+
RUN repo=qiime2; mkdir -p /$repo && wget -O- https://github.com/qiime2/$repo/archive/refs/tags/${QIIME2RELEASE}.1.tar.gz | tar -xz --strip-components=1 -C /$repo
82+
RUN repo=q2-taxa; mkdir -p /$repo && wget -O- https://github.com/qiime2/$repo/archive/refs/tags/${QIIME2RELEASE}.0.tar.gz | tar -xz --strip-components=1 -C /$repo
83+
RUN repo=q2-diversity; mkdir -p /$repo && wget -O- https://github.com/qiime2/$repo/archive/refs/tags/${QIIME2RELEASE}.1.tar.gz | tar -xz --strip-components=1 -C /$repo
84+
RUN repo=q2-diversity-lib; mkdir -p /$repo && wget -O- https://github.com/qiime2/$repo/archive/refs/tags/${QIIME2RELEASE}.0.tar.gz | tar -xz --strip-components=1 -C /$repo
85+
86+
RUN repo=unifrac; mkdir -p /$repo && wget -O- https://github.com/biocore/unifrac/archive/refs/tags/1.1.1.tar.gz | tar -xz --strip-components=1 -C /$repo
87+
88+
COPY requirements.txt ./requirements.txt
89+
RUN pip wheel --no-cache-dir --wheel-dir /wheels -r requirements.txt
90+
91+
RUN sed -n '/channels/,/dependencies/p' qiime2-${QIIME2RELEASE}-py38-linux-conda.yml > deps.yml && \
92+
echo " - umap-learn" >> deps.yml && \
93+
echo " - unifrac=1.1.1" >> deps.yml && \
94+
echo " - unifrac-binaries=1.1.1" >> deps.yml && \
95+
echo " - qiime2" >> deps.yml
96+
97+
# Create conda env
98+
RUN conda config --set channel_priority strict && conda env create --name dependencies -y --file deps.yml
99+
100+
# ==========================
101+
# Stage 2: Runtime
102+
# ==========================
103+
FROM python:3.8-slim
104+
105+
# python package compile in build stage
106+
COPY --from=builder /wheels /wheels
107+
108+
RUN pip install --no-cache-dir /wheels/* \
109+
&& rm -rf rm -rf `find /usr/local/lib/python3.8/site-packages -type d -name "tests" | grep -v numpy`
110+
55111
COPY start_qtp-diversity.sh .
56112
RUN chmod 755 start_qtp-diversity.sh
57113

58114
RUN mkdir -p /unshared_plugins
59115
ENV QIITA_PLUGINS_DIR=/unshared_plugins/
60116

61-
## Export cert and config filepaths
62-
COPY qiita_server_certificates/qiita_server_certificates.pem /qiita_server_certificates/qiita_server_certificates.pem
63-
ENV REQUESTS_CA_BUNDLE=/qiita_server_certificates/qiita_server_certificates.pem
64-
ENV SSL_CERT_FILE=/qiita_server_certificates/qiita_server_certificates.pem
117+
COPY --from=builder /opt/conda/envs/dependencies/bin/* /opt/conda/bin/
118+
COPY --from=builder /opt/conda/envs/dependencies/sbin/* /opt/conda/sbin/
119+
COPY --from=builder /opt/conda/envs/dependencies/lib/* /opt/conda/lib/
120+
COPY --from=builder /opt/conda/envs/dependencies/x86_64-conda-linux-gnu/* /opt/conda/x86_64-conda-linux-gnu/
121+
ENV PATH=$PATH:/opt/conda/bin:/opt/conda/sbin
122+
# RUN for d in `echo bin lib sbin x86_64-conda-linux-gnu`; do cp -r /dep_conda/$d/* /usr/local/$d/; done
123+
124+
RUN ln -s /usr/local/lib/python3.8/site-packages/scikit_learn.libs/libgomp-a34b3233.so.1.0.0 /lib/x86_64-linux-gnu/libgomp.so.1
125+
126+
# # everything for unifrac
127+
# # COPY --from=builder /opt/conda/envs/qiime2/bin/ssu /usr/local/bin/ssu
128+
# # COPY --from=builder /opt/conda/envs/qiime2/bin/faithpd /usr/local/bin/faithpd
129+
# # COPY --from=builder /opt/conda/envs/qiime2/lib/libssu.so /usr/local/lib/libssu.so
130+
# # COPY --from=builder /opt/conda/envs/qiime2/lib/libopenblasp-r0.3.25.so /usr/local/lib/libopenblasp-r0.3.25.so
131+
# # COPY --from=builder /opt/conda/envs/qiime2/lib/libhdf5_hl_cpp.so.100.1.4 /usr/local/lib/libhdf5_hl_cpp.so.100
132+
# # COPY --from=builder /opt/conda/envs/qiime2/lib/libhdf5_hl.so.100.1.3 /usr/local/lib/libhdf5_hl.so.100
133+
# # COPY --from=builder /opt/conda/envs/qiime2/lib/libhdf5.so.103.2.0 /usr/local/lib/libhdf5.so.103
134+
# # COPY --from=builder /opt/conda/envs/qiime2/lib/libgfortran.so.5.0.0 /usr/local/lib/libgfortran.so.5
135+
# # COPY --from=builder /opt/conda/envs/qiime2/lib/libquadmath.so.0.0.0 /usr/local/lib/libquadmath.so.0
136+
# # COPY --from=builder /opt/conda/envs/qiime2/lib/libhdf5_cpp.so.103.2.0 /usr/local/lib/libhdf5_cpp.so.103
137+
# # COPY --from=builder /opt/conda/envs/qiime2/lib/libhdf5_hl_cpp.so.100.1.4 /usr/local/lib/libhdf5_hl_cpp.so.100
138+
# # COPY --from=builder /opt/conda/envs/qiime2/lib/libhdf5_hl.so.100.1.3 /usr/local/lib/libhdf5_hl.so.100
139+
# # COPY --from=builder /opt/conda/envs/qiime2/lib/libhdf5.so.103.2.0 /usr/local/lib/libhdf5.so.103
140+
# # COPY --from=builder /opt/conda/envs/qiime2/lib/libcrypto.so.1.1 /usr/local/lib/libcrypto.so.1.1
141+
# # COPY --from=builder /opt/conda/envs/qiime2/lib/libcurl.so.4.8.0 /usr/local/lib/libcurl.so.4
142+
# # COPY --from=builder /opt/conda/envs/qiime2/lib/libnghttp2.so.14.24.1 /usr/local/lib/libnghttp2.so.14
143+
# # COPY --from=builder /opt/conda/envs/qiime2/lib/libssh2.so.1.0.1 /usr/local/lib/libssh2.so.1
144+
# # COPY --from=builder /opt/conda/envs/qiime2/lib/libssl.so.1.1 /usr/local/lib/libssl.so.1.1
145+
# # RUN ln -s /usr/local/lib/libopenblasp-r0.3.25.so /usr/local/lib/libcblas.so.3
146+
# # RUN ln -s /usr/local/lib/libopenblasp-r0.3.25.so /usr/local/lib/liblapacke.so.3
147+
# # RUN for f in `echo "libssu.so libhdf5_cpp.so.103 liblapacke.so.3 libcblas.so.3 libhdf5_hl_cpp.so.100 libhdf5_hl.so.100 libhdf5.so.103 libcrypto.so.1.1 libcurl.so.4 libgfortran.so.5 libnghttp2.so.14 libssh2.so.1 libssl.so.1.1 libquadmath.so.0"`; do ln -s /usr/local/lib/$f /lib/x86_64-linux-gnu/$f; done
148+
149+
# # COPY trigger_noconda.py /trigger.py
150+
# # ENV LC_ALL=C.UTF-8
151+
# # ENV LANG=C.UTF-8
152+
153+
# # ## Export cert and config filepaths
154+
# # COPY qiita_server_certificates/qiita_server_certificates.pem /qiita_server_certificates/qiita_server_certificates.pem
155+
# # ENV REQUESTS_CA_BUNDLE=/qiita_server_certificates/qiita_server_certificates.pem
156+
# # ENV SSL_CERT_FILE=/qiita_server_certificates/qiita_server_certificates.pem
157+
158+
# # #RUN export QIITA_ROOTCA_CERT=/unshared_certificates/ci_rootca.crt
159+
# # RUN chmod u+x /usr/local/bin/configure_diversity_types /usr/local/bin/start_diversity_types
160+
# # COPY qiita_server_certificates/*_server.* /qiita_server_certificates/
161+
# # RUN configure_diversity_types --env-script "true" --ca-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f`
162+
# # RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qtp-diversity/" /unshared_plugins/*.conf
65163

66-
#RUN export QIITA_ROOTCA_CERT=/unshared_certificates/ci_rootca.crt
67-
RUN chmod u+x /qtp-diversity/scripts/configure_diversity_types /qtp-diversity/scripts/start_diversity_types
68-
COPY qiita_server_certificates/*_server.* /qiita_server_certificates/
69-
RUN /qtp-diversity/scripts/configure_diversity_types --env-script "true" --ca-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f`
70-
RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qtp-diversity/" /unshared_plugins/*.conf
164+
# # CMD ["./start_qtp-diversity.sh"]
71165

72-
CMD ["./start_qtp-diversity.sh"]

‎Images/qtp-visualization/qtp-visualization.dockerfile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,4 +113,7 @@ ENV CONDA_PREFIX=/usr/local
113113
RUN configure_visualization_types --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f`
114114
RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qtp-visualization/" /unshared_plugins/*.conf
115115

116+
# for testing
117+
COPY test_plugin.sh /test_plugin.sh
118+
116119
CMD ["./start_qtp-visualization.sh"]

0 commit comments

Comments
 (0)