make qp-target-gene work

Author: sjanssen2

.github/workflows/buildContainer.yaml

@@ -40,7 +40,7 @@ jobs:
       - name: Make tinqiita targets
         # second copy of "qiita_server_certificates" is necessary to match path for docker build, first copy for mounting into container
         run: |
-          make ./references/qiita_server_certificates ./references/qp-deblur/reference-gg-raxml-bl.tre ./environments/qiita_db.env ./environments/qiita.env config
+          make ./references/qiita_server_certificates ./references/qp-deblur/reference-gg-raxml-bl.tre ./references/qp-target-gene ./environments/qiita_db.env ./environments/qiita.env config
           cp -r ./references/qiita_server_certificates ./qiita_server_certificates
 
       - name: Build Image (but do not push yet)

Images/qiita/qiita.dockerfile

@@ -56,6 +56,9 @@ RUN sed -i "s|'source /home/runner/.profile; conda activate qiita'|'source /opt/
 RUN sed -i "s|'source ~/virtualenv/python2.7/bin/activate; export PATH=\$HOME/miniconda3/bin/:\$PATH; . activate qtp-biom'|'true'|" /qiita/qiita_db/support_files/populate_test_db.sql
 RUN sed -i "s|'source activate qiita'|'true'|" /qiita/qiita_db/support_files/populate_test_db.sql
 
+# there seems to be a conflict with parameter names für qp-target-gene. See: https://github.com/qiita-spots/qp-target-gene/issues/24
+RUN sed -i "s|'1.9.1',|'1.9.hide',|" /qiita/qiita_db/support_files/populate_test_db.sql
+
 # We need to install necessary dependencies
 # as well as some extra dependencies for psycopg2 to work
 RUN git clone https://github.com/psycopg/psycopg2.git

Images/qp-target-gene/qp-target-gene.dockerfile

@@ -45,7 +45,7 @@ RUN python2.7 get-pip2.7.py --force-reinstall
 RUN pip install -U pip
 #RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip
 RUN git clone -b master https://github.com/qiita-spots/qiita_client.git
-RUN cd qiita_client && pip install --no-cache-dir .
+RUN cd /qiita_client && pip install --no-cache-dir .
 
 RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip
 RUN git clone https://github.com/qiita-spots/qp-target-gene.git
@@ -57,9 +57,19 @@ RUN pip install pip-system-certs
 
 WORKDIR /
 
+# qiime 1.9.1 comes with https://pypi.org/project/qiime-default-reference/ as dependency, which is ~184MB
+# we "hide" it here, as necessary files will be downloaded from ftp.microbio.me/greengenes_release while setting up qiita anyway
+RUN pip download --dest /qiime_default_reference qiime_default_reference \
+    && cd /qiime_default_reference \
+	&& tar xzvf *.tar.gz \
+	&& cd qiime-default-reference-0.1.3 \
+	&& for fzip in `find . -type f -name "97*"`; do fplain=`echo $fzip | sed "s|.gz$||g"`; echo "content erased to generate small wheel file, as reference shall be mounted to target container later on." > $fplain; gzip -f $fplain; done
+
 COPY requirements.txt ./requirements.txt
 RUN pip wheel --no-cache-dir --wheel-dir /wheels -r requirements.txt
 
+# download sortmerna / index_db sources for version 2.0 and re-compile statically as different glibc and libstdc++ couse issues
+RUN wget https://github.com/sortmerna/sortmerna/archive/refs/tags/2.0.tar.gz && tar xzvf 2.0.tar.gz && cd /sortmerna-2.0 && ./configure LDFLAGS=" -static " && make -j
 
 # ==========================
 # Stage 2: Runtime
@@ -73,23 +83,18 @@ RUN mkdir -p /usr/share/man/man1 && \
     apt-get install -y --no-install-recommends python2 python3 curl python-tk && \
     rm -rf /var/lib/apt/lists/*
 
-# # RUN apk add --no-cache python2 python3  curl
-
-# pip2 installieren
+# install pip2
 COPY --from=builder /get-pip2.7.py /get-pip3.7.py /
 RUN python2 get-pip2.7.py \
  	&& rm get-pip2.7.py
 
-# pip3 installieren
+# install pip3
 RUN python3 get-pip3.7.py \
  	&& rm get-pip3.7.py
 
 # python package compile in build stage
 COPY --from=builder /wheels /wheels
 
-# dependent binaries + necessary libraries: sortmerna
-COPY --from=builder /opt/conda/envs/qp-target-gene/bin/indexdb_rna /opt/conda/envs/qp-target-gene/bin/sortmerna /usr/local/bin/
-
 RUN pip2 install --no-cache-dir /wheels/* \
 	&& rm -rf rm -rf `find /usr/local/lib/python2.7/site-packages -type d -name "tests" | grep -v numpy`
 COPY --from=builder /opt/conda/envs/qp-target-gene/lib/libpython2.7.so.1.0 /usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0
@@ -106,6 +111,10 @@ ENV QIITA_PLUGINS_DIR=/unshared_plugins/
 RUN pip3 install tornado
 COPY trigger_noconda.py /trigger.py
 
+# copy sortmerna binaries
+COPY --from=builder /sortmerna-2.0/sortmerna /usr/local/bin/sortmerna
+COPY --from=builder /sortmerna-2.0/indexdb_rna /usr/local/bin/indexdb_rna
+
 ##  Export cert and config filepaths
 COPY qiita_server_certificates/qiita_server_certificates.pem /qiita_server_certificates/qiita_server_certificates.pem
 ENV REQUESTS_CA_BUNDLE=/qiita_server_certificates/qiita_server_certificates.pem

Images/qp-target-gene/requirements.txt

@@ -1,3 +1,4 @@
+-e /qiime_default_reference/qiime-default-reference-0.1.3
 pip-system-certs
 
 https://github.com/qiita-spots/qiita-files/archive/master.zip

Makefile

@@ -47,9 +47,20 @@ plugin: Images/trigger.py Images/trigger_noconda.py $(DIR_REFERENCES)/qiita_serv
 	$(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1`
 	touch .built_image_`basename $< | cut -d "." -f 1`
 
-.built_image_qp-target-gene: Images/qp-target-gene/qp-target-gene.dockerfile Images/qp-target-gene/start_qp-target-gene.sh Images/qp-target-gene/requirements.txt
+# download GG13.8 reference sets from ftp://ftp.microbio.me/greengenes_release/gg_13_8_otus, instead of storing these large files within the qp-target-gene image 149 MB
+$(DIR_REFERENCES)/qp-target-gene:
+	mkdir -p $(DIR_REFERENCES)/qp-target-gene
+	echo '56ef15dccf2e931ec173f4f977ed649b  97_otu_taxonomy.txt' > $(DIR_REFERENCES)/qp-target-gene/exp.md5
+	echo '50b2269712b3738afb41892bed936c29  97_otus.fasta' >> $(DIR_REFERENCES)/qp-target-gene/exp.md5
+	echo 'b7e76593bce82913af1cfb06edf15732  97_otus.tree' >> $(DIR_REFERENCES)/qp-target-gene/exp.md5
+	wget 'ftp://ftp.microbio.me/greengenes_release/gg_13_8_otus/trees/97_otus.tree' -O $(DIR_REFERENCES)/qp-target-gene/97_otus.tree
+	wget 'ftp://ftp.microbio.me/greengenes_release/gg_13_8_otus/taxonomy/97_otu_taxonomy.txt' -O $(DIR_REFERENCES)/qp-target-gene/97_otu_taxonomy.txt
+	wget 'ftp://ftp.microbio.me/greengenes_release/gg_13_8_otus/rep_set/97_otus.fasta' -O $(DIR_REFERENCES)/qp-target-gene/97_otus.fasta
+	cd $(DIR_REFERENCES)/qp-target-gene/ && md5sum -c exp.md5 || rm -rf $(DIR_REFERENCES)/qp-target-gene/
+
+.built_image_qp-target-gene: Images/qp-target-gene/qp-target-gene.dockerfile Images/qp-target-gene/start_qp-target-gene.sh $(DIR_REFERENCES)/qp-target-gene Images/qp-target-gene/requirements.txt
 	tmpdir=$(TMPDIR) $(MAKE) plugin
-	cp $^ $(TMPDIR)
+	cp -r $^ $(TMPDIR)
 	$(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1`
 	touch .built_image_`basename $< | cut -d "." -f 1`
 

compose.yaml

@@ -235,6 +235,7 @@ services:
       - qiita-data:/qiita_data
       - ./references/qiita_server_certificates:/qiita_server_certificates
       - test_tmp_dir:/tmp
+      - ./references/qp-target-gene:/databases/gg/13_8/rep_set
     environment:
       - QIITA_CLIENT_DEBUG_LEVEL=DEBUG
     networks:
@@ -355,6 +356,7 @@ services:
         condition: service_started
     environment:
       - QIITA_PLUGINS="qtp-biom:qtp-sequencing:qp-target-gene:qtp-visualization:qtp-diversity:qp-deblur:qp-qiime2:qtp-job-output-folder:"
+      #- QIITA_PLUGINS="qp-target-gene:"
     command: ['/start_plugin_collector.sh']
 
 networks:

compose_github.yaml

@@ -218,6 +218,7 @@ services:
       - qiita-data:/qiita_data
       - ./references/qiita_server_certificates:/qiita_server_certificates
       - test_tmp_dir:/tmp
+      - ./references/qp-target-gene:/databases/gg/13_8/rep_set
     environment:
       - QIITA_CLIENT_DEBUG_LEVEL=DEBUG
     networks: