Use OpenVINO from pip

.github/workflows/main.yml

@@ -0,0 +1,71 @@
+# This is a basic workflow to help you get started with Actions
+
+name: CI
+
+# Controls when the action will run. Triggers the workflow on push or pull request
+# events but only for the master branch
+on: [push, pull_request]
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+ # This workflow contains a single job called "build"
+ build:
+ # The type of runner that the job will run on
+ runs-on: ubuntu-latest
+
+ # Steps represent a sequence of tasks that will be executed as part of the job
+ steps:
+ # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+ - uses: actions/checkout@v2
+
+ # Runs a single command using the runners shell
+ - name: Build
+ run: docker build -t deepvariant . --build-arg DV_OPENVINO_BUILD=1
+
+ - name: Download testdata
+ run: |
+ mkdir -p input
+ FTPDIR=ftp://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/000/001/405/GCA_000001405.15_GRCh38/seqs_for_alignment_pipelines.ucsc_ids
+ curl ${FTPDIR}/GCA_000001405.15_GRCh38_no_alt_analysis_set.fna.gz | gunzip > input/GRCh38_no_alt_analysis_set.fasta
+ curl ${FTPDIR}/GCA_000001405.15_GRCh38_no_alt_analysis_set.fna.fai > input/GRCh38_no_alt_analysis_set.fasta.fai
+ HTTPDIR=https://storage.googleapis.com/deepvariant/case-study-testdata
+ curl ${HTTPDIR}/HG002.novaseq.pcr-free.35x.dedup.grch38_no_alt.chr20.bam > input/HG002.novaseq.pcr-free.35x.dedup.grch38_no_alt.chr20.bam
+ curl ${HTTPDIR}/HG002.novaseq.pcr-free.35x.dedup.grch38_no_alt.chr20.bam.bai > input/HG002.novaseq.pcr-free.35x.dedup.grch38_no_alt.chr20.bam.bai
+ - name: Test
+ run: |
+ for model in WGS WES PACBIO HYBRID_PACBIO_ILLUMINA; do
+ INPUT_DIR="${PWD}/input"
+ REF_OUTPUT_DIR="${PWD}/reference"
+ docker run \
+ -v "${INPUT_DIR}":"/input" \
+ -v "${REF_OUTPUT_DIR}:/output" \
+ deepvariant \
+ /opt/deepvariant/bin/run_deepvariant \
+ --model_type=${model} \
+ --ref=/input/GRCh38_no_alt_analysis_set.fasta \
+ --reads=/input/HG002.novaseq.pcr-free.35x.dedup.grch38_no_alt.chr20.bam \
+ --regions "chr20:10,000,000-10,010,000" \
+ --output_vcf=/output/HG002.output.vcf.gz \
+ --output_gvcf=/output/HG002.output.g.vcf.gz \
+ --call_variants_extra_args="use_openvino=False" \
+ --num_shards=$(nproc)
+ OUTPUT_DIR="${PWD}/quickstart-output-ovino"
+ docker run \
+ -v "${INPUT_DIR}":"/input" \
+ -v "${OUTPUT_DIR}:/output" \
+ deepvariant \
+ /opt/deepvariant/bin/run_deepvariant \
+ --model_type=${model} \
+ --ref=/input/GRCh38_no_alt_analysis_set.fasta \
+ --reads=/input/HG002.novaseq.pcr-free.35x.dedup.grch38_no_alt.chr20.bam \
+ --regions "chr20:10,000,000-10,010,000" \
+ --output_vcf=/output/HG002.output.vcf.gz \
+ --output_gvcf=/output/HG002.output.g.vcf.gz \
+ --call_variants_extra_args="use_openvino=True" \
+ --num_shards=$(nproc)
+ files=$(ls $REF_OUTPUT_DIR); \
+ for f in $files; do \
+ cmp $REF_OUTPUT_DIR/$f $OUTPUT_DIR/$f; \
+ done
+ sudo rm -r $REF_OUTPUT_DIR $OUTPUT_DIR
+ done

Dockerfile

@@ -152,10 +152,6 @@ RUN apt-get -y update && \
 
 ENV PATH="/opt/deepvariant/bin:${PATH}"
 
-# Setup OpenVINO environment.
-ENV LD_LIBRARY_PATH=/opt/intel/openvino/inference_engine/lib/intel64/:/opt/intel/openvino/inference_engine/external/tbb/lib/:/opt/intel/openvino/deployment_tools/ngraph/lib/:$LD_LIBRARY_PATH \
- PYTHONPATH=/opt/intel/openvino/python/python3.6/:/opt/intel/openvino/deployment_tools/model_optimizer/:$PYTHONPATH
-
 WORKDIR /opt/deepvariant
 
 CMD ["/opt/deepvariant/bin/run_deepvariant", "--help"]

deepvariant/openvino_estimator.py

@@ -28,6 +28,7 @@
 # POSSIBILITY OF SUCH DAMAGE.
 """An estimator that uses OpenVINO."""
 import os
+import sys
 import subprocess
 from absl import logging
 import tensorflow as tf
@@ -88,7 +89,7 @@ def __init__(self, checkpoint_path, input_fn, model):
  tensor_shape = input_fn.tensor_shape
  freeze_graph(model, checkpoint_path, tensor_shape)
  subprocess.run([
- mo_tf.__file__, '--input_model=model.pb', '--scale=128',
+ sys.executable, mo_tf.__file__, '--input_model=model.pb', '--scale=128',
  '--mean_values', '[{}]'.format(','.join(['128'] * tensor_shape[-1]))
  ],
  check=True)

run-prereq.sh

@@ -106,7 +106,7 @@ echo "$(pip3 --version)"
 note_build_stage "Install python3 packages"
 
 pip3 install "${PIP_ARGS[@]}" contextlib2
-pip3 install "${PIP_ARGS[@]}" enum34
+pip3 install "${PIP_ARGS[@]}" 'enum34==1.1.8'
 pip3 install "${PIP_ARGS[@]}" 'sortedcontainers==2.1.0'
 pip3 install "${PIP_ARGS[@]}" 'intervaltree==3.0.2'
 pip3 install "${PIP_ARGS[@]}" 'mock>=2.0.0'
@@ -231,14 +231,8 @@ fi
 # OpenVINO
 ################################################################################
 if [[ "${DV_OPENVINO_BUILD}" = "1" ]]; then
- sudo -H apt-get install "${APT_ARGS[@]}" apt-transport-https > /dev/null
- curl -o GPG-PUB-KEY-INTEL-OPENVINO-2020 https://apt.repos.intel.com/openvino/2020/GPG-PUB-KEY-INTEL-OPENVINO-2020
- sudo apt-key add GPG-PUB-KEY-INTEL-OPENVINO-2020
- sudo echo "deb https://apt.repos.intel.com/openvino/2020 all main" | sudo tee /etc/apt/sources.list.d/intel-openvino-2020.list
- sudo -H apt-get update "${APT_ARGS[@]}" > /dev/null
- sudo apt-get install -y --no-install-recommends intel-openvino-dev-ubuntu16-2020.4.287 > /dev/null
- rm -f GPG-PUB-KEY-INTEL-OPENVINO-2020
- pip3 install "${PIP_ARGS[@]}" networkx defusedxml test-generator==0.1.1
+ pip3 install "${PIP_ARGS[@]}" git+https://github.com/openvinotoolkit/openvino.git@releases/2021/3#subdirectory=model-optimizer
+ pip3 install "${PIP_ARGS[@]}" openvino==2021.3
 fi
 
 ################################################################################