MIPTools.def

Bootstrap: docker
From: amd64/ubuntu:20.04

##################################################################
##                        Labels Section                        ##
##################################################################
%labels
    Author Bailey Lab
    Version v0.4.0.9000

##################################################################
##                         Post Section                         ##
##################################################################
%post
    # set number of cpus to use in build
    CPU_COUNT=20

    # set build environment
    export DEBIAN_FRONTEND=noninteractive \
        SHELL=/bin/bash \
        LANG=en_US.UTF-8 \
        LANGUAGE=en_US.UTF-8 \
        LC_ALL=en_US.UTF-8

    # install system packages
    apt-get update \
    && apt-get -yq dist-upgrade \
    && apt-get install -yq --no-install-recommends \
    wget \
    bzip2 \
    ca-certificates \
    sudo \
    locales \
    fonts-liberation \
    fonts-dejavu \
    git \
    build-essential \
    gcc-10 \
    g++-10 \
    openssh-client \
    nano \
    libtbb-dev \
    libz-dev \
    libxrender1 \
    cmake \
    automake \
    autoconf \
    rsync \
    pigz \
    perl-tk \
    less \
    software-properties-common \
    libxext6 \
    libxrender1 \
    ghostscript \
    openjdk-11-jdk \
    liblzma-dev \
    libbz2-dev \
    libssl-dev \
    libcurl4-gnutls-dev \
    alien \
    unzip \
    tree \
    pandoc

    # set environment locale
    echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen
    echo "LANG=en_US.UTF-8" > /etc/locale.conf
    echo "LC_ALL=en_US.UTF-8" >> /etc/environment
    echo "LANGUAGE=en_US.UTF-8" >> /etc/environment
    locale-gen en_US.UTF-8
    update-locale LANG=en_US.UTF-8

    # install bcl2fastq, if the file is there
    cd /opt/programs
    unzip bcl2fastq2*.zip || true
    alien bcl2fastq2*.rpm || true
    dpkg -i bcl2fastq2*.deb || true

    # install conda and python via Miniconda3
    PYTHON_VERSION=3.8
    PYTHON_VERSION=$(echo ${PYTHON_VERSION} | sed 's/[^0-9]//g' | head -c2)
    MINICONDA_VERSION=4.8.3
    MINICONDA_MD5=d63adf39f2c220950a063e0529d4ff74
    CONDA_DIR=/opt/conda
    export PATH=${CONDA_DIR}/bin:${PATH}
    cd /tmp && \
        wget --quiet "https://repo.anaconda.com/miniconda/Miniconda3-py${PYTHON_VERSION}_${MINICONDA_VERSION}-Linux-x86_64.sh" && \
        echo "${MINICONDA_MD5} *Miniconda3-py${PYTHON_VERSION}_${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \
        /bin/bash Miniconda3-py${PYTHON_VERSION}_${MINICONDA_VERSION}-Linux-x86_64.sh -bfp ${CONDA_DIR} && \
        rm Miniconda3-py${PYTHON_VERSION}_${MINICONDA_VERSION}-Linux-x86_64.sh && \
        ${CONDA_DIR}/bin/conda config --system --set show_channel_urls true && \
        conda clean --all --force-pkgs-dirs --yes

    # Install mamba
    # Note that the mamba installation will update conda to the latest version
    conda install mamba --channel conda-forge --quiet

    # Install mamba packages using an environment file.
    # If the versioned file exists, use it as a template to ensure version
    # numbers are fixed. Otherwise, install packages with the latest versions.
    # Note that instead of creating a new environment, we update the base
    # environment, which is activated by default.
    if [ -f "/opt/environment_versioned.yml" ]; then
        mamba env update --prefix ${CONDA_DIR} --file /opt/environment_versioned.yml --quiet
    else
        # Update environment and save information to a file
        mamba env update --prefix ${CONDA_DIR} --file /opt/environment.yml --quiet
        mamba env export --prefix ${CONDA_DIR} > /opt/environment_versioned.yml
    fi

    # Clean mamba installs
    mamba clean --all --yes

    # install vt variant tool set
    cd /opt/programs
    git clone --branch 0.577 https://github.com/atks/vt.git
    cd vt
    make -j $CPU_COUNT
    scp vt /opt/bin

    # install magrittr
    Rscript -e 'devtools::install_version(
        package = "magrittr",
        version = "2.0.3",
        repos = "https://cloud.r-project.org"
    )'

    # install RealMcCoil
    Rscript -e 'devtools::install_github("OJWatson/McCOILR", ref = "v1.3.1")'

    # install rehh
    Rscript -e 'devtools::install_version(
        package = "rehh", 
        version = "3.2.2",
        repos = "https://cloud.r-project.org"
    )'

    # install MIPWrangler
    cd /opt/programs
    git clone --branch v1.2.0 https://github.com/bailey-lab/MIPWrangler.git
    cd MIPWrangler
    ./install.sh $CPU_COUNT

    # install elucidator
    cd /opt/programs
    git clone --branch develop https://github.com/nickjhathaway/elucidator.git
    cd elucidator
    ./install.sh $CPU_COUNT

    # install parasight
    cd /opt/programs
    git clone --branch v7.6 https://github.com/bailey-lab/parasight.git
    scp parasight/parasight.pl /opt/bin

    # install basespace cli
    BS_VERSION=1.5.1
    BS_PATH="https://launch.basespace.illumina.com/CLI/${BS_VERSION}/amd64-linux/bs"
    wget $BS_PATH -O /opt/bin/bs

    # Remove programs to reduce image size. These programs have been installed
    # and the binaries moved to /opt/bin
    cd /opt/programs
    rm -r parasight vt

    # add executable flag to executables
    chmod -R +xr /usr/bin
    chmod -R +xr /opt/bin

    # create work and resources directories in /opt
    mkdir /opt/work \
        /opt/project_resources \
        /opt/species_resources \
        /opt/data \
        /opt/analysis \
        /opt/host_species \
        /opt/extras

#################################################################
##                        Files Section                        ##
#################################################################
%files
    environment* /opt
    programs /opt
    bin /opt
    src /opt
    base_resources/ /opt/resources

#################################################################
##                     Environment Section                     ##
#################################################################
%environment
    path=/opt/bin:/opt/conda/bin:
    path=$path/opt/programs/MIPWrangler/bin:
    path=$path/opt/programs/elucidator/bin:
    path=$path$PATH
    export PATH=$path
    export XDG_RUNTIME_DIR=""
    export DEBIAN_FRONTEND=noninteractive
    export LANG=en_US.UTF-8
    export LANGUAGE="en_US.UTF-8"
    export LC_ALL="en_US.UTF-8"

#################################################################
##                         Jupyter App                         ##
#################################################################
%apprun jupyter
    # Exit if something fails or if have unset object
    set -eu

    # Port forwarding setup
    nb_port=$(shuf -i 8000-9999 -n 1)
    server_ip=$(hostname -i)
    server_user=$(whoami)@$(hostname -f)
    nb_dir=/opt

    help() {
        echo "Open an interactive Jupyter Notebook. The notebook can be used"
        echo "for post-wrangler mapping and variant calling."
        echo ""
        echo "Usage:"
        echo "  singularity run [options] --app jupyter <container>"\
        "[app_options]"
        echo ""
        echo "Options:"
        echo "  See 'singularity run'."
        echo ""
        echo "App Options:"
        echo "  -d    The notebook directory."
        echo "  -h    Print the help page."
        echo "  -p    The port to be used to load the Jupyter Notebook."
        echo ""
        echo "Examples:"
        echo "  # Set paths"
        echo "  $ resource_dir=/bin/MIPTools/base_resources"
        echo "  $ project_resources=/work/usr/DR1_project_resources"
        echo "  $ species_resources=/work/usr/pf_species_resources"
        echo "  $ wrangler_dir=/work/usr/wrangler"
        echo "  $ variant_dir=/work/usr/variant"
        echo ""
        echo "  # Run app"
        echo "  $ singularity run \\"
        echo "    -B \${resource_dir}:/opt/resources \\"
        echo "    -B \${project_resources}:/opt/project_resources \\"
        echo "    -B \${species_resources}:/opt/species_resources \\"
        echo "    -B \${wrangler_dir}:/opt/data \\"
        echo "    -B \${variant_dir}:/opt/analysis \\"
        echo "    --app jupyter <container>"
    }

    # Parse options
    while getopts "d:hp:" opt; do
        case ${opt} in
            d) nb_dir=${OPTARG} ;;
            h) help
               exit 1 ;;
            p) nb_port=${OPTARG} ;;
            *) help
               exit 1 ;;
        esac
    done

    # Save templates
    rsync /opt/resources/*.ipynb /opt/analysis --ignore-existing \
        --ignore-missing-args

    # Inform the user how to access the notebook
    echo "\nIf you are running this command from a remote server, you will need"
    echo "to forward the port to your local machine. To do so, run:\n"
    echo "ssh -fNL localhost:${nb_port}:${server_ip}:${nb_port}"\
    "${server_user}\n"

    # Setup juptyr notebook settings
    jupyter nbextension enable plotlywidget/extension
    jupyter nbextension enable toc2/main
    jupyter nbextension enable codefolding/main
    jupyter nbextension enable highlighter/highlighter
    jupyter nbextension enable keyboard_shortcut_editor/main
    jupyter nbextension enable spellchecker/main

    # Run notebook
    jupyter notebook --notebook-dir=${nb_dir} --ip=${server_ip} \
            --port=${nb_port} --no-browser

##################################################################
##                         Wrangler App                         ##
##################################################################
%apprun wrangler
    # Exit if something fails or if have unset object
    set -eu

    help() {
        echo "Run MIPWrangler on demultiplexed data."
        echo ""
        echo "Usage:"
        echo "  singularity run [options] --app wrangler <container>"\
        "[app_options]"
        echo ""
        echo "Options:"
        echo "  See 'singularity run'."
        echo ""
        echo "App Options:"
        echo "  -c    Number of available processors to use. Default: 1."
        echo "  -e    Required. A unique ID given to each sequencing run by"
        echo "        the user."
        echo "  -f    The population fraction cutoff used by MIPWrangler."
        echo "        Default: 0.005."
        echo "  -h    Print the help page."
        echo "  -k    Keep intermediate files generated by MIPWrangler."
        echo "  -l    Required. File providing a list of samples with "
        echo "        associated information."
        echo "  -m    Minimum capture length for stitching excluding probe"
        echo "        arms."
        echo "  -n    Starting number for MIP server. Default: 1."
        echo "  -o    Absolute path to MIPWrangler run script. "
        echo "        Default: '/opt/bin/runMIPWranglerCurrent.sh'."
        echo "  -p    Required. Probe sets to be processed."
        echo "  -s    Required. Sample sets to be processed."
        echo "  -t    The threshold at which UMIs will be downsampled."
        echo "        Defualt: 2000"
        echo "  -w    Whether to apply a weight when randomly sampling UMIs."
        echo "        UMIs are weighted by their read counts."
        echo "        Default: false"
        echo "  -x    Required. Additional arguments to pass to MIPWrangler"
        echo "        mipSetupAndExtractByArm. This command extracts sequences"
        echo "        and stitches paired end reads to single sequences."
        echo ""
        echo "Examples:"
        echo "  # Define variables"
        echo "  $ probe_sets='DR1,VAR4'"
        echo "  $ sample_sets='JJJ'"
        echo "  $ stitch_options='--stitchGapExtend=1,--overWriteDirs'"
        echo ""
        echo "  $ singularity run "
        echo "    -B project_resources:/opt/project_resources \\"
        echo "    -B fastq_dir:/opt/data \\"
        echo "    -B wrangler_dir:/opt/analysis \\"
        echo "    --app wrangler <container> \\"
        echo "    -e <experiment_id> -l <sample_list.file> -p \${probe_sets} \\"
        echo "    -s \${sample_sets} -x \${stitch_options}"
        echo ""
        echo "  # Run app"
        echo "  $ singularity run "
        echo "    -B project_resources:/opt/project_resources \\"
        echo "    -B fastq_dir:/opt/data \\"
        echo "    -B wrangler_dir:/opt/analysis \\"
        echo "    --app wrangler <container> \\"
        echo "    -c <cpu_count> -e <experiment_id> -l <sample_list.file> \\"
        echo "    -m <min_capture_length> -p \${probe_sets} -s \\"
        echo "    \${sample_sets} -x \${stitch_options} -k"
    }

    # Set defaults
    cluster_script="runMIPWranglerCurrent.sh"
    server_number=1
    cpu_count=1
    min_capture_length="none"
    stitch_options="none"
    keep_files=""
    population_fraction_cutoff=0.005
    downsample_threshold=2000
    weighted=""

    # Parse options
    while getopts "c:e:f:hkl:m:n:o:p:s:t:wx:" opt; do
        case ${opt} in
            c) cpu_count=${OPTARG} ;;
            e) experiment_id=${OPTARG} ;;
            f) population_fraction_cutoff=${OPTARG} ;;
            h) help
               exit 1 ;;
            k) keep_files=-k ;;
            l) sample_list=${OPTARG} ;;
            m) min_capture_length=${OPTARG} ;;
            n) server_number=${OPTARG} ;;
            o) cluster_script=${OPTARG} ;;
            p) probe_sets=${OPTARG} ;;
            s) sample_sets=${OPTARG} ;;
            t) downsample_threshold=${OPTARG} ;;
            w) weighted=-w ;;
            x) stitch_options=${OPTARG} ;;
            *) help
               exit 1 ;;
        esac
    done

    # Remove whitespace from arguments
    probe_sets=$(echo ${probe_sets} | sed "s/[[:space:]]//g")
    sample_sets=$(echo ${sample_sets} | sed "s/[[:space:]]//g")
    stitch_options=$(echo ${stitch_options} | sed "s/[[:space:]]//g")
    
    # Ensure that stitch options begin with a comma if not the default value
    # This is done as the arguments are fed in with leading dashes and the
    # python script will crash if dashes are fed in. By adding a leading comma,
    # the script will run.
    stitch_first_char=$(echo ${stitch_options} | head -c1)
    if [ ${stitch_first_char} != "," ] && [ ${stitch_options} != "none" ]; then
        stitch_options=",${stitch_options}"
    fi

    # Create wrangler bash scripts using python
    python /opt/src/generate_wrangler_scripts.py \
    -c ${cpu_count} -e ${experiment_id} ${keep_files} \
    -l /opt/analysis/${sample_list} -m ${min_capture_length} \
    -n ${server_number} -p ${probe_sets} -s ${sample_sets} \
    -o ${cluster_script} -x ${stitch_options} -f ${population_fraction_cutoff} \
    -t ${downsample_threshold} ${weighted}

    # Run wrangler scripts.
    # The dot space is used to let the sourced script modify the current
    # environment. If this is not needed, can just write the path.
    . /opt/analysis/wrangle.sh

##################################################################
##                    Basespace Download App                    ##
##################################################################
%apprun download
    # Exit if something fails
    set -eu

    # Set default values for paths
    output_path="/opt/analysis"
    config_path="/opt/resources/basespace.cfg"

    help() {
        echo "Download data from the Illumina BaseSpace Sequence Hub."
        echo ""
        echo "Usage:"
        echo "  singularity run [options] --app download <container>"\
        "[app_options]"
        echo ""
        echo "Options:"
        echo "  See 'singularity run'."
        echo ""
        echo "App Options:"
        echo "  -i    Required. The run ID of the data to download."
        echo "  -o    The path to the output directory."
        echo "        Default: '/opt/analysis'."
        echo "  -c    The path to the authentication credentials file."
        echo "        This file is created by 'bs auth'. For additional"
        echo "        information see the help page for that command."
        echo "        Default: '/opt/resources/basespace.cfg'."
        echo "  -h    Print the help page."
        echo ""
        echo "Examples:"
        echo "  # Set paths"
        echo "  $ resource_dir=/bin/MIPTools/base_resources"
        echo "  $ run_dir=/work/usr/example"
        echo ""
        echo "  # Run app"
        echo "  $ singularity run \\"
        echo "    -B \${resource_dir}:/opt/resources \\"
        echo "    -B \${run_dir}:/opt/analysis \\"
        echo "    --app download <container> -i <run_id>"
    }
    
    # Parse options
    while getopts "i:o:c:h" opt; do
      case "${opt}" in
        c) config_path=${OPTARG} ;;
        h) help
           exit 1 ;;
        i) run_id=${OPTARG} ;;
        o) output_path=${OPTARG} ;;
        *) help
           exit 1 ;;
      esac
    done
    
    # Ensure run_id is specified
    if [ -z ${run_id} ]; then
      echo "Argument -i must be provided"
      help >&2
      exit 1
    fi

    # Read data from config file
    # Remove whitespace from each line and export each line as a variable
    export BASESPACE_API_SERVER=$(sed "1q;d" ${config_path} | sed "s/.*=.//g")
    export BASESPACE_ACCESS_TOKEN=$(sed "2q;d" ${config_path} | sed "s/.*=.//g")

    # Download data
    bs download run --summary -i ${run_id} -o ${output_path}/${run_id}

#################################################################
##                   Superseded Download App                   ##
#################################################################
%apprun download_superseded
    # Exit if something fails or if have unset object
    set -eu

    help() {
        echo "Download data from the Illumina BaseSpace Sequence Hub."
        echo ""
        echo "Superseded Note:"
        echo "  Please note that this app has been superseded by the download" 
        echo "  app, which uses the basespace command line interface for" 
        echo "  downloading data."
        echo ""
        echo "Usage:"
        echo "  singularity run [options] --app download_superseded \\"
        echo "    <container> [app_options]"
        echo ""
        echo "Options:"
        echo "  See 'singularity run'."
        echo ""
        echo "App Options:"
        echo "  -h    Print the help page."
        echo "  -r    Required. The run ID of the data to download."
        echo ""
        echo "Additional Details:"
        echo "  An 'access_token.txt' file with a valid access token is"
        echo "  required. It must be present in the 'base_resources' directory."
        echo "  A data directory where the data will be downloaded to must be"
        echo "  mounted to '/opt/analysis'."
        echo ""
        echo "Examples:"
        echo "  # Set paths"
        echo "  $ resource_dir=/bin/MIPTools/base_resources"
        echo "  $ output_dir=/work/usr/downloaded"
        echo ""
        echo "  # Run app"
        echo "  $ singularity run \\"
        echo "    -B \${resource_dir}:/opt/resources"\
        "-B \${output_dir}:/opt/analysis \\"
        echo "    --app download_superseded <container> -r <run_id>"
    }

    while getopts "hr:" opt; do
        case ${opt} in
            h) help
               exit 1 ;;
            r) run_id=${OPTARG} ;;
            *) help
               exit 1 ;;
        esac
    done

    # Print to CLI
    echo "Downloading NextSeq run ${run_id} from BaseSpace."
    echo "Depending on the data size, this can take very long (up to 10 h)."
    echo "It is recommended to run this app in a screen (GNU screen)."
    echo "A message indicating the end of download will be printed when done."
    echo "Check nohup.out file in your output directory for the download log."

    # cd and run app
    # Use nohup to make command keep running even if get hangup signal
    cd /opt/analysis
    nohup python /opt/bin/BaseSpaceRunDownloader_v2.py \
     -r ${run_id} -a "$(cat /opt/resources/access_token.txt)"

    # Print to CLI
    echo "Download finished."

#################################################################
##                          Demux App                          ##
#################################################################
%apprun demux
    # Exit if something fails or if have unset object
    set -eu

    help() {
        echo "Demultiplex data. Generates per-sample fastq files from the raw"
        echo "sequence data consisting of bcl files."
        echo ""
        echo "Usage:"
        echo "  singularity run [options] --app demux <container> [app_options]"
        echo ""
        echo "Options:"
        echo "  See 'singularity run'."
        echo ""
        echo "App Options:"
        echo "  -h    Print the help page."
        echo "  -s    Required. Sample sheet for demultiplexing. "
        echo "        This file must be present in the directory mounted to "
        echo "        '/opt/analysis'."
        echo ""
        echo "Examples:"
        echo "  # Set paths"
        echo "  $ resource_dir=/bin/MIPTools/base_resources"
        echo "  $ bcl_dir=/work/usr/downloaded"
        echo "  $ fastq_root_dir=/work/usr/"
        echo ""
        echo "  # Run app"
        echo "  $ singularity run \\"
        echo "    -B \${resource_dir}:/opt/resources \\"
        echo "    -B \${bcl_dir}:/opt/data \\"
        echo "    -B \${fastq_root_dir}:/opt/analysis \\"
        echo "    --app demux <container> -s SampleSheet.csv"
    }

    while getopts "hs:" opt; do
        case ${opt} in
            h) help
               exit 1 ;;
            s) sample_list=${OPTARG} ;;
            *) help
               exit 1 ;;
        esac
    done

    # Define variables
    sample_sheet="/opt/analysis/${sample_list}"
    # cd to where bcl files are
    cd /opt/data

    # Create a fastq directory for saving fastqs
    mkdir -p /opt/analysis/fastq

    # Increase limit of open number of files.
    ulimit -Sn $(ulimit -Hn)

    # Run bcl2fastq
    # Use nohup to make command keep running even if get hangup signal
    nohup bcl2fastq -o /opt/analysis/fastq \
        --sample-sheet ${sample_sheet} \
        --no-lane-splitting

##################################################################
##                         Demux QC App                         ##
##################################################################
%apprun demux_qc
    # Exit if something fails or if have unset object
    set -eu

    help() {
        echo "Run quality control on demultiplexed data."
        echo ""
        echo "Usage:"
        echo "  singularity run [options] --app demux_qc <container>"\
        "[app_options]"
        echo ""
        echo "Options:"
        echo "  See 'singularity run'."
        echo ""
        echo "App Options:"
        echo "  -h    Print the help page."
        echo "  -p    Required. The sequencing platform used. Either 'miseq'"
        echo "        or 'nextseq'."
        echo ""
        echo "Examples:"
        echo "  # Set paths"
        echo "  $ resource_dir=/bin/MIPTools/base_resources"
        echo "  $ fastq_dir=/work/usr/example"
        echo ""
        echo "  # Run app"
        echo "  $ singularity run \\"
        echo "    -B \${resource_dir}:/opt/resources"\
        "-B \${fastq_dir}:/opt/analysis \\"
        echo "    --app demux_qc <container> -p 'nextseq'"
    }

    # Argument handling
    while getopts "hp:" opt; do
        case ${opt} in
            h) help
               exit 1 ;;
            p) platform=${OPTARG} ;;
            *) help
               exit 1 ;;
        esac
    done

    # Run python script
    python /opt/src/demux_qc.py -p ${platform}