From b3df1f021e39e662c5a4403e39ee6aef92ec0dc3 Mon Sep 17 00:00:00 2001 From: Danny Park Date: Sat, 18 Jun 2016 21:10:25 -0400 Subject: [PATCH 01/19] remove tools/build from travis cache, remove apt packages for building diamond from scratch --- .travis.yml | 18 ------------------ travis/before_install.sh | 12 ------------ 2 files changed, 30 deletions(-) diff --git a/.travis.yml b/.travis.yml index aa19d2728..a48b6db76 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,20 +20,6 @@ python: - 3.4 - 3.5 -# Container pkgs for building diamond. -# These versioned and sources hacks should be removed once travis upgrades to -# Ubuntu 14.04. -addons: - apt: - sources: - - boost-latest # For newer boost - - ubuntu-toolchain-r-test # For newer gcc - packages: - - cmake - - gcc-4.9 - - g++-4.9 - - libboost1.55-all-dev - git: depth: 3 @@ -56,7 +42,3 @@ script: after_success: - coveralls - -notifications: - slack: - secure: dVmGF86ZaVkL7LhqmcjqTgmmc8ONZfLH2H2GO5AUHce3q61dA4tuZrfW0T+6puQWtIvvMC+5ioD0vRce+3iZELs83KR4Xm1vqNu6RZdAHx/ec3QQU4/7ulcEs50wWQl+WmyRqciGaBThUv+Un1K6AA2HYTcb8gQaFkpPmI3huZo= diff --git a/travis/before_install.sh b/travis/before_install.sh index eeaeee4b3..140197fd1 100755 --- a/travis/before_install.sh +++ b/travis/before_install.sh @@ -3,18 +3,6 @@ # cache depending on whether we're on the master branch or not. set -e -# Only sometimes cache the tools/build directory -if [ -z "$TRAVIS_TAG" ]; then - echo "Travis docker caches allowed for branch $TRAVIS_BRANCH" - rm -rf tools/build - mkdir -p $CACHE_DIR/tools_build - ln -s $CACHE_DIR/tools_build tools/build - -else - echo "Travis docker cache disabled for tools/build on tag: $TRAVIS_TAG" - -fi - # Report how big things are echo "Docker cache space usage:" du -hs $CACHE_DIR/* From 06f83a5f0c3619f15d5b4bbbc91e62a72f44f38a Mon Sep 17 00:00:00 2001 From: Danny Park Date: Sun, 19 Jun 2016 21:07:43 -0400 Subject: [PATCH 02/19] remove a bunch of non-conda based tool installers --- tools/blast.py | 32 ---------------------- tools/bmtagger.py | 45 ------------------------------- tools/last.py | 43 ------------------------------ tools/mafft.py | 67 ----------------------------------------------- tools/mummer.py | 8 +----- tools/muscle.py | 64 -------------------------------------------- tools/prinseq.py | 14 ---------- 7 files changed, 1 insertion(+), 272 deletions(-) diff --git a/tools/blast.py b/tools/blast.py index 4b76b86d4..d9824d4cb 100644 --- a/tools/blast.py +++ b/tools/blast.py @@ -3,28 +3,10 @@ import os import util.misc -URL_PREFIX = 'ftp://ftp.ncbi.nlm.nih.gov/blast/executables' \ - '/blast+/2.2.29/ncbi-blast-2.2.29+-' - TOOL_NAME = "blast" TOOL_VERSION = "2.2.31" -def get_url(): - """ creates the download url for this tool """ - uname = os.uname() - if uname[0] == 'Darwin': - os_str = 'universal-macosx' - elif uname[0] == 'Linux': - if uname[4].endswith('64'): - os_str = 'x64-linux' - else: - os_str = 'ia32-linux' - else: - raise NotImplementedError('OS {} not implemented'.format(uname[0])) - return URL_PREFIX + os_str + '.tar.gz' - - class BlastTools(tools.Tool): """'Abstract' base class for tools in the blast+ suite. Subclasses must define class member subtool_name.""" @@ -37,22 +19,8 @@ def __init__(self, install_methods=None): ] self.subtool_name = self.subtool_name if hasattr(self, "subtool_name") else "blastn" if install_methods is None: - target_rel_path = 'ncbi-blast-2.2.29+/bin/' + self.subtool_name install_methods = [] install_methods.append(tools.CondaPackage(TOOL_NAME, executable=self.subtool_name, version=TOOL_VERSION)) - install_methods.append( - tools.DownloadPackage( - get_url(), - target_rel_path, - post_download_command=' '.join( - ['rm'] + [ - 'ncbi-blast-2.2.29+/bin/' + f for f in unwanted - ] - ), - post_download_ret=None - ) - ) - #tools.Tool.__init__(self, install_methods=install_methods) super(BlastTools, self).__init__(install_methods=install_methods) def execute(self, *args): diff --git a/tools/bmtagger.py b/tools/bmtagger.py index d073d9fca..e3fd8aa02 100644 --- a/tools/bmtagger.py +++ b/tools/bmtagger.py @@ -32,7 +32,6 @@ def __init__(self, install_methods=None): if install_methods is None: install_methods = [] install_methods.append(tools.CondaPackage(TOOL_NAME, executable=self.subtool_name, version=TOOL_VERSION)) - install_methods.append(DownloadBmtagger(self.subtool_name)) tools.Tool.__init__(self, install_methods=install_methods) def execute(self, *args): @@ -125,47 +124,3 @@ def build_database(self, fasta_files, database_prefix_path): self.execute(*args) return database_prefix_path - - -class DownloadBmtagger(tools.InstallMethod): - """ InstallMethod class for downloading platform-specific bmtagger """ - executables = ['bmtagger.sh', 'bmfilter', 'extract_fullseq', 'srprism'] - - def __init__(self, subtool_name): - self.installed = False - self.target_dir = os.path.join(util.file.get_build_path(), 'bmtagger') - self.target_path = os.path.join(self.target_dir, subtool_name) - tools.InstallMethod.__init__(self) - - def is_installed(self): - return self.installed - - def executable_path(self): - return self.installed and self.target_path or None - - def verify_install(self): - """ confirms that the tools are present and executable """ - self.installed = all( - os.access( - os.path.join(self.target_dir, executable), (os.X_OK | os.R_OK)) for executable in self.executables - ) - return self.installed - - def _attempt_install(self): - if self.verify_install(): - return - util.file.mkdir_p(self.target_dir) - url_base = 'ftp://ftp.ncbi.nlm.nih.gov/pub/agarwala/bmtagger/' - uname = os.uname() - if uname[0] == 'Darwin': - url_base += 'mac-os/' - elif uname[0] != 'Linux' or not uname[4].endswith('64'): - _log.debug('OS %s not implemented', uname[0]) - return - for executable in self.executables: - path = os.path.join(self.target_dir, executable) - url = url_base + executable - _log.info('Downloading from %s ...', url) - urlretrieve(url, path) - os.system('chmod +x ' + path) - self.verify_install() diff --git a/tools/last.py b/tools/last.py index e04b22c80..1c7e3bbec 100644 --- a/tools/last.py +++ b/tools/last.py @@ -26,52 +26,9 @@ def __init__(self, install_methods=None): if install_methods is None: install_methods = [] install_methods.append(tools.CondaPackage(TOOL_NAME, executable=self.subtool_name, version=TOOL_VERSION)) - install_methods.append(DownloadAndBuildLast(self.subtool_name)) - # Version of last on broad is old, database-incompatible with newer - # one, so don't use it and always load the newer version - #path = os.path.join(lastBroadUnixPath, self.subtool_name_on_broad) - # install_methods.append(tools.PrexistingUnixCommand(path)) tools.Tool.__init__(self, install_methods=install_methods) -class DownloadAndBuildLast(tools.DownloadPackage): - """ class for platform-specific last download and install """ - last_with_version = 'last-719' - - def __init__(self, subtool_name): - url = 'http://last.cbrc.jp/{}.zip'.format(self.last_with_version) - target_rel_path = os.path.join(self.last_with_version, 'bin', subtool_name) - tools.DownloadPackage.__init__(self, url, target_rel_path) - - def post_download(self): - path = os.path.join(self.destination_dir, self.last_with_version) - os.system('cd {}; make -s; make -s install prefix=.'.format(path)) - - # maf_convert doesn't run in Python 3.x. Fix it (in place). - binpath = os.path.join(path, 'bin') - maf_convert_path = os.path.join(binpath, 'maf-convert') - os.system('2to3 {maf_convert_path} -w --no-diffs'.format(**locals())) - # Still more fixes needed: the first for 3.x, the second for 2.7 - with open(maf_convert_path, 'rt') as inf: - file_contents = inf.read() - file_contents = file_contents.replace('string.maketrans("", "")', 'None') - file_contents = file_contents.replace( - '#! /usr/bin/env python', '#! /usr/bin/env python\nfrom __future__ import print_function' - ) - with open(maf_convert_path, 'wt') as outf: - outf.write(file_contents) - - def verify_install(self): - 'Default checks + verify python 2.7/3.x compatibility fixes were done' - if not tools.DownloadPackage.verify_install(self): - return False - maf_convert_path = os.path.join(self.destination_dir, self.last_with_version, 'bin', 'maf-convert') - if not os.access(maf_convert_path, os.X_OK | os.R_OK): - return False - with open(maf_convert_path, 'rt') as inf: - return 'print_function' in inf.read() - - class Lastal(LastTools): """ wrapper for lastal subtool """ subtool_name = 'lastal' diff --git a/tools/mafft.py b/tools/mafft.py index b87fdbb2c..da5088fb3 100644 --- a/tools/mafft.py +++ b/tools/mafft.py @@ -10,14 +10,12 @@ import tools import util.file import util.misc - import os import os.path import subprocess TOOL_NAME = "mafft" TOOL_VERSION = '7.221' -TOOL_URL = 'http://mafft.cbrc.jp/alignment/software/mafft-{ver}-{os}.{ext}' _log = logging.getLogger(__name__) @@ -27,35 +25,7 @@ class MafftTool(tools.Tool): def __init__(self, install_methods=None): if install_methods is None: install_methods = [] - mafft_os = get_mafft_os() - mafft_bitdepth = get_mafft_bitdepth() - mafft_archive_extension = get_mafft_archive_extension(mafft_os) - binaryPath = get_mafft_binary_path(mafft_os, mafft_bitdepth) - binaryDir = get_mafft_binary_path(mafft_os, mafft_bitdepth, full=False) - - target_rel_path = '{binPath}'.format(binPath=binaryPath) - verify_command = 'cd {dir}/mafft-{ver}/{bin_dir} && {dir}/mafft-{ver}/{binPath} --version > /dev/null 2>&1'.format( - dir=util.file.get_build_path(), - ver=TOOL_VERSION, - binPath=binaryPath, - bin_dir=binaryDir - ) - destination_dir = '{dir}/mafft-{ver}'.format(dir=util.file.get_build_path(), ver=TOOL_VERSION) - install_methods.append(tools.CondaPackage(TOOL_NAME, version=TOOL_VERSION)) - install_methods.append( - tools.DownloadPackage( - TOOL_URL.format( - ver=TOOL_VERSION, - os=mafft_os, - ext=mafft_archive_extension - ), - target_rel_path=target_rel_path, - destination_dir=destination_dir, - verifycmd=verify_command - ) - ) - tools.Tool.__init__(self, install_methods=install_methods) def version(self): @@ -186,40 +156,3 @@ def execute( return outFile # pylint: enable=W0221 - - -def get_mafft_os(): - uname = os.uname() - if uname[0] == "Darwin": - return "mac" - if uname[0] == "Linux": - return "linux" - - -def get_mafft_archive_extension(mafft_os): - if mafft_os == "mac": - return "zip" - elif mafft_os == "linux": - return "tgz" - - -def get_mafft_bitdepth(): - uname = os.uname() - if uname[4] == "x86_64": - return "64" - if uname[4] in ['i386', 'i686', "x86"]: - return "32" - - -def get_mafft_binary_path(mafft_os, bitdepth, full=True): - mafftPath = "" - - if mafft_os == "mac": - mafftPath += "mafft-mac/" - elif mafft_os == "linux": - mafftPath += "mafft-linux{bit}".format(bit=bitdepth) + "/" - - if full: - mafftPath += "mafft.bat" - - return mafftPath diff --git a/tools/mummer.py b/tools/mummer.py index e6588a5fb..8d234890b 100644 --- a/tools/mummer.py +++ b/tools/mummer.py @@ -15,7 +15,6 @@ TOOL_NAME = "mummer" tool_version = '3.23' -url = 'http://iweb.dl.sourceforge.net/project/mummer/mummer/{ver}/MUMmer{ver}.tar.gz' log = logging.getLogger(__name__) @@ -25,12 +24,7 @@ class MummerTool(tools.Tool): def __init__(self, install_methods=None): if install_methods is None: install_methods = [ - tools.CondaPackage(TOOL_NAME, version=tool_version), - tools.DownloadPackage(url.format(ver=tool_version), - 'MUMmer{}'.format(tool_version), - post_download_command='cd MUMmer{}; make -s'.format(tool_version), - verifycmd='{}/MUMmer{}/mummer -h > /dev/null 2>&1'.format( - util.file.get_build_path(), tool_version)) + tools.CondaPackage(TOOL_NAME, version=tool_version) ] tools.Tool.__init__(self, install_methods=install_methods) diff --git a/tools/muscle.py b/tools/muscle.py index 8da018e13..8e68dce41 100644 --- a/tools/muscle.py +++ b/tools/muscle.py @@ -16,9 +16,6 @@ TOOL_NAME = "muscle" TOOL_VERSION = '3.8.1551' CONDA_TOOL_VERSION = '3.8.1551' -TOOL_URL = 'http://www.drive5.com/muscle/downloads{ver}/muscle{ver}_{os}.tar.gz' - -UNRELEASED_URL = 'http://www.drive5.com/muscle/muscle_src_{ver}.tar.gz' _log = logging.getLogger(__name__) @@ -29,9 +26,6 @@ def __init__(self, install_methods=None): if install_methods is None: install_methods = [] install_methods.append(tools.CondaPackage(TOOL_NAME, version=CONDA_TOOL_VERSION)) - install_methods.append( - DownloadAndBuildMuscleUnreleased(ver=TOOL_VERSION) - ) tools.Tool.__init__(self, install_methods=install_methods) def version(self): @@ -71,61 +65,3 @@ def execute( subprocess.check_call(tool_cmd) # pylint: enable=W0221 - -class DownloadAndBuildMuscleUnreleased(tools.DownloadPackage): - - def __init__(self, ver): - tools.DownloadPackage.__init__(self, - UNRELEASED_URL.format(ver=ver, os='src'), - 'muscle', - destination_dir=os.path.join( - util.file.get_build_path(), - 'muscle-{}'.format(ver)), - verifycmd='{}/muscle-{}/muscle -version > /dev/null 2>&1'.format( - util.file.get_build_path(), TOOL_VERSION), - post_download_command='make -s') - - def post_download(self): - if os.uname()[0] == 'Darwin': - # Makefile needs to be modified for MacOSX - badFilePath = os.path.join(self.destination_dir, 'Makefile') - os.rename(badFilePath, badFilePath + '.orig') - with open(badFilePath, 'wt') as outf: - with open(badFilePath + '.orig', 'rt') as inf: - for line in inf: - if line.strip() == 'LDLIBS = -lm -static': - line = '# ' + line - elif line.strip() == '# LDLIBS = -lm': - line = line[2:] - outf.write(line) - tools.DownloadPackage.post_download(self) - - -class DownloadAndBuildMuscle(tools.DownloadPackage): - - def __init__(self, ver): - tools.DownloadPackage.__init__(self, - TOOL_URL.format(ver=ver, os='src'), - 'muscle', - destination_dir=os.path.join( - util.file.get_build_path(), - 'muscle-{}'.format(ver)), - verifycmd='{}/muscle-{}/muscle -version > /dev/null 2>&1'.format( - util.file.get_build_path(), TOOL_VERSION), - post_download_command='make -s') - - def post_download(self): - if os.uname()[0] == 'Darwin': - # Makefile needs to be modified for MacOSX - badFilePath = os.path.join(self.destination_dir, 'Makefile') - os.rename(badFilePath, badFilePath + '.orig') - with open(badFilePath, 'wt') as outf: - with open(badFilePath + '.orig', 'rt') as inf: - for line in inf: - if line.strip() == 'LDLIBS = -lm -static': - line = '# ' + line - elif line.strip() == '# LDLIBS = -lm': - line = line[2:] - outf.write(line) - tools.DownloadPackage.post_download(self) - diff --git a/tools/prinseq.py b/tools/prinseq.py index c3b60d7d6..8a796c132 100644 --- a/tools/prinseq.py +++ b/tools/prinseq.py @@ -4,25 +4,11 @@ TOOL_NAME = "prinseq" TOOL_VERSION = '0.20.4' -TOOL_URL = 'http://sourceforge.net/projects/prinseq/files/standalone/' \ - 'prinseq-lite-{ver}.tar.gz'.format(ver=TOOL_VERSION) - class PrinseqTool(tools.Tool): def __init__(self, install_methods=None): if install_methods is None: install_methods = [] - install_methods.append(tools.CondaPackage(TOOL_NAME, executable="prinseq-lite.pl", version=TOOL_VERSION)) - - target_rel_path = 'prinseq-lite-{ver}/prinseq-lite.pl'.format(ver=TOOL_VERSION) - install_methods.append( - tools.DownloadPackage( - TOOL_URL, - target_rel_path, - post_download_command='chmod +x {}'.format(target_rel_path), - require_executability=True - ) - ) tools.Tool.__init__(self, install_methods=install_methods) From 8f257d30ae3810fe54364857676524664fefa97d Mon Sep 17 00:00:00 2001 From: Danny Park Date: Sun, 19 Jun 2016 21:37:06 -0400 Subject: [PATCH 03/19] remove concurrent py.test xdist on tool installation --- pytest.ini | 2 +- travis/tests-long.sh | 2 +- travis/tests-unit.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pytest.ini b/pytest.ini index ee37b1220..9b16d49e9 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -addopts = -rsxX -n 2 --durations=50 --junit-xml=pytest.xml --cov-report= +addopts = -rsxX --durations=50 --junit-xml=pytest.xml --cov-report= --cov broad_utils --cov illumina --cov assembly diff --git a/travis/tests-long.sh b/travis/tests-long.sh index b58af1ea5..b4f8e08b5 100755 --- a/travis/tests-long.sh +++ b/travis/tests-long.sh @@ -6,7 +6,7 @@ echo "TRAVIS_PULL_REQUEST: $TRAVIS_PULL_REQUEST" if [ $TRAVIS_PULL_REQUEST != "false" -o $TRAVIS_BRANCH = "master" -o -n "$TRAVIS_TAG" ]; then echo "This is on master or is a pull request: executing long running tests..." - py.test --cov-append test/integration + py.test -n auto --cov-append test/integration else echo "This is not a pull request: skipping long running tests." fi diff --git a/travis/tests-unit.sh b/travis/tests-unit.sh index 3f4fcdc7e..bac164da9 100755 --- a/travis/tests-unit.sh +++ b/travis/tests-unit.sh @@ -1,4 +1,4 @@ #!/bin/bash #set -e -py.test test/unit +py.test -n auto test/unit From 9c79f7c5f66c83adc4c1a556ba47f3f38508da19 Mon Sep 17 00:00:00 2001 From: Danny Park Date: Mon, 20 Jun 2016 13:59:23 -0400 Subject: [PATCH 04/19] move conda-cache and conda-tools up one directory level (to tools/) and go back to caching the non-conda tools --- .gitignore | 4 ++++ tools/__init__.py | 4 ++-- travis/before_install.sh | 12 ++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 463413027..a7b697912 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,7 @@ easy-deploy/data/config.yaml easy-deploy/data/viral-ngs/ easy-deploy/.vagrant/ + +tools/build/ +tools/conda-cache/ +tools/conda-tools/ diff --git a/tools/__init__.py b/tools/__init__.py index cf1c08ebf..7eb2a9ec5 100644 --- a/tools/__init__.py +++ b/tools/__init__.py @@ -271,14 +271,14 @@ def __init__( # if the env is being overridden, or if we could not find an active conda env if env_root_path or env or not self.env_path: - env_root_path = env_root_path or os.path.join(util.file.get_build_path(), 'conda-tools') + env_root_path = env_root_path or os.path.join(util.file.get_build_path(), '..', 'conda-tools') env = env or 'default' self.env_path = os.path.realpath(os.path.expanduser( os.path.join(env_root_path, env))) # set an env variable to the conda cache path. this env gets passed to the # the subprocess, and the variable instructs conda where to place its cache files - conda_cache_path = conda_cache_path or os.path.join(util.file.get_build_path(), 'conda-cache') + conda_cache_path = conda_cache_path or os.path.join(util.file.get_build_path(), '..', 'conda-cache') self.conda_cache_path = os.path.realpath(os.path.expanduser(conda_cache_path)) self.conda_env = os.environ old_envs_path = os.environ.get('CONDA_DEFAULT_ENV') diff --git a/travis/before_install.sh b/travis/before_install.sh index 140197fd1..eeaeee4b3 100755 --- a/travis/before_install.sh +++ b/travis/before_install.sh @@ -3,6 +3,18 @@ # cache depending on whether we're on the master branch or not. set -e +# Only sometimes cache the tools/build directory +if [ -z "$TRAVIS_TAG" ]; then + echo "Travis docker caches allowed for branch $TRAVIS_BRANCH" + rm -rf tools/build + mkdir -p $CACHE_DIR/tools_build + ln -s $CACHE_DIR/tools_build tools/build + +else + echo "Travis docker cache disabled for tools/build on tag: $TRAVIS_TAG" + +fi + # Report how big things are echo "Docker cache space usage:" du -hs $CACHE_DIR/* From d481daf76cc5183b5ed88be3d26148449b018ddf Mon Sep 17 00:00:00 2001 From: Danny Park Date: Mon, 20 Jun 2016 16:21:24 -0400 Subject: [PATCH 05/19] move pytest.ini into .travis.yml PYTEST_ADDOPTS variable, so that local developers arent burdened with travis-specific options when they run py.test --- .travis.yml | 1 + pytest.ini | 14 -------------- travis/install-tools.sh | 2 +- 3 files changed, 2 insertions(+), 15 deletions(-) delete mode 100644 pytest.ini diff --git a/.travis.yml b/.travis.yml index a48b6db76..05c128d25 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ env: - NOVOALIGN_PATH="$CACHE_DIR/bin_bundles/novocraft_v3" - PYTHONIOENCODING=UTF8 - secure: l9tLtFKGNhaRdRN2N7Fiks63VatVCOtDUG7FI/pi7JNJu/EriTwDRlncoVCRCJZKOdxG8OrwC1BLX6CNqpVjJISEPGV/djsf2wCV9vi6oa+OsvMymsJAjOYkLezwRLVZp/0l/sGumPGz+q+XIM8VnkOZezIvZjGaaAtBpRTHdmA= + - PYTEST_ADDOPTS="-rsxX --durations=50 --junit-xml=pytest.xml --cov-report= --cov broad_utils --cov illumina --cov assembly --cov interhost --cov intrahost --cov metagenomics --cov ncbi --cov read_utils --cov reports --cov taxon_filter --cov tools --cov util" python: - 2.7 diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 9b16d49e9..000000000 --- a/pytest.ini +++ /dev/null @@ -1,14 +0,0 @@ -[pytest] -addopts = -rsxX --durations=50 --junit-xml=pytest.xml --cov-report= - --cov broad_utils - --cov illumina - --cov assembly - --cov interhost - --cov intrahost - --cov metagenomics - --cov ncbi - --cov read_utils - --cov reports - --cov taxon_filter - --cov tools - --cov util diff --git a/travis/install-tools.sh b/travis/install-tools.sh index 8245625cb..3ef9ad8db 100755 --- a/travis/install-tools.sh +++ b/travis/install-tools.sh @@ -17,4 +17,4 @@ if [ ! -d $GATK_PATH -o ! -d $NOVOALIGN_PATH ]; then fi echo "Installing and validating bioinformatic tools" -py.test test/unit/test_tools.py +py.test -n 1 -v --durations=50 test/unit/test_tools.py From e99b1a6a1c4413a649cfcce80b3dc47e2ec9e338 Mon Sep 17 00:00:00 2001 From: Danny Park Date: Mon, 20 Jun 2016 17:07:53 -0400 Subject: [PATCH 06/19] add a requirements-conda.txt file which repeats whats stated in each Tool (omit GATK, Megan, and Krona for now) so that we can download from conda all in one shot in travis/install-tools as a speed optimization --- requirements-conda.txt | 17 +++++++++++++++++ travis/install-tools.sh | 2 ++ 2 files changed, 19 insertions(+) create mode 100644 requirements-conda.txt diff --git a/requirements-conda.txt b/requirements-conda.txt new file mode 100644 index 000000000..8384ec2f8 --- /dev/null +++ b/requirements-conda.txt @@ -0,0 +1,17 @@ +blast=2.2.31 +bmtagger=3.101 +diamond=0.7.10-boost1.60_1 +kraken-all=0.10.6_eaf8fb68 +last=719 +mafft=7.221 +mummer=3.23 +muscle=3.8.1551 +mvicuna=1.0 +novoalign=3.03.02 +picard=1.126 +prinseq=0.20.4 +samtools=1.2 +snpeff=4.1l +trimmomatic=0.35 +trinity=date.2011_11_26 +vphaser2=2.0 diff --git a/travis/install-tools.sh b/travis/install-tools.sh index 3ef9ad8db..a08e57ba6 100755 --- a/travis/install-tools.sh +++ b/travis/install-tools.sh @@ -17,4 +17,6 @@ if [ ! -d $GATK_PATH -o ! -d $NOVOALIGN_PATH ]; then fi echo "Installing and validating bioinformatic tools" +export CONDA_ENVS_PATH=tools/conda-cache:tools/conda-tools/default +conda create -y -m -c bioconda -p tools/conda-tools/default --file requirements-conda.txt py.test -n 1 -v --durations=50 test/unit/test_tools.py From f9f8da73a6280823d4c8b8111d6d2c2e28644290 Mon Sep 17 00:00:00 2001 From: Danny Park Date: Tue, 21 Jun 2016 09:03:28 -0400 Subject: [PATCH 07/19] fix diamond string --- requirements-conda.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-conda.txt b/requirements-conda.txt index 8384ec2f8..bce238159 100644 --- a/requirements-conda.txt +++ b/requirements-conda.txt @@ -1,6 +1,6 @@ blast=2.2.31 bmtagger=3.101 -diamond=0.7.10-boost1.60_1 +diamond=0.7.10=boost1.60_1 kraken-all=0.10.6_eaf8fb68 last=719 mafft=7.221 From d97e4d702cc8a03beba59cb5e97d67982b078480 Mon Sep 17 00:00:00 2001 From: Danny Park Date: Tue, 21 Jun 2016 09:18:32 -0400 Subject: [PATCH 08/19] move PYTEST_ADDOPTS into test-unit and test-long (from travis.yml) so that they dont apply to install-tools. reduce threading from -n auto to -n 2 for travis. --- .travis.yml | 1 - travis/tests-long.sh | 3 ++- travis/tests-unit.sh | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 05c128d25..a48b6db76 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,6 @@ env: - NOVOALIGN_PATH="$CACHE_DIR/bin_bundles/novocraft_v3" - PYTHONIOENCODING=UTF8 - secure: l9tLtFKGNhaRdRN2N7Fiks63VatVCOtDUG7FI/pi7JNJu/EriTwDRlncoVCRCJZKOdxG8OrwC1BLX6CNqpVjJISEPGV/djsf2wCV9vi6oa+OsvMymsJAjOYkLezwRLVZp/0l/sGumPGz+q+XIM8VnkOZezIvZjGaaAtBpRTHdmA= - - PYTEST_ADDOPTS="-rsxX --durations=50 --junit-xml=pytest.xml --cov-report= --cov broad_utils --cov illumina --cov assembly --cov interhost --cov intrahost --cov metagenomics --cov ncbi --cov read_utils --cov reports --cov taxon_filter --cov tools --cov util" python: - 2.7 diff --git a/travis/tests-long.sh b/travis/tests-long.sh index b4f8e08b5..f461c5963 100755 --- a/travis/tests-long.sh +++ b/travis/tests-long.sh @@ -6,7 +6,8 @@ echo "TRAVIS_PULL_REQUEST: $TRAVIS_PULL_REQUEST" if [ $TRAVIS_PULL_REQUEST != "false" -o $TRAVIS_BRANCH = "master" -o -n "$TRAVIS_TAG" ]; then echo "This is on master or is a pull request: executing long running tests..." - py.test -n auto --cov-append test/integration + PYTEST_ADDOPTS="-rsxX -n 2 --durations=50 --junit-xml=pytest.xml --cov-report= --cov broad_utils --cov illumina --cov assembly --cov interhost --cov intrahost --cov metagenomics --cov ncbi --cov read_utils --cov reports --cov taxon_filter --cov tools --cov util" + py.test --cov-append test/integration else echo "This is not a pull request: skipping long running tests." fi diff --git a/travis/tests-unit.sh b/travis/tests-unit.sh index bac164da9..3075a2b13 100755 --- a/travis/tests-unit.sh +++ b/travis/tests-unit.sh @@ -1,4 +1,5 @@ #!/bin/bash #set -e -py.test -n auto test/unit +PYTEST_ADDOPTS="-rsxX -n 2 --durations=50 --junit-xml=pytest.xml --cov-report= --cov broad_utils --cov illumina --cov assembly --cov interhost --cov intrahost --cov metagenomics --cov ncbi --cov read_utils --cov reports --cov taxon_filter --cov tools --cov util" +py.test test/unit From 09a432c6fdcd2528d1ce5609bd5ee35f106cc3f0 Mon Sep 17 00:00:00 2001 From: Danny Park Date: Tue, 21 Jun 2016 09:44:48 -0400 Subject: [PATCH 09/19] create install_tools.py, an introspective all-tools installer that avoids the need for py.test or requirements-tests to install all tools --- install_tools.py | 33 +++++++++++++++++++++++++++++++++ test/unit/test_tools.py | 39 +++------------------------------------ tools/__init__.py | 18 +++++++++++++++++- tools/diamond.py | 1 + travis/install-tools.sh | 3 ++- 5 files changed, 56 insertions(+), 38 deletions(-) create mode 100755 install_tools.py diff --git a/install_tools.py b/install_tools.py new file mode 100755 index 000000000..3ef187096 --- /dev/null +++ b/install_tools.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +''' This script installs every Tool needed by viral-ngs +''' + +from __future__ import print_function +import timeit +import tools +from tools import * + +__author__ = "dpark@broadinstitute.org" + +def install_all_tools(): + sumtime = 0.0 + n_tools = 0 + n_success = 0 + for tool_class in tools.all_tool_classes(): + t = tool_class() + print("installing %s .. " % tool_class.__name__, end="") + runtime = timeit.timeit(t.install) + sumtime += runtime + success = t.is_installed() + print("SUCCESS" if success else "FAILED", end="") + print(" (%0.1f seconds)" % runtime) + if success: + n_success += 1 + n_tools += 1 + print("Total %d tools attempted, %d succeeded, %d failed, cumulative install time %0.1f seconds" % ( + n_tools, n_success, n_tools - n_success, sumtime)) + return (n_tools == n_success) + + +if __name__ == '__main__': + return install_all_tools() diff --git a/test/unit/test_tools.py b/test/unit/test_tools.py index 28d1d7a14..bb403d54c 100644 --- a/test/unit/test_tools.py +++ b/test/unit/test_tools.py @@ -1,50 +1,17 @@ # Unit tests for tools/__init__.py -__author__ = "dpark@broadinstitute.org" +__author__ = "yesimon@broadinstitute.org" import tools from tools import * -import unittest -import tempfile -import shutil -import os -import logging -import util.cmd -import util.file -from test import TestCaseWithTmp -import operator import pytest -log = logging.getLogger(__name__) -util.cmd.setup_logger('INFO') - - -def iter_leaf_subclasses(aClass): - "Iterate over subclasses at all levels that don't themselves have a subclass" - isLeaf = True - for subclass in sorted(aClass.__subclasses__(), key=operator.attrgetter("__name__")): - isLeaf = False - for leafClass in iter_leaf_subclasses(subclass): - if not getattr(leafClass, '_skiptest', False): - yield leafClass - if isLeaf: - yield aClass - - -def all_tool_tests(): - for tool_class in iter_leaf_subclasses(tools.Tool): - yield tool_class - - -@pytest.fixture(params=all_tool_tests()) +@pytest.fixture(params=tools.all_tool_classes()) def tool_class(request): print(request.param) return request.param - def test_tool_install(tool_class): t = tool_class() t.install() - assert t.is_installed(), "installation of tool %s failed" % tool_class.__name__ - log.info(".. installation of %s succeeded with installer %s" % - (tool_class.__name__, t.installed_method.__class__.__name__)) + assert t.is_installed() diff --git a/tools/__init__.py b/tools/__init__.py index 7eb2a9ec5..a200c2f2c 100644 --- a/tools/__init__.py +++ b/tools/__init__.py @@ -3,6 +3,8 @@ __author__ = "dpark@broadinstitute.org,irwin@broadinstitute.org" import collections +import json +import operator import os import re import logging @@ -11,7 +13,6 @@ import subprocess import util.file import util.misc -import json try: # Python 3.x @@ -39,6 +40,21 @@ _log = logging.getLogger(__name__) +def iter_leaf_subclasses(aClass): + "Iterate over subclasses at all levels that don't themselves have a subclass" + isLeaf = True + for subclass in sorted(aClass.__subclasses__(), key=operator.attrgetter("__name__")): + isLeaf = False + for leafClass in iter_leaf_subclasses(subclass): + if not getattr(leafClass, '_skiptest', False): + yield leafClass + if isLeaf: + yield aClass + +def all_tool_classes(): + return iter_leaf_subclasses(Tool) + + def get_tool_by_name(name): if name not in installed_tools: raise NotImplementedError diff --git a/tools/diamond.py b/tools/diamond.py index e9da1896e..1de1fc992 100644 --- a/tools/diamond.py +++ b/tools/diamond.py @@ -18,6 +18,7 @@ log = logging.getLogger(__name__) +@tools.skip_install_test(condition=tools.is_osx()) class Diamond(tools.Tool): SUBCOMMANDS = ['makedb', 'blastx', 'blastp', 'view'] diff --git a/travis/install-tools.sh b/travis/install-tools.sh index a08e57ba6..d95e82307 100755 --- a/travis/install-tools.sh +++ b/travis/install-tools.sh @@ -19,4 +19,5 @@ fi echo "Installing and validating bioinformatic tools" export CONDA_ENVS_PATH=tools/conda-cache:tools/conda-tools/default conda create -y -m -c bioconda -p tools/conda-tools/default --file requirements-conda.txt -py.test -n 1 -v --durations=50 test/unit/test_tools.py +#py.test -n 1 -v --durations=50 test/unit/test_tools.py +./install_tools.py From 80e50e0088d7493fa9576091c5ef2a2c827c489a Mon Sep 17 00:00:00 2001 From: Danny Park Date: Tue, 21 Jun 2016 10:02:47 -0400 Subject: [PATCH 10/19] return exit code for install_tools and flush print staement --- install_tools.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/install_tools.py b/install_tools.py index 3ef187096..cab098dbd 100755 --- a/install_tools.py +++ b/install_tools.py @@ -3,6 +3,7 @@ ''' from __future__ import print_function +import sys import timeit import tools from tools import * @@ -15,12 +16,12 @@ def install_all_tools(): n_success = 0 for tool_class in tools.all_tool_classes(): t = tool_class() - print("installing %s .. " % tool_class.__name__, end="") + print("installing %s .. " % tool_class.__name__, end="", flush=True) runtime = timeit.timeit(t.install) sumtime += runtime success = t.is_installed() print("SUCCESS" if success else "FAILED", end="") - print(" (%0.1f seconds)" % runtime) + print(" (%0.1f seconds)" % runtime, flush=True) if success: n_success += 1 n_tools += 1 @@ -30,4 +31,4 @@ def install_all_tools(): if __name__ == '__main__': - return install_all_tools() + sys.exit(1 if install_all_tools() else 0) From e315e79f89302fe26e9308467391a8660a8192fe Mon Sep 17 00:00:00 2001 From: Danny Park Date: Tue, 21 Jun 2016 10:13:05 -0400 Subject: [PATCH 11/19] oops, backwards return codes --- install_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_tools.py b/install_tools.py index cab098dbd..883571af2 100755 --- a/install_tools.py +++ b/install_tools.py @@ -31,4 +31,4 @@ def install_all_tools(): if __name__ == '__main__': - sys.exit(1 if install_all_tools() else 0) + sys.exit(0 if install_all_tools() else 1) From 08d3647b47ac72c08ade5cecec4b229130010877 Mon Sep 17 00:00:00 2001 From: Danny Park Date: Tue, 21 Jun 2016 10:24:12 -0400 Subject: [PATCH 12/19] flush print in a py2-compatible manner --- install_tools.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/install_tools.py b/install_tools.py index 883571af2..d88b6bea6 100755 --- a/install_tools.py +++ b/install_tools.py @@ -16,12 +16,14 @@ def install_all_tools(): n_success = 0 for tool_class in tools.all_tool_classes(): t = tool_class() - print("installing %s .. " % tool_class.__name__, end="", flush=True) + print("installing %s .. " % tool_class.__name__, end="") + sys.stdout.flush() runtime = timeit.timeit(t.install) sumtime += runtime success = t.is_installed() print("SUCCESS" if success else "FAILED", end="") - print(" (%0.1f seconds)" % runtime, flush=True) + print(" (%0.1f seconds)" % runtime) + sys.stdout.flush() if success: n_success += 1 n_tools += 1 From 80c954ad88f7f03711fa0f44014f39d5ed164b7e Mon Sep 17 00:00:00 2001 From: Danny Park Date: Tue, 21 Jun 2016 10:26:16 -0400 Subject: [PATCH 13/19] make sure intrahost rule params are serializable --- pipes/rules/intrahost.rules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipes/rules/intrahost.rules b/pipes/rules/intrahost.rules index 4ee3f3738..8a6bb843c 100644 --- a/pipes/rules/intrahost.rules +++ b/pipes/rules/intrahost.rules @@ -53,7 +53,7 @@ rule isnvs_vcf: logid="all", refGenome=os.path.join(config["ref_genome_dir"],"reference"+".fasta"), snpEff_ref=config["accessions_for_ref_genome_build"], - samples=read_samples_file(config["samples_assembly"]), + samples=list(read_samples_file(config["samples_assembly"])), emailAddress=config["email_point_of_contact_for_ncbi"] run: shell("{config[bin_dir]}/intrahost.py merge_to_vcf {params.refGenome} {output[0]}" @@ -87,7 +87,7 @@ rule isnvs_vcf_filtered: logid="all", refGenome=os.path.join(config["ref_genome_dir"],"reference"+".fasta"), snpEff_ref=config["accessions_for_ref_genome_build"], - samples=read_samples_file(config["samples_assembly"]), + samples=list(read_samples_file(config["samples_assembly"])), emailAddress=config["email_point_of_contact_for_ncbi"] run: shell("{config[bin_dir]}/intrahost.py merge_to_vcf {params.refGenome} {output[0]}" From 2e5151cb509fc876e001638fc25959e7c4bd442e Mon Sep 17 00:00:00 2001 From: Danny Park Date: Tue, 21 Jun 2016 10:33:12 -0400 Subject: [PATCH 14/19] move flake8 to travis/flake8.sh and only run on PRs and master. export PYTEST_ADDOPTS to ensure it gets used for unit and long tests --- .travis.yml | 2 +- requirements-tests.txt | 1 - travis/flake8.sh | 13 +++++++++++++ travis/tests-long.sh | 2 +- travis/tests-unit.sh | 2 +- 5 files changed, 16 insertions(+), 4 deletions(-) create mode 100755 travis/flake8.sh diff --git a/.travis.yml b/.travis.yml index a48b6db76..3ffebfc42 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,7 +34,7 @@ install: - travis/install-tools.sh before_script: - - flake8 --exit-zero . + - travis/flake8.sh script: - travis/tests-unit.sh diff --git a/requirements-tests.txt b/requirements-tests.txt index 9ffd618d0..831c9d690 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,5 +1,4 @@ coveralls==1.1 -flake8<=3 pycodestyle mock==2.0.0 six<2 diff --git a/travis/flake8.sh b/travis/flake8.sh new file mode 100755 index 000000000..91cc230c4 --- /dev/null +++ b/travis/flake8.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +echo "TRAVIS_BRANCH: $TRAVIS_BRANCH" +echo "TRAVIS_PULL_REQUEST: $TRAVIS_PULL_REQUEST" + +if [ $TRAVIS_PULL_REQUEST != "false" -o $TRAVIS_BRANCH = "master" -o -n "$TRAVIS_TAG" ]; then + echo "This is on master or is a pull request: checking code style..." + pip install 'flake8<=3' + flake8 --exit-zero . +else + echo "This is not a pull request: skipping code style check." +fi diff --git a/travis/tests-long.sh b/travis/tests-long.sh index f461c5963..2be1bdc70 100755 --- a/travis/tests-long.sh +++ b/travis/tests-long.sh @@ -6,7 +6,7 @@ echo "TRAVIS_PULL_REQUEST: $TRAVIS_PULL_REQUEST" if [ $TRAVIS_PULL_REQUEST != "false" -o $TRAVIS_BRANCH = "master" -o -n "$TRAVIS_TAG" ]; then echo "This is on master or is a pull request: executing long running tests..." - PYTEST_ADDOPTS="-rsxX -n 2 --durations=50 --junit-xml=pytest.xml --cov-report= --cov broad_utils --cov illumina --cov assembly --cov interhost --cov intrahost --cov metagenomics --cov ncbi --cov read_utils --cov reports --cov taxon_filter --cov tools --cov util" + export PYTEST_ADDOPTS="-rsxX -n 2 --durations=50 --junit-xml=pytest.xml --cov-report= --cov broad_utils --cov illumina --cov assembly --cov interhost --cov intrahost --cov metagenomics --cov ncbi --cov read_utils --cov reports --cov taxon_filter --cov tools --cov util" py.test --cov-append test/integration else echo "This is not a pull request: skipping long running tests." diff --git a/travis/tests-unit.sh b/travis/tests-unit.sh index 3075a2b13..3f4f89a2c 100755 --- a/travis/tests-unit.sh +++ b/travis/tests-unit.sh @@ -1,5 +1,5 @@ #!/bin/bash #set -e -PYTEST_ADDOPTS="-rsxX -n 2 --durations=50 --junit-xml=pytest.xml --cov-report= --cov broad_utils --cov illumina --cov assembly --cov interhost --cov intrahost --cov metagenomics --cov ncbi --cov read_utils --cov reports --cov taxon_filter --cov tools --cov util" +export PYTEST_ADDOPTS="-rsxX -n 2 --durations=50 --junit-xml=pytest.xml --cov-report= --cov broad_utils --cov illumina --cov assembly --cov interhost --cov intrahost --cov metagenomics --cov ncbi --cov read_utils --cov reports --cov taxon_filter --cov tools --cov util" py.test test/unit From a32dad5ff67f0c0dc88f26d87765326fd399ad10 Mon Sep 17 00:00:00 2001 From: Danny Park Date: Tue, 21 Jun 2016 10:46:50 -0400 Subject: [PATCH 15/19] add placeholders for where extra caching might go if need be --- travis/before_install.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/travis/before_install.sh b/travis/before_install.sh index eeaeee4b3..08b8bc466 100755 --- a/travis/before_install.sh +++ b/travis/before_install.sh @@ -7,8 +7,10 @@ set -e if [ -z "$TRAVIS_TAG" ]; then echo "Travis docker caches allowed for branch $TRAVIS_BRANCH" rm -rf tools/build - mkdir -p $CACHE_DIR/tools_build + mkdir -p $CACHE_DIR/tools_build $CACHE_DIR/conda-tools $CACHE_DIR/conda-cache ln -s $CACHE_DIR/tools_build tools/build + #ln -s $CACHE_DIR/conda-cache tools + #ln -s $CACHE_DIR/conda-tools tools else echo "Travis docker cache disabled for tools/build on tag: $TRAVIS_TAG" From 0e0c2070be569aa608546cb479f856bdaafd27db Mon Sep 17 00:00:00 2001 From: Danny Park Date: Tue, 21 Jun 2016 11:04:07 -0400 Subject: [PATCH 16/19] some debug reporting for tool installer --- install_tools.py | 6 ++++++ tools/__init__.py | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/install_tools.py b/install_tools.py index d88b6bea6..0f7610d32 100755 --- a/install_tools.py +++ b/install_tools.py @@ -3,10 +3,12 @@ ''' from __future__ import print_function +import os.path import sys import timeit import tools from tools import * +import util.file __author__ = "dpark@broadinstitute.org" @@ -33,4 +35,8 @@ def install_all_tools(): if __name__ == '__main__': + print("this install script is %s" % ( + os.path.abspath(os.path.expanduser(__file__)))) + print("installing tools into: %s (build, conda-tools, conda-cache)" % ( + os.path.join(util.file.get_project_path(), 'tools'))) sys.exit(0 if install_all_tools() else 1) diff --git a/tools/__init__.py b/tools/__init__.py index a200c2f2c..e471c487b 100644 --- a/tools/__init__.py +++ b/tools/__init__.py @@ -287,14 +287,14 @@ def __init__( # if the env is being overridden, or if we could not find an active conda env if env_root_path or env or not self.env_path: - env_root_path = env_root_path or os.path.join(util.file.get_build_path(), '..', 'conda-tools') + env_root_path = env_root_path or os.path.join(util.file.get_project_path(), 'tools', 'conda-tools') env = env or 'default' self.env_path = os.path.realpath(os.path.expanduser( os.path.join(env_root_path, env))) # set an env variable to the conda cache path. this env gets passed to the # the subprocess, and the variable instructs conda where to place its cache files - conda_cache_path = conda_cache_path or os.path.join(util.file.get_build_path(), '..', 'conda-cache') + conda_cache_path = conda_cache_path or os.path.join(util.file.get_project_path(), 'tools', 'conda-cache') self.conda_cache_path = os.path.realpath(os.path.expanduser(conda_cache_path)) self.conda_env = os.environ old_envs_path = os.environ.get('CONDA_DEFAULT_ENV') From ee928072a7eeddc8bc8f7bf5132b4f2eb8ab2ef1 Mon Sep 17 00:00:00 2001 From: Danny Park Date: Tue, 21 Jun 2016 11:19:54 -0400 Subject: [PATCH 17/19] move travis cache dirs around --- .travis.yml | 4 +++- travis/before_install.sh | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3ffebfc42..f8d5dfa1f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,13 +3,15 @@ sudo: false cache: directories: + - $HOME/misc_cache - $HOME/virtualenv - $HOME/miniconda env: global: - - CACHE_DIR="$HOME/virtualenv" + - CACHE_DIR="$HOME/cache" - MINICONDA_DIR="$HOME/miniconda" + - PIP_DIR="$HOME/virtualenv" - GATK_PATH="$CACHE_DIR/bin_bundles/GenomeAnalysisTK-3.3-0-g37228af" - NOVOALIGN_PATH="$CACHE_DIR/bin_bundles/novocraft_v3" - PYTHONIOENCODING=UTF8 diff --git a/travis/before_install.sh b/travis/before_install.sh index 08b8bc466..d4813319c 100755 --- a/travis/before_install.sh +++ b/travis/before_install.sh @@ -19,4 +19,4 @@ fi # Report how big things are echo "Docker cache space usage:" -du -hs $CACHE_DIR/* +du -hs $MINICONDA_DIR $PIP_DIR $CACHE_DIR/* From 3381719d18d239a1b0ebe910bba07e4009e23145 Mon Sep 17 00:00:00 2001 From: Danny Park Date: Tue, 21 Jun 2016 12:10:34 -0400 Subject: [PATCH 18/19] Revert "make sure intrahost rule params are serializable" This reverts commit 80c954ad88f7f03711fa0f44014f39d5ed164b7e. --- pipes/rules/intrahost.rules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipes/rules/intrahost.rules b/pipes/rules/intrahost.rules index 8a6bb843c..4ee3f3738 100644 --- a/pipes/rules/intrahost.rules +++ b/pipes/rules/intrahost.rules @@ -53,7 +53,7 @@ rule isnvs_vcf: logid="all", refGenome=os.path.join(config["ref_genome_dir"],"reference"+".fasta"), snpEff_ref=config["accessions_for_ref_genome_build"], - samples=list(read_samples_file(config["samples_assembly"])), + samples=read_samples_file(config["samples_assembly"]), emailAddress=config["email_point_of_contact_for_ncbi"] run: shell("{config[bin_dir]}/intrahost.py merge_to_vcf {params.refGenome} {output[0]}" @@ -87,7 +87,7 @@ rule isnvs_vcf_filtered: logid="all", refGenome=os.path.join(config["ref_genome_dir"],"reference"+".fasta"), snpEff_ref=config["accessions_for_ref_genome_build"], - samples=list(read_samples_file(config["samples_assembly"])), + samples=read_samples_file(config["samples_assembly"]), emailAddress=config["email_point_of_contact_for_ncbi"] run: shell("{config[bin_dir]}/intrahost.py merge_to_vcf {params.refGenome} {output[0]}" From 5992507dab92c6b568252338c475e708c83a7b6a Mon Sep 17 00:00:00 2001 From: Danny Park Date: Tue, 21 Jun 2016 12:11:52 -0400 Subject: [PATCH 19/19] fix travis cache dir --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f8d5dfa1f..db9b0e754 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ cache: env: global: - - CACHE_DIR="$HOME/cache" + - CACHE_DIR="$HOME/misc_cache" - MINICONDA_DIR="$HOME/miniconda" - PIP_DIR="$HOME/virtualenv" - GATK_PATH="$CACHE_DIR/bin_bundles/GenomeAnalysisTK-3.3-0-g37228af"