Merge pull request #359 from broadinstitute/dp-build-optim

build optimizations

.gitignore

@@ -65,3 +65,7 @@ easy-deploy/data/config.yaml
 easy-deploy/data/viral-ngs/
 
 easy-deploy/.vagrant/
+
+tools/build/
+tools/conda-cache/
+tools/conda-tools/

.travis.yml

@@ -3,13 +3,15 @@ sudo: false
 
 cache:
  directories:
+ - $HOME/misc_cache
  - $HOME/virtualenv
  - $HOME/miniconda
 
 env:
  global:
- - CACHE_DIR="$HOME/virtualenv"
+ - CACHE_DIR="$HOME/misc_cache"
  - MINICONDA_DIR="$HOME/miniconda"
+ - PIP_DIR="$HOME/virtualenv"
  - GATK_PATH="$CACHE_DIR/bin_bundles/GenomeAnalysisTK-3.3-0-g37228af"
  - NOVOALIGN_PATH="$CACHE_DIR/bin_bundles/novocraft_v3"
  - PYTHONIOENCODING=UTF8
@@ -20,20 +22,6 @@ python:
  - 3.4
  - 3.5
 
-# Container pkgs for building diamond.
-# These versioned and sources hacks should be removed once travis upgrades to
-# Ubuntu 14.04.
-addons:
- apt:
- sources:
- - boost-latest # For newer boost
- - ubuntu-toolchain-r-test # For newer gcc
- packages:
- - cmake
- - gcc-4.9
- - g++-4.9
- - libboost1.55-all-dev
-
 git:
  depth: 3
 
@@ -48,15 +36,11 @@ install:
  - travis/install-tools.sh
 
 before_script:
- - flake8 --exit-zero .
+ - travis/flake8.sh
 
 script:
  - travis/tests-unit.sh
  - travis/tests-long.sh
 
 after_success:
  - coveralls
-
-notifications:
- slack:
- secure: dVmGF86ZaVkL7LhqmcjqTgmmc8ONZfLH2H2GO5AUHce3q61dA4tuZrfW0T+6puQWtIvvMC+5ioD0vRce+3iZELs83KR4Xm1vqNu6RZdAHx/ec3QQU4/7ulcEs50wWQl+WmyRqciGaBThUv+Un1K6AA2HYTcb8gQaFkpPmI3huZo=

install_tools.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+''' This script installs every Tool needed by viral-ngs
+'''
+
+from __future__ import print_function
+import os.path
+import sys
+import timeit
+import tools
+from tools import *
+import util.file
+
+__author__ = "dpark@broadinstitute.org"
+
+def install_all_tools():
+ sumtime = 0.0
+ n_tools = 0
+ n_success = 0
+ for tool_class in tools.all_tool_classes():
+ t = tool_class()
+ print("installing %s .. " % tool_class.__name__, end="")
+ sys.stdout.flush()
+ runtime = timeit.timeit(t.install)
+ sumtime += runtime
+ success = t.is_installed()
+ print("SUCCESS" if success else "FAILED", end="")
+ print(" (%0.1f seconds)" % runtime)
+ sys.stdout.flush()
+ if success:
+ n_success += 1
+ n_tools += 1
+ print("Total %d tools attempted, %d succeeded, %d failed, cumulative install time %0.1f seconds" % (
+ n_tools, n_success, n_tools - n_success, sumtime))
+ return (n_tools == n_success)
+
+
+if __name__ == '__main__':
+ print("this install script is %s" % (
+ os.path.abspath(os.path.expanduser(__file__))))
+ print("installing tools into: %s (build, conda-tools, conda-cache)" % (
+ os.path.join(util.file.get_project_path(), 'tools')))
+ sys.exit(0 if install_all_tools() else 1)

requirements-conda.txt

@@ -0,0 +1,17 @@
+blast=2.2.31
+bmtagger=3.101
+diamond=0.7.10=boost1.60_1
+kraken-all=0.10.6_eaf8fb68
+last=719
+mafft=7.221
+mummer=3.23
+muscle=3.8.1551
+mvicuna=1.0
+novoalign=3.03.02
+picard=1.126
+prinseq=0.20.4
+samtools=1.2
+snpeff=4.1l
+trimmomatic=0.35
+trinity=date.2011_11_26
+vphaser2=2.0

requirements-tests.txt

@@ -1,5 +1,4 @@
 coveralls==1.1
-flake8<=3
 pycodestyle
 mock==2.0.0
 six<2

test/unit/test_tools.py

@@ -1,50 +1,17 @@
 # Unit tests for tools/__init__.py
 
-__author__ = "dpark@broadinstitute.org"
+__author__ = "yesimon@broadinstitute.org"
 
 import tools
 from tools import *
-import unittest
-import tempfile
-import shutil
-import os
-import logging
-import util.cmd
-import util.file
-from test import TestCaseWithTmp
-import operator
 import pytest
 
-log = logging.getLogger(__name__)
-util.cmd.setup_logger('INFO')
-
-
-def iter_leaf_subclasses(aClass):
- "Iterate over subclasses at all levels that don't themselves have a subclass"
- isLeaf = True
- for subclass in sorted(aClass.__subclasses__(), key=operator.attrgetter("__name__")):
- isLeaf = False
- for leafClass in iter_leaf_subclasses(subclass):
- if not getattr(leafClass, '_skiptest', False):
- yield leafClass
- if isLeaf:
- yield aClass
-
-
-def all_tool_tests():
- for tool_class in iter_leaf_subclasses(tools.Tool):
- yield tool_class
-
-
-@pytest.fixture(params=all_tool_tests())
+@pytest.fixture(params=tools.all_tool_classes())
 def tool_class(request):
  print(request.param)
  return request.param
 
-
 def test_tool_install(tool_class):
  t = tool_class()
  t.install()
- assert t.is_installed(), "installation of tool %s failed" % tool_class.__name__
- log.info(".. installation of %s succeeded with installer %s" %
- (tool_class.__name__, t.installed_method.__class__.__name__))
+ assert t.is_installed()

tools/__init__.py

@@ -3,6 +3,8 @@
 __author__ = "dpark@broadinstitute.org,irwin@broadinstitute.org"
 
 import collections
+import json
+import operator
 import os
 import re
 import logging
@@ -11,7 +13,6 @@
 import subprocess
 import util.file
 import util.misc
-import json
 
 try:
  # Python 3.x
@@ -39,6 +40,21 @@
 _log = logging.getLogger(__name__)
 
 
+def iter_leaf_subclasses(aClass):
+ "Iterate over subclasses at all levels that don't themselves have a subclass"
+ isLeaf = True
+ for subclass in sorted(aClass.__subclasses__(), key=operator.attrgetter("__name__")):
+ isLeaf = False
+ for leafClass in iter_leaf_subclasses(subclass):
+ if not getattr(leafClass, '_skiptest', False):
+ yield leafClass
+ if isLeaf:
+ yield aClass
+
+def all_tool_classes():
+ return iter_leaf_subclasses(Tool)
+
+
 def get_tool_by_name(name):
  if name not in installed_tools:
  raise NotImplementedError
@@ -271,14 +287,14 @@ def __init__(
 
  # if the env is being overridden, or if we could not find an active conda env
  if env_root_path or env or not self.env_path:
- env_root_path = env_root_path or os.path.join(util.file.get_build_path(), 'conda-tools')
+ env_root_path = env_root_path or os.path.join(util.file.get_project_path(), 'tools', 'conda-tools')
  env = env or 'default'
  self.env_path = os.path.realpath(os.path.expanduser(
  os.path.join(env_root_path, env)))
 
  # set an env variable to the conda cache path. this env gets passed to the
  # the subprocess, and the variable instructs conda where to place its cache files
- conda_cache_path = conda_cache_path or os.path.join(util.file.get_build_path(), 'conda-cache')
+ conda_cache_path = conda_cache_path or os.path.join(util.file.get_project_path(), 'tools', 'conda-cache')
  self.conda_cache_path = os.path.realpath(os.path.expanduser(conda_cache_path))
  self.conda_env = os.environ
  old_envs_path = os.environ.get('CONDA_DEFAULT_ENV')

tools/blast.py

@@ -3,28 +3,10 @@
 import os
 import util.misc
 
-URL_PREFIX = 'ftp://ftp.ncbi.nlm.nih.gov/blast/executables' \
- '/blast+/2.2.29/ncbi-blast-2.2.29+-'
-
 TOOL_NAME = "blast"
 TOOL_VERSION = "2.2.31"
 
 
-def get_url():
- """ creates the download url for this tool """
- uname = os.uname()
- if uname[0] == 'Darwin':
- os_str = 'universal-macosx'
- elif uname[0] == 'Linux':
- if uname[4].endswith('64'):
- os_str = 'x64-linux'
- else:
- os_str = 'ia32-linux'
- else:
- raise NotImplementedError('OS {} not implemented'.format(uname[0]))
- return URL_PREFIX + os_str + '.tar.gz'
-
-
 class BlastTools(tools.Tool):
  """'Abstract' base class for tools in the blast+ suite.
  Subclasses must define class member subtool_name."""
@@ -37,22 +19,8 @@ def __init__(self, install_methods=None):
  ]
  self.subtool_name = self.subtool_name if hasattr(self, "subtool_name") else "blastn"
  if install_methods is None:
- target_rel_path = 'ncbi-blast-2.2.29+/bin/' + self.subtool_name
  install_methods = []
  install_methods.append(tools.CondaPackage(TOOL_NAME, executable=self.subtool_name, version=TOOL_VERSION))
- install_methods.append(
- tools.DownloadPackage(
- get_url(),
- target_rel_path,
- post_download_command=' '.join(
- ['rm'] + [
- 'ncbi-blast-2.2.29+/bin/' + f for f in unwanted
- ]
- ),
- post_download_ret=None
- )
- )
- #tools.Tool.__init__(self, install_methods=install_methods)
  super(BlastTools, self).__init__(install_methods=install_methods)
 
  def execute(self, *args):

tools/bmtagger.py

@@ -32,7 +32,6 @@ def __init__(self, install_methods=None):
  if install_methods is None:
  install_methods = []
  install_methods.append(tools.CondaPackage(TOOL_NAME, executable=self.subtool_name, version=TOOL_VERSION))
- install_methods.append(DownloadBmtagger(self.subtool_name))
  tools.Tool.__init__(self, install_methods=install_methods)
 
  def execute(self, *args):
@@ -125,47 +124,3 @@ def build_database(self, fasta_files, database_prefix_path):
  self.execute(*args)
 
  return database_prefix_path
-
-
-class DownloadBmtagger(tools.InstallMethod):
- """ InstallMethod class for downloading platform-specific bmtagger """
- executables = ['bmtagger.sh', 'bmfilter', 'extract_fullseq', 'srprism']
-
- def __init__(self, subtool_name):
- self.installed = False
- self.target_dir = os.path.join(util.file.get_build_path(), 'bmtagger')
- self.target_path = os.path.join(self.target_dir, subtool_name)
- tools.InstallMethod.__init__(self)
-
- def is_installed(self):
- return self.installed
-
- def executable_path(self):
- return self.installed and self.target_path or None
-
- def verify_install(self):
- """ confirms that the tools are present and executable """
- self.installed = all(
- os.access(
- os.path.join(self.target_dir, executable), (os.X_OK | os.R_OK)) for executable in self.executables
- )
- return self.installed
-
- def _attempt_install(self):
- if self.verify_install():
- return
- util.file.mkdir_p(self.target_dir)
- url_base = 'ftp://ftp.ncbi.nlm.nih.gov/pub/agarwala/bmtagger/'
- uname = os.uname()
- if uname[0] == 'Darwin':
- url_base += 'mac-os/'
- elif uname[0] != 'Linux' or not uname[4].endswith('64'):
- _log.debug('OS %s not implemented', uname[0])
- return
- for executable in self.executables:
- path = os.path.join(self.target_dir, executable)
- url = url_base + executable
- _log.info('Downloading from %s ...', url)
- urlretrieve(url, path)
- os.system('chmod +x ' + path)
- self.verify_install()

tools/diamond.py

@@ -18,6 +18,7 @@
 log = logging.getLogger(__name__)
 
 
+@tools.skip_install_test(condition=tools.is_osx())
 class Diamond(tools.Tool):
 
  SUBCOMMANDS = ['makedb', 'blastx', 'blastp', 'view']