Merge pull request #76 from broadinstitute/dpark-dev

more docs, add align_and_fix python command

.gitignore

@@ -1,5 +1,7 @@
 VERSION
 test/output/
+
+# Sphinx documentation
 docs/_build/
 
 # Mac OSX

assembly.py

@@ -17,8 +17,9 @@
 log = logging.getLogger(__name__)
 
 
-def trim_rmdup_subsamp_reads(inBam, clipDb, outBam, n_reads=100000, outFastqs=None):
+def trim_rmdup_subsamp_reads(inBam, clipDb, outBam, n_reads=100000):
  ''' Take reads through Trimmomatic, Prinseq, and subsampling.
+ This should probably move over to read_utils or taxon_filter.
  '''
 
  # BAM -> fastq
@@ -57,6 +58,9 @@ def trim_rmdup_subsamp_reads(inBam, clipDb, outBam, n_reads=100000, outFastqs=No
  os.unlink(purgefq[1])
 
  # Fastq -> BAM
+ # Note: this destroys RG IDs! We should instead just pull the list
+ # of IDs out of purge_unmated, random.sample them ourselves, and
+ # use FilterSamReadsTool to go straight from inBam -> outBam
  tmp_bam = util.file.mkstempfname('.subsamp.bam')
  tmp_header = util.file.mkstempfname('.header.sam')
  tools.picard.FastqToSamTool().execute(
@@ -65,11 +69,6 @@ def trim_rmdup_subsamp_reads(inBam, clipDb, outBam, n_reads=100000, outFastqs=No
  tools.samtools.SamtoolsTool().reheader(tmp_bam, tmp_header, outBam)
  os.unlink(tmp_bam)
  os.unlink(tmp_header)
-
- # Save fastqs if requested
- if outFastqs:
- shutil.copyfile(subsampfq[0], outFastqs[0])
- shutil.copyfile(subsampfq[1], outFastqs[1])
  os.unlink(subsampfq[0])
  os.unlink(subsampfq[1])
 
@@ -83,14 +82,16 @@ def assemble_trinity(inBam, outFasta, clipDb, n_reads=100000, outReads=None):
  subsamp_bam = outReads
  else:
  subsamp_bam = util.file.mkstempfname('.subsamp.bam')
+
+ trim_rmdup_subsamp_reads(inBam, clipDb, subsamp_bam, n_reads=n_reads)
  subsampfq = list(map(util.file.mkstempfname, ['.subsamp.1.fastq', '.subsamp.2.fastq']))
- trim_rmdup_subsamp_reads(inBam, clipDb, subsamp_bam,
- n_reads=n_reads, outFastqs=subsampfq)
+ tools.picard.SamToFastqTool().execute(subsamp_bam, subsampfq[0], subsampfq[1])
  tools.trinity.TrinityTool().execute(subsampfq[0], subsampfq[1], outFasta)
- if not outReads:
- os.unlink(subsamp_bam)
  os.unlink(subsampfq[0])
  os.unlink(subsampfq[1])
+
+ if not outReads:
+ os.unlink(subsamp_bam)
 
 def parser_assemble_trinity(parser=argparse.ArgumentParser()):
  parser.add_argument('inBam',

docs/broad_utils.rst

@@ -1,5 +1,5 @@
 broad_utils.py - for data generated at the Broad Institute
-=====================================
+==========================================================
 
 .. argparse::
  :module: broad_utils

docs/conf.py

@@ -21,14 +21,22 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.insert(0, os.path.dirname(os.path.abspath('.')))
 
-import util.version
-
 # -- Mock out the heavyweight pip packages, esp those that require C ----
 import mock
 MOCK_MODULES = ['numpy', 'scipy', 'matplotlib', 'pysam',
  'Bio', 'Bio.AlignIO', 'Bio.SeqIO', 'Bio.Data.IUPACData']
 for mod_name in MOCK_MODULES:
  sys.modules[mod_name] = mock.Mock()
+
+# -- Obtain GIT version --
+import subprocess
+def _git_version():
+ cmd = ['git', 'describe', '--tags', '--always'] # omit "--dirty" from doc build
+ out = subprocess.check_output(cmd)
+ if type(out) != str:
+ out = out.decode('utf-8')
+ return out.strip()
+__version__ = _git_version()
 
 # -- General configuration ------------------------------------------------
 
@@ -65,7 +73,7 @@
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
-release = util.version.get_version()
+release = __version__
 version = '.'.join(release.split('.')[:2])
 
 # The language for content autogenerated by Sphinx. Refer to documentation
@@ -76,7 +84,7 @@
 # non-false value, then it is used:
 #today = ''
 # Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+today_fmt = '%Y-%m-%d'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -149,7 +157,7 @@
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
+html_last_updated_fmt = '%Y-%m-%d. {}'.format(release)
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.

docs/description.rst

@@ -0,0 +1,33 @@
+Description of the methods
+==========================
+
+Much more documentation to come...
+
+TO DO: here we will put a high level description of the various tools that
+exist here, perhaps with some pictures and such. We will describe why we
+used certain tools and approaches / how other approaches fell short / what
+kinds of problems certain steps are trying to solve. Perhaps some links to
+papers and such. Kind of a mini-methods paper here.
+
+
+Viral genome analysis
+---------------------
+
+*De novo* assembly, reference assisted assembly improvements,
+gene annotaion, species-level variation, within-host variation, etc.
+
+
+Taxonomic read filtration
+-------------------------
+
+Especially human read depletion (prior to submission to NCBI SRA).
+But also the part where we restrict to a particular taxa of interest
+(the species you're studying).
+
+
+Taxonomic read identification
+-----------------------------
+
+Nothing much here at the moment. That comes later, but we will later
+integrate it when it's ready.
+

docs/index.rst

@@ -14,6 +14,7 @@ Contents
  :maxdepth: 2
  :numbered:
 
+ description
  install
  cmdline
  pipeuse

docs/install.rst

@@ -70,3 +70,7 @@ Alternatively, if you are using the Snakemake pipelines, you can create
 a dictionary called "env_vars" in the config.json file for Snakemake,
 and the pipelines will automatically set all environment variables prior
 to running any scripts.
+
+The version of MOSAIK we use seems to fail compile on GCC-4.9 but compiles
+fine on GCC-4.4. We have not tried intermediate versions of GCC, nor the
+latest versions of MOSAIK.

docs/interhost.rst

@@ -1,5 +1,5 @@
 interhost.py - species and population-level genetic variation
-=====================================
+=============================================================
 
 .. argparse::
  :module: interhost

docs/read_utils.rst

@@ -1,5 +1,5 @@
 read_utils.py - utilities that manipulate bam and fastq files
-=====================================
+=============================================================
 
 .. argparse::
  :module: read_utils

docs/reports.rst

@@ -1,5 +1,5 @@
 reports.py - produce various metrics and reports
-=====================================
+================================================
 
 .. argparse::
  :module: reports

docs/taxon_filter.rst

@@ -1,5 +1,5 @@
 taxon_filter.py - tools for taxonomic removal or filtration of reads
-=====================================
+====================================================================
 
 .. argparse::
  :module: taxon_filter