Merge branch 'release/v3.0.3'

.github/ISSUE_TEMPLATE/issue.md

@@ -8,7 +8,7 @@ about: General issue
 
  - [ ] The issue is strongly related to the MiXCR software
  - [ ] The issue can be reproduced with [the most recent version](https://github.com/milaboratory/mixcr/releases) of MiXCR
- - [ ] There is no answer to the question in the [official documentation](http://mixcr.readthedocs.io/) and there is no duplicate issue in the [bug tracker](https://github.com/milaboratory/mixcr/issues) 
+ - [ ] There is no answer to the question in the [official documentation](http://mixcr.readthedocs.io/) and there is no duplicate issue in the [bug tracker](https://github.com/milaboratory/mixcr/issues?utf8=✓&q=) 
  - [ ] Inspection of raw alignments with [exportAlignmentsPretty](http://mixcr.readthedocs.io/en/master/export.html#exporting-well-formatted-alignments-for-manual-inspection) shows that data has the expected architecture, and sample preparation artefacts are not the reason of the problem (if this is the matter of the issue)
 
 ## Expected Result

CHANGELOG

@@ -1,4 +1,14 @@
 
+MiXCR 3.0.3 (18 Nov 2018)
+========================
+
+-- Major fix for `analyze` action (wrong aligner paramters were used before)
+-- Also several minor fixes to the overal logic of `analyze`
+-- Fix for exported clonotype tables for specific receptor chains in `analyze` (#454)
+-- `--no-export` option added to `analyze` action (#454)
+-- Minor fixes for `clonesDiff` and `alignmentsDiff` actions
+
+
 MiXCR 3.0.2 (30 Oct 2018)
 ========================
 

doc/analyze.rst

@@ -290,23 +290,3 @@ The pipeline is equivalent to execution of the following MiXCR actions:
 
 As in the case of ``analyze amplicon``, required option ``--starting-material`` affects the choice of V gene region which will be used as target in ``align`` step (``vParameters.geneFeatureToAlign``, see :ref:`align documentation <ref-aligner-parameters>`): ``rna`` corresponds to the ``VTranscriptWithout5UTRWithP`` and ``dna`` to ``VGeneWithP`` (see :ref:`ref-geneFeatures` for details).
 
-
-
-Resuming execution / re-analyzing
----------------------------------
-
-MiXCR allows to continue aborted execution or re-analyze existing data with updated parameters, without complete re-processing of each of pipeline steps. This is possible with ``--overwrite-if-required`` option. For example, suppose the analysis was performed with the default options:
-
-::
- 
- mixcr analyze shotgun --species hs --starting-material rna data_R1.fastq data_R2.fastq analysis_name
-
-
-Now to re-analyze the data with updated options for :ref:`assemble <ref-assemble>`, one can pass ``--overwrite-if-required`` option in order to avoid unnecessary invocation of ``align``, ``assemblePartial`` and ``extend``:
-
-:: 
-
- mixcr analyze shotgun --overwrite-if-required --species hs --starting-material rna --assemble "-ObadQualityThreshold=0" data_R1.fastq data_R2.fastq analysis_name
-
-This way, the previous results of :ref:`align <ref-align>`, :ref:`assemblePartial <ref-assemblePartial>` and :ref:`extend <ref-extend>` will be used, while :ref:`assembly step <ref-assemble>` will be re-executed.
-

pom.xml

@@ -33,7 +33,7 @@
 
  <groupId>com.milaboratory</groupId>
  <artifactId>mixcr</artifactId>
- <version>3.0.2</version>
+ <version>3.0.3</version>
  <packaging>jar</packaging>
  <name>MiXCR</name>
 

src/main/java/com/milaboratory/mixcr/cli/CommandAlign.java

@@ -306,6 +306,7 @@ public void validate() {
  public final AlignerReport report = new AlignerReport();
 
  @Override
+ @SuppressWarnings("unchecked")
  public void run1() throws Exception {
  // Saving initial timestamp
  long beginTimestamp = System.currentTimeMillis();

src/main/java/com/milaboratory/mixcr/cli/CommandAlignmentsDiff.java

@@ -23,13 +23,13 @@
  separator = " ",
  description = "Calculates the difference between two .vdjca files.")
 public class CommandAlignmentsDiff extends ACommandWithOutput {
- @Parameters(description = "input_file1")
+ @Parameters(description = "input_file1", index = "0")
  public String in1;
 
- @Parameters(description = "input_file2")
+ @Parameters(description = "input_file2", index = "1")
  public String in2;
 
- @Parameters(description = "report")
+ @Parameters(description = "report", index = "2")
  public String report;
 
  @Option(names = {"-o1", "--only-in-first"}, description = "output for alignments contained only " +

src/main/java/com/milaboratory/mixcr/cli/CommandAnalyze.java

@@ -12,10 +12,7 @@
 import picocli.CommandLine.Parameters;
 
 import java.lang.reflect.Field;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
+import java.util.*;
 import java.util.stream.Collectors;
 
 public abstract class CommandAnalyze extends ACommandWithOutput {
@@ -171,7 +168,7 @@ private static abstract class EnumCandidates extends ArrayList<String> {
 
  static class _StartingMaterialCandidates extends EnumCandidates {
  _StartingMaterialCandidates() {
- super(_Chains.class);
+ super(_StartingMaterial.class);
  }
  }
 
@@ -241,6 +238,9 @@ public void setChains(String chains) {
  "%nNOTE: this will substantially increase analysis time.")
  public boolean contigAssembly = false;
 
+ @Option(names = {"--no-export"}, description = "Do not export clonotypes to tab-delimited file.")
+ public boolean noExport = false;
+
  @Option(names = {"-r", "--report"}, description = "Report file path")
  public String report = null;
 
@@ -283,6 +283,18 @@ public final CommandAlign getAlign() {
  return cmdAlign = inheritOptionsAndValidate(mkAlign());
  }
 
+ boolean forceUseRnaSeqOps() { return false; }
+
+ boolean include5UTRInRNA() { return true; }
+
+ Collection<String> pipelineSpecificAlignParameters() {
+ return Collections.EMPTY_LIST;
+ }
+
+ Collection<String> pipelineSpecificAssembleParameters() {
+ return Collections.EMPTY_LIST;
+ }
+
  CommandAlign mkAlign() {
  // align parameters
  List<String> alignParameters = new ArrayList<>(initialAlignParameters);
@@ -299,8 +311,26 @@ CommandAlign mkAlign() {
  alignParameters.add("--report");
  alignParameters.add(getReport());
 
- if (!chains.intersects(Chains.TCR))
+ if (!forceUseRnaSeqOps() && !chains.intersects(Chains.TCR))
  alignParameters.add("-p kAligner2");
+ else
+ alignParameters.add("-p rna-seq"); // always use rna-seq by default
+
+ // add v feature to align
+ switch (startingMaterial) {
+ case rna:
+ alignParameters.add("-OvParameters.geneFeatureToAlign=" +
+ (include5UTRInRNA()
+ ? "VTranscriptWithP"
+ : "VTranscriptWithout5UTRWithP"));
+ break;
+ case dna:
+ alignParameters.add("-OvParameters.geneFeatureToAlign=VGeneWithP");
+ break;
+ }
+
+ // pipeline specific parameters
+ alignParameters.addAll(this.pipelineSpecificAlignParameters());
 
  // add all override parameters
  alignParameters.addAll(this.alignParameters);
@@ -318,18 +348,6 @@ CommandAlign mkAlign() {
  .flatMap(s -> Arrays.stream(s.split(" ")))
  .toArray(String[]::new));
 
- switch (startingMaterial) {
- case rna:
- ap.getAlignerParameters()
- .getVAlignerParameters()
- .setGeneFeatureToAlign(GeneFeature.VTranscriptWithout5UTRWithP);
- break;
- case dna:
- ap.getAlignerParameters()
- .getVAlignerParameters()
- .setGeneFeatureToAlign(GeneFeature.VGeneWithP);
- break;
- }
 
  return ap;
  }
@@ -413,6 +431,9 @@ CommandAssemble mkAssemble(String input, String output) {
  // we always write clna
  assembleParameters.add("--write-alignments");
 
+ // pipeline specific parameters
+ assembleParameters.addAll(this.pipelineSpecificAssembleParameters());
+
  // add all override parameters
  assembleParameters.addAll(this.assembleParameters);
 
@@ -561,7 +582,9 @@ public void validate() {
  }
 
  @Override
- public void run0() throws Exception {
+ public void run0() {
+ JsonOverrider.suppressSameValueOverride = true;
+
  // --- Running alignments
  getAlign().run();
  String fileWithAlignments = fNameForAlignments();
@@ -590,13 +613,17 @@ public void run0() throws Exception {
  fileWithClones = fileWithContigs;
  }
 
- // --- Running export
- if (!chains.equals(Chains.ALL))
- for (String chain : chains)
- mkExport(fileWithClones, fNameForExportClones(chain)).run();
- else
- for (String chain : new String[]{"ALL", "TRA", "TRB", "TRG", "TRD", "IGH", "IGK", "IGL"})
- mkExport(fileWithClones, fNameForExportClones(chain)).run();
+ if (!noExport)
+ // --- Running export
+ if (!chains.equals(Chains.ALL))
+ for (String chain : chains)
+ mkExport(fileWithClones, fNameForExportClones(chain)).run();
+ else
+ for (String chain : new String[]{"ALL", "TRA", "TRB", "TRG", "TRD", "IGH", "IGK", "IGL"}) {
+ CommandExport.CommandExportClones export = mkExport(fileWithClones, fNameForExportClones(chain));
+ export.chains = chain;
+ export.run();
+ }
  }
 
 
@@ -662,32 +689,51 @@ private void setRegionOfInterest(String v) {
  }
 
  @Override
- CommandAlign mkAlign() {
- CommandAlign align = super.mkAlign();
+ boolean include5UTRInRNA() {
+ // (1) [ adapters == _Adapters.noAdapters ]
+ // If user specified that no adapter sequences are present in the data
+ // we can safely extend reference V region to cover 5'UTR, as there is
+ // no chance of false alignment extension over non-mRNA derived sequence
+ //
+ // (2) If [ vPrimers == _5EndPrimers.vPrimers && adapters == _Adapters.adaptersPresent ]
+ // VAlignerParameters.floatingLeftBound will be true, so it is also safe to add 5'UTR to the
+ // reference as the alignment will not be extended if sequences don't match.
+ //
+ // In all other cases addition of 5'UTR to the reference may lead to false extension of V alignment
+ // over adapter sequence.
+ // return adapters == _Adapters.noAdapters || vPrimers == _5EndPrimers.vPrimers; // read as adapters == _Adapters.noAdapters || floatingV()
+ return true;
+ }
 
- VDJCAlignerParameters alignmentParameters = align.getAlignerParameters();
- alignmentParameters.getVAlignerParameters().getParameters().setFloatingLeftBound(
- vPrimers == _5EndPrimers.vPrimers && adapters == _Adapters.adaptersPresent
- );
+ boolean floatingV() {
+ return vPrimers == _5EndPrimers.vPrimers || adapters == _Adapters.adaptersPresent;
+ }
 
-  alignmentParameters.getJAlignerParameters().getParameters().setFloatingRightBound(
-  jcPrimers == _3EndPrimers.jPrimers && adapters == _Adapters.adaptersPresent
-  );
+ boolean floatingJ() {
+ return jcPrimers == _3EndPrimers.jPrimers && adapters == _Adapters.adaptersPresent;
+ }
 
-  alignmentParameters.getCAlignerParameters().getParameters().setFloatingRightBound(
-  jcPrimers == _3EndPrimers.cPrimers && adapters == _Adapters.adaptersPresent
-  );
+ boolean floatingC() {
+ return jcPrimers == _3EndPrimers.cPrimers && adapters == _Adapters.adaptersPresent;
+ }
 
- return align;
+ @Override
+ Collection<String> pipelineSpecificAlignParameters() {
+ return Arrays.asList(
+ "-OvParameters.parameters.floatingLeftBound=" + floatingV(),
+ "-OjParameters.parameters.floatingRightBound=" + floatingJ(),
+ "-OcParameters.parameters.floatingRightBound=" + floatingC()
+ );
  }
 
  @Override
- public CommandAssemble mkAssemble(String input, String output) {
- CommandAssemble assemble = super.mkAssemble(input, output);
- CloneAssemblerParameters cloneAssemblyParameters = assemble.getCloneAssemblerParameters();
- cloneAssemblyParameters.setAssemblingFeatures(new GeneFeature[]{assemblingFeature});
- cloneAssemblyParameters.updateFrom(getAlign().getAlignerParameters());
- return assemble;
+ Collection<String> pipelineSpecificAssembleParameters() {
+ return Arrays.asList(
+ "-OassemblingFeatures=\"[" + GeneFeature.encode(assemblingFeature).replaceAll(" ", "") + "]\"",
+ "-OseparateByV=" + !floatingV(),
+ "-OseparateByJ=" + !floatingJ(),
+ "-OseparateByC=" + !(floatingC() || floatingJ())
+ );
  }
  }
 
@@ -727,30 +773,54 @@ public static CommandSpec mkAmplicon() {
  @Command(name = "shotgun",
  sortOptions = false,
  separator = " ",
- description = "Analyze random fragments (RNA-Seq, Exome-Seq, etc).")
+ description = "Analyze random-fragmented data (like RNA-Seq, Exome-Seq, etc). " +
+ "This pipeline assumes the data contain no adapter / primer sequences. " +
+ "Adapter trimming must be performed for the data containing any artificial sequence parts " +
+ "(e.g. single-cell / molecular-barcoded data).")
  public static class CommandShotgun extends CommandAnalyze {
  public CommandShotgun() {
  chains = Chains.ALL;
  nAssemblePartialRounds = 2;
  doNotExtendAlignments = false;
  }
 
+ @Override
+ boolean forceUseRnaSeqOps() {
+ return true;
+ }
+
  @Override
  CommandAlign mkAlign() {
  CommandAlign align = super.mkAlign();
  VDJCAlignerParameters alignmentParameters = align.getAlignerParameters();
- alignmentParameters.getVAlignerParameters().getParameters().setFloatingLeftBound(false);
- alignmentParameters.getJAlignerParameters().getParameters().setFloatingRightBound(false);
- alignmentParameters.getCAlignerParameters().getParameters().setFloatingRightBound(false);
+ if (alignmentParameters.getVAlignerParameters().getParameters().isFloatingLeftBound())
+ throwValidationException("'shotgun' pipeline requires '-OvParameters.parameters.floatingLeftBound=false'.");
+ if (alignmentParameters.getJAlignerParameters().getParameters().isFloatingRightBound())
+ throwValidationException("'shotgun' pipeline requires '-OjParameters.parameters.floatingRightBound=false'.");
+ if (alignmentParameters.getCAlignerParameters().getParameters().isFloatingRightBound())
+ throwValidationException("'shotgun' pipeline requires '-OcParameters.parameters.floatingRightBound=false'.");
  return align;
  }
 
+ @Override
+ Collection<String> pipelineSpecificAssembleParameters() {
+ return Arrays.asList(
+ "-OseparateByV=true",
+ "-OseparateByJ=true",
+ "-OseparateByC=true"
+ );
+ }
+
  @Override
  public CommandAssemble mkAssemble(String input, String output) {
  CommandAssemble assemble = super.mkAssemble(input, output);
  CloneAssemblerParameters cloneAssemblyParameters = assemble.getCloneAssemblerParameters();
- cloneAssemblyParameters.setAssemblingFeatures(new GeneFeature[]{GeneFeature.CDR3});
- cloneAssemblyParameters.updateFrom(getAlign().getAlignerParameters());
+
+ if (!Arrays.equals(cloneAssemblyParameters.getAssemblingFeatures(), new GeneFeature[]{GeneFeature.CDR3}))
+ throwValidationException("'shotgun' pipeline can only use CDR3 as assembling feature. " +
+ "See --contig-assembly and --impute-germline-on-export options if you want to " +
+ "cover wider part of the receptor sequence.");
+
  return assemble;
  }
  }