Splitting out gatk-testUtils as a separate artifact

build.gradle

@@ -182,8 +182,19 @@ compileTestJava {
   options.compilerArgs = ['-proc:none', '-Xlint:all', '-Werror', '-Xdiags:verbose']
 }
 
+
+sourceSets {
+    testUtils
+}
+
 // Dependency change for including MLLib
 configurations {
+    testUtilsCompile.extendsFrom compile
+    testUtilsRuntime.extendsFrom runtime
+
+    testCompile.extendsFrom testUtilsCompile
+    testRuntime.extendsFrom testUtilsRuntime
+
     compile.exclude module: 'jul-to-slf4j'
     compile.exclude module: 'javax.servlet'
     compile.exclude module: 'servlet-api'
@@ -279,8 +290,6 @@ dependencies {
     }
 
     compile 'org.jgrapht:jgrapht-core:0.9.1'
-    compile 'org.testng:testng:' + testNGVersion //compile instead of testCompile because it is needed for test infrastructure that needs to be packaged
-    compile 'org.apache.hadoop:hadoop-minicluster:' + hadoopVersion
 
     compile('org.seqdoop:hadoop-bam:' + hadoopBamVersion) {
         exclude group: 'org.apache.hadoop'
@@ -325,6 +334,12 @@ dependencies {
     // Required for SV Discovery machine learning
     compile group: 'biz.k11i', name: 'xgboost-predictor', version: '0.3.0'
 
+    testUtilsCompile sourceSets.main.output
+    testUtilsCompile 'org.testng:testng:' + testNGVersion
+    testUtilsCompile 'org.apache.hadoop:hadoop-minicluster:' + hadoopVersion
+
+    testCompile sourceSets.testUtils.output
+
     testCompile "org.mockito:mockito-core:2.10.0"
     testCompile "com.google.jimfs:jimfs:1.1"
 }
@@ -677,6 +692,11 @@ task sourcesJar(type: Jar) {
     classifier = 'sources'
 }
 
+task testUtilsJar(type: Jar){
+    baseName = "$project.name-test-utils"
+    from sourceSets.testUtils.output
+}
+
 tasks.withType(Javadoc) {
     // do this for all javadoc tasks, including gatkDoc
     options.addStringOption('Xdoclint:none')
@@ -690,6 +710,29 @@ javadoc {
     include '**/*.java'
 }
 
+
+task testUtilsJavadoc(type: Javadoc) {
+    // This is a hack to disable the java 8 default javadoc lint until we fix the html formatting
+    // We only want to do this for the javadoc task, not gatkDoc
+    options.addStringOption('Xdoclint:none', '-quiet')
+    source = sourceSets.testUtils.allJava
+    classpath = sourceSets.testUtils.runtimeClasspath
+    destinationDir = file("$docBuildDir/testUtilsJavadoc")
+    include '**/*.java'
+}
+
+task testUtilsJavadocJar(type: Jar, dependsOn: testUtilsJavadoc){
+    baseName = "$project.name-test-utils"
+    classifier = 'javadoc'
+    from "$docBuildDir/testUtilsJavadoc"
+}
+
+task testUtilsSourceJar(type: Jar){
+    baseName = "$project.name-test-utils"
+    classifier = 'source'
+    from sourceSets.testUtils.allSource
+}
+
 // Generate GATK Online Doc
 task gatkDoc(type: Javadoc, dependsOn: classes) {
     final File gatkDocDir = new File("$docBuildDir/gatkdoc")
@@ -782,13 +825,18 @@ task gatkTabComplete(type: Javadoc, dependsOn: classes) {
     options.addStringOption("caller-post-arg-max-occurs", "1 1 1 1 1 1 1 1 1 1")
 }
 
+
+
 /**
  *This specifies what artifacts will be built and uploaded when performing a maven upload.
  */
 artifacts {
     archives jar
     archives javadocJar
     archives sourcesJar
+    archives testUtilsJar
+    archives testUtilsJavadocJar
+    archives testUtilsSourceJar
 
 }
 //remove zip and tar added by the application plugin
@@ -803,6 +851,17 @@ signing {
     sign configurations.archives
 }
 
+def gatkArtifactFilter = { artifact, file -> artifact.name == 'gatk' }
+def gatkTestUtilsArtifactFilter = {artifact, file -> artifact.name == 'gatk-test-utils' }
+
+install {
+    install {
+        repositories.mavenInstaller {
+            addFilter('gatk', gatkArtifactFilter)
+            addFilter('gatk-test-utils', gatkTestUtilsArtifactFilter)
+        }
+    }
+}
 /**
  * Upload a release to sonatype.  You must be an authorized uploader and have your sonatype
  * username and password information in your gradle properties file.  See the readme for more info.
@@ -825,6 +884,9 @@ uploadArchives {
                 authentication(userName: System.env.ARTIFACTORY_USERNAME, password: System.env.ARTIFACTORY_PASSWORD)
             }
 
+            addFilter('gatk', gatkArtifactFilter)
+            addFilter('gatk-test-utils', gatkTestUtilsArtifactFilter)
+
             pom.project {
                 name 'GATK4'
                 packaging 'jar'

src/main/java/org/broadinstitute/hellbender/engine/MultiVariantWalkerGroupedOnStart.java

@@ -3,18 +3,13 @@
 import com.google.common.annotations.VisibleForTesting;
 import htsjdk.samtools.util.Locatable;
 import htsjdk.samtools.util.OverlapDetector;
-import htsjdk.variant.variantcontext.Allele;
-import htsjdk.variant.variantcontext.GenotypesContext;
 import htsjdk.variant.variantcontext.VariantContext;
-import htsjdk.variant.variantcontext.VariantContextBuilder;
 import org.broadinstitute.barclay.argparser.Advanced;
 import org.broadinstitute.barclay.argparser.Argument;
-import org.broadinstitute.hellbender.exceptions.UserException;
 import org.broadinstitute.hellbender.utils.SimpleInterval;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Test;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
 
 
 /**

src/main/java/org/broadinstitute/hellbender/tools/funcotator/dataSources/gencode/GencodeFuncotationFactory.java

@@ -1,6 +1,7 @@
 package org.broadinstitute.hellbender.tools.funcotator.dataSources.gencode;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Sets;
 import htsjdk.samtools.SAMSequenceRecord;
 import htsjdk.samtools.reference.ReferenceSequence;
 import htsjdk.samtools.util.Locatable;
@@ -23,7 +24,6 @@
 import org.broadinstitute.hellbender.utils.param.ParamUtils;
 import org.broadinstitute.hellbender.utils.read.ReadUtils;
 import org.broadinstitute.hellbender.utils.variant.GATKVariantContextUtils;
-import org.testng.collections.Sets;
 
 import java.nio.file.Path;
 import java.util.*;
@@ -108,16 +108,16 @@ public class GencodeFuncotationFactory extends DataSourceFuncotationFactory {
      */
     private static final Set<GencodeFuncotation.VariantClassification> codingVariantClassifications =
             Sets.newHashSet(Arrays.asList(GencodeFuncotation.VariantClassification.MISSENSE,
-                            GencodeFuncotation.VariantClassification.NONSENSE,
-                            GencodeFuncotation.VariantClassification.NONSTOP,
-                            GencodeFuncotation.VariantClassification.SILENT,
-                            GencodeFuncotation.VariantClassification.IN_FRAME_DEL,
-                            GencodeFuncotation.VariantClassification.IN_FRAME_INS,
-                            GencodeFuncotation.VariantClassification.FRAME_SHIFT_INS,
-                            GencodeFuncotation.VariantClassification.FRAME_SHIFT_DEL,
-                            GencodeFuncotation.VariantClassification.START_CODON_SNP,
-                            GencodeFuncotation.VariantClassification.START_CODON_INS,
-                            GencodeFuncotation.VariantClassification.START_CODON_DEL));
+                                          GencodeFuncotation.VariantClassification.NONSENSE,
+                                          GencodeFuncotation.VariantClassification.NONSTOP,
+                                          GencodeFuncotation.VariantClassification.SILENT,
+                                          GencodeFuncotation.VariantClassification.IN_FRAME_DEL,
+                                          GencodeFuncotation.VariantClassification.IN_FRAME_INS,
+                                          GencodeFuncotation.VariantClassification.FRAME_SHIFT_INS,
+                                          GencodeFuncotation.VariantClassification.FRAME_SHIFT_DEL,
+                                          GencodeFuncotation.VariantClassification.START_CODON_SNP,
+                                          GencodeFuncotation.VariantClassification.START_CODON_INS,
+                                          GencodeFuncotation.VariantClassification.START_CODON_DEL));
 
 
     //==================================================================================================================

src/main/java/org/broadinstitute/hellbender/tools/walkers/annotator/BaseQuality.java

@@ -2,14 +2,11 @@
 
 import com.google.common.primitives.Ints;
 import htsjdk.variant.variantcontext.VariantContext;
-import org.apache.commons.math.util.FastMath;
+import org.apache.commons.math3.util.FastMath;
 import org.broadinstitute.barclay.help.DocumentedFeature;
 import org.broadinstitute.hellbender.utils.MathUtils;
-import org.broadinstitute.hellbender.utils.Utils;
 import org.broadinstitute.hellbender.utils.help.HelpConstants;
-import org.broadinstitute.hellbender.utils.read.AlignmentUtils;
 import org.broadinstitute.hellbender.utils.read.GATKRead;
-import org.broadinstitute.hellbender.utils.read.ReadUtils;
 
 import java.util.List;
 import java.util.OptionalDouble;

src/main/java/org/broadinstitute/hellbender/tools/walkers/annotator/ReadOrientationArtifact.java

@@ -5,7 +5,7 @@
 import htsjdk.variant.variantcontext.GenotypeBuilder;
 import htsjdk.variant.variantcontext.VariantContext;
 import htsjdk.variant.vcf.VCFFormatHeaderLine;
-import org.apache.commons.math.util.FastMath;
+import org.apache.commons.math3.util.FastMath;
 import org.broadinstitute.hellbender.engine.ReferenceContext;
 import org.broadinstitute.hellbender.tools.walkers.readorientation.*;
 import org.broadinstitute.hellbender.utils.GATKProtectedVariantContextUtils;

src/main/java/org/broadinstitute/hellbender/tools/walkers/mutect/Mutect2Engine.java

@@ -3,9 +3,12 @@
 import htsjdk.samtools.SAMFileHeader;
 import htsjdk.variant.variantcontext.VariantContext;
 import htsjdk.variant.variantcontext.writer.VariantContextWriter;
-import htsjdk.variant.vcf.*;
-import org.apache.commons.math.special.Beta;
-import org.apache.commons.math.special.Gamma;
+import htsjdk.variant.vcf.VCFConstants;
+import htsjdk.variant.vcf.VCFHeader;
+import htsjdk.variant.vcf.VCFHeaderLine;
+import htsjdk.variant.vcf.VCFStandardHeaderLines;
+import org.apache.commons.math3.special.Beta;
+import org.apache.commons.math3.special.Gamma;
 import org.apache.commons.math3.util.FastMath;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -21,7 +24,10 @@
 import org.broadinstitute.hellbender.tools.walkers.haplotypecaller.readthreading.ReadThreadingAssembler;
 import org.broadinstitute.hellbender.transformers.PalindromeArtifactClipReadTransformer;
 import org.broadinstitute.hellbender.transformers.ReadTransformer;
-import org.broadinstitute.hellbender.utils.*;
+import org.broadinstitute.hellbender.utils.MathUtils;
+import org.broadinstitute.hellbender.utils.QualityUtils;
+import org.broadinstitute.hellbender.utils.SimpleInterval;
+import org.broadinstitute.hellbender.utils.Utils;
 import org.broadinstitute.hellbender.utils.activityprofile.ActivityProfileState;
 import org.broadinstitute.hellbender.utils.fasta.CachingIndexedFastaSequenceFile;
 import org.broadinstitute.hellbender.utils.genotyper.IndexedSampleList;

src/main/java/org/broadinstitute/hellbender/utils/python/StreamingPythonScriptExecutor.java

@@ -7,9 +7,11 @@
 import org.broadinstitute.hellbender.exceptions.UserException;
 import org.broadinstitute.hellbender.utils.Utils;
 import org.broadinstitute.hellbender.utils.runtime.*;
-import org.testng.Assert;
 
-import java.io.*;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.Future;
@@ -293,7 +295,9 @@ protected Process getProcess() {
     public void terminate() {
         if (dataTransferFIFOWriter != null) {
             if (asyncWriter != null) {
-                Assert.assertTrue(asyncWriter.terminate());
+                if(!asyncWriter.terminate()){
+                    throw new GATKException("failed to close asyncWriter");
+                }
             }
             spController.writeProcessInput(PYTHON_CLOSE_DATA_FIFO);
             sendAckRequest();

src/test/java/org/broadinstitute/hellbender/BwaMemIntegrationTest.java

@@ -13,7 +13,7 @@
 import org.broadinstitute.hellbender.utils.bwa.BwaMemAligner;
 import org.broadinstitute.hellbender.utils.bwa.BwaMemAlignment;
 import org.broadinstitute.hellbender.utils.bwa.BwaMemIndex;
-import org.broadinstitute.hellbender.utils.test.ReadTestUtils;
+import org.broadinstitute.hellbender.testutils.ReadTestUtils;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;

src/test/java/org/broadinstitute/hellbender/CommandLineProgramTest.java

@@ -1,13 +1,8 @@
 package org.broadinstitute.hellbender;
 
-import org.broadinstitute.hellbender.utils.runtime.ProcessController;
-import org.broadinstitute.hellbender.utils.runtime.ProcessOutput;
-import org.broadinstitute.hellbender.utils.runtime.ProcessSettings;
-import org.broadinstitute.hellbender.utils.test.CommandLineProgramTester;
-import org.testng.Assert;
+import org.broadinstitute.hellbender.testutils.CommandLineProgramTester;
 
 import java.io.File;
-import java.util.Arrays;
 import java.util.List;
 
 /**

src/test/java/org/broadinstitute/hellbender/GATKBaseTest.java

@@ -5,7 +5,7 @@
 import org.broadinstitute.hellbender.utils.GenomeLocParser;
 import org.broadinstitute.hellbender.utils.fasta.CachingIndexedFastaSequenceFile;
 import org.broadinstitute.hellbender.utils.io.IOUtils;
-import org.broadinstitute.hellbender.utils.test.BaseTest;
+import org.broadinstitute.hellbender.testutils.BaseTest;
 import org.testng.annotations.BeforeClass;
 
 import java.io.File;

src/test/java/org/broadinstitute/hellbender/IntelInflaterDeflaterIntegrationTest.java

@@ -10,7 +10,7 @@
 import htsjdk.samtools.util.zip.InflaterFactory;
 import org.broadinstitute.hellbender.utils.NativeUtils;
 import org.broadinstitute.hellbender.utils.RandomDNA;
-import org.broadinstitute.hellbender.utils.test.SamAssertionUtils;
+import org.broadinstitute.hellbender.testutils.SamAssertionUtils;
 import org.testng.Assert;
 import org.testng.SkipException;
 import org.testng.annotations.DataProvider;

src/test/java/org/broadinstitute/hellbender/cmdline/PicardCommandLineProgramExecutorIntegrationTest.java

@@ -2,7 +2,7 @@
 
 import org.broadinstitute.hellbender.CommandLineProgramTest;
 import org.broadinstitute.hellbender.exceptions.PicardNonZeroExitException;
-import org.broadinstitute.hellbender.utils.test.IntegrationTestSpec;
+import org.broadinstitute.hellbender.testutils.IntegrationTestSpec;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 

src/test/java/org/broadinstitute/hellbender/engine/CRAMSupportIntegrationTest.java

@@ -9,7 +9,7 @@
 import org.broadinstitute.hellbender.exceptions.UserException;
 import org.broadinstitute.hellbender.tools.PrintReads;
 import org.broadinstitute.hellbender.GATKBaseTest;
-import org.broadinstitute.hellbender.utils.test.SamAssertionUtils;
+import org.broadinstitute.hellbender.testutils.SamAssertionUtils;
 import org.testng.Assert;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;