Upgrade numpy to version 1.17.0

scripts/gatkcondaenv.yml.template

@@ -1,5 +1,9 @@
 # $condaEnvDescription
 #
+# NOTE: When the changing dependencies or the dependency versions in this file, check to see if the
+# "supportedPythonPackages" DataProvider used by the testGATKPythonEnvironmentPackagePresent test in
+# PythonEnvironmentIntegrationTest needs to be updated to reflect the new dependency or version.
+#
 name: $condaEnvName
 channels:
 - defaults
@@ -8,6 +12,7 @@ dependencies:
 - intel-openmp=2018.0.0
 - mkl=2018.0.1
 - mkl-service=1.1.2
+- defaults::numpy==1.17.0
 - openssl=1.0.2l=0
 - pip=9.0.1=py36_1
 - python=3.6.2=0
@@ -20,7 +25,6 @@ dependencies:
 - xz=5.2.3=0
 - zlib=1.2.11=0
 - pip:
-  - numpy==1.13.3
   - biopython==1.70
   - bleach==1.5.0
   - cycler==0.10.0

src/test/java/org/broadinstitute/hellbender/utils/python/PythonEnvironmentIntegrationTest.java

@@ -4,39 +4,63 @@
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
+import java.util.Collections;
+
 // Tests to nominally validate that the GATK conda environment is activated, and that it's
 // dependencies are accessible.
 //
-// We could validate the version numbers of the dependencies, but that would require
-// changes every time we change the gatkcondaenv.yml file.
-//
 public class PythonEnvironmentIntegrationTest {
     private final static String NL = System.lineSeparator();
 
     @DataProvider(name="supportedPythonPackages")
     public Object[][] getSupportedPythonPackages() {
         return new Object[][] {
                 // names of base packages that we should be able to import from within the GATK conda environment
-                // this list isn't exhaustive
-                { "numpy"},
-                { "scipy"},
-                { "tensorflow"},
-                { "theano" },
-                { "keras" },
-                { "pymc3" },
-                { "argparse" },
-                { "gcnvkernel" }
+                // NOTE: these must be kept in sync with the versions in gatkcondaenv.yml.template
+                { "numpy",      "1.17.0" },
+                { "scipy",      "1.0.0" },
+                { "tensorflow", "1.12.0" },
+                { "theano",     "1.0.4" },
+                { "keras",      "2.2.0" },
+                { "matplotlib", "2.1.0" },
+                { "pandas",     "0.21.0" },
+                { "pymc3",      "3.1" },
+                { "argparse",   null },
+                { "gcnvkernel", null }
         };
     }
 
     @Test(groups = {"python"}, dataProvider="supportedPythonPackages")
-    public void testGATKPythonEnvironmentPackagePresent(final String packageName) {
-        // Do a basic sanity check of the GATK Python conda environment. This test should only be run on
-        // the GATK docker image, or if the conda environment has been activated manually.
+    public void testGATKPythonEnvironmentPackagePresent(final String packageName, final String expectedVersion) {
+        // Sanity check to ensure that we can import these packages, and that conda is resolving them to
+        // the specific version we requested in the conda environment definition.
+        final StreamingPythonScriptExecutor<String> streamingPythonExecutor =
+                new StreamingPythonScriptExecutor<>(true);
+        Assert.assertNotNull(streamingPythonExecutor);
+        Assert.assertTrue(streamingPythonExecutor.start(Collections.emptyList(), true, null));
+
+        final String failedDependencyMessage = String.format(
+                "The installed version of %s does not match the %s version that was requested. " +
+                        "Check the build log to see the actual version that was resolved by conda.",
+                packageName,
+                expectedVersion);
+        try {
+            streamingPythonExecutor.sendSynchronousCommand(String.format("import %s" + NL, packageName));
+        } catch (final PythonScriptExecutorException e) {
+            throw new RuntimeException(failedDependencyMessage, e);
+        }
 
-        // We use the default python executable name ("python"), which in the activated gatk conda env should be Python 3.6.1
-        final PythonScriptExecutor pythonExecutor = new PythonScriptExecutor(true);
-        Assert.assertTrue(pythonExecutor.executeCommand(String.format("import %s", packageName) + NL,null,null));
+        // for some dependencies, we also validate that the conda environment got the version that we asked for
+        if (expectedVersion != null) {
+            try {
+                streamingPythonExecutor.sendSynchronousCommand(
+                    String.format("assert(%s.__version__ == '%s')" + NL,
+                            packageName,
+                            expectedVersion));
+            } catch (final PythonScriptExecutorException e) {
+                throw new RuntimeException(failedDependencyMessage, e);
+            }
+        }
     }
 
 }