Keep test suite memory usage under 3GB for CI (#360)

* Split tests to be under Travis 3GB memory limit

* CI should install package

* Loss function test, fix random seed for reproducibility

.appveyor.yml

@@ -49,7 +49,7 @@ install:
   # - appveyor DownloadFile %LAPACK_URL% dlls\lapack.dll
 
 build_script:
-  - nimble.exe refresh
+  - nimble.exe install -y
 
 test_script:
   - nimble.exe test_no_lapack

.travis.yml

@@ -72,7 +72,7 @@ install:
   - choosenim $CHANNEL
 
 script:
-    - nimble refresh
+    - nimble install -y
     - if [[$BLIS]];
         then nimble test_blis;
         else nimble test;

arraymancer.nimble

@@ -40,55 +40,52 @@ srcDir = "src"
 ## TODO: auto detection or at least check in common directories
 ## Note: It is import to gate compiler flags like -march=native  behind Xcompiler "-Xcompiler -march=native"
 
-template cudaSwitches() =
-  switch("cincludes", "/opt/cuda/include")
-  switch("cc", "gcc") # We trick Nim about nvcc being gcc, pending https://github.com/nim-lang/Nim/issues/6372
-  switch("gcc.exe", "/opt/cuda/bin/nvcc")
-  switch("gcc.linkerexe", "/opt/cuda/bin/nvcc")
-  switch("gcc.cpp.exe", "/opt/cuda/bin/nvcc")
-  switch("gcc.cpp.linkerexe", "/opt/cuda/bin/nvcc")
+template cudaSwitches(switches: var string) =
+  switches.add " --cincludes:/opt/cuda/include"
+  switches.add " --cc:gcc" # We trick Nim about nvcc being gcc, pending https://github.com/nim-lang/Nim/issues/6372
+  switches.add " --gcc.exe:/opt/cuda/bin/nvcc"
+  switches.add " --gcc.linkerexe:/opt/cuda/bin/nvcc"
+  switches.add " --gcc.cpp.exe:/opt/cuda/bin/nvcc"
+  switches.add " --gcc.cpp.linkerexe:/opt/cuda/bin/nvcc"
   # Due to the __ldg intrinsics in kernels
   # we only support compute capabilities 3.5+
   # See here: http://docs.nvidia.com/cuda/pascal-compatibility-guide/index.html
   # And wikipedia for GPU capabilities: https://en.wikipedia.org/wiki/CUDA
-  switch("gcc.options.always", "-arch=sm_61 --x cu") # Interpret .c files as .cu
-  switch("gcc.cpp.options.always", "-arch=sm_61 --x cu -Xcompiler -fpermissive") # Interpret .c files as .cu, gate fpermissive behind Xcompiler
-  switch("define", "cudnn")
-
-when defined(cuda):
-  cudaSwitches
-
-template mkl_threadedSwitches() =
-  switch("define","openmp")
-  switch("stackTrace","off")
-  switch("define","blas=mkl_intel_lp64")
-  switch("define","lapack=mkl_intel_lp64")
-  switch("clibdir", "/opt/intel/mkl/lib/intel64")
-  switch("passl", "/opt/intel/mkl/lib/intel64/libmkl_intel_lp64.a")
-  switch("passl", "-lmkl_core")
-  switch("passl", "-lmkl_gnu_thread")
-  switch("passl", "-lgomp")
-  switch("dynlibOverride","mkl_intel_lp64")
-
-template mkl_singleSwitches() =
-  switch("define","blas=mkl_intel_lp64")
-  switch("define","lapack=mkl_intel_lp64")
-  switch("clibdir", "/opt/intel/mkl/lib/intel64")
-  switch("passl", "/opt/intel/mkl/lib/intel64/libmkl_intel_lp64.a")
-  switch("passl", "-lmkl_core")
-  switch("passl", "-lmkl_sequential")
-  switch("dynlibOverride","mkl_intel_lp64")
-
-template cuda_mkl_openmp() =
-  mkl_threadedSwitches()
-  switch("cincludes", "/opt/cuda/include")
-  switch("cc", "gcc") # We trick Nim about nvcc being gcc, pending https://github.com/nim-lang/Nim/issues/6372
-  switch("gcc.exe", "/opt/cuda/bin/nvcc")
-  switch("gcc.linkerexe", "/opt/cuda/bin/nvcc")
-  switch("gcc.cpp.exe", "/opt/cuda/bin/nvcc")
-  switch("gcc.cpp.linkerexe", "/opt/cuda/bin/nvcc")
-  switch("gcc.options.always", "-arch=sm_61 --x cu -Xcompiler -fopenmp -Xcompiler -march=native")
-  switch("gcc.cpp.options.always", "-arch=sm_61 --x cu -Xcompiler -fopenmp -Xcompiler -march=native")
+  switches.add " --gcc.options.always:\"-arch=sm_61 --x cu\"" # Interpret .c files as .cu
+  switches.add " --gcc.cpp.options.always:\"-arch=sm_61 --x cu -Xcompiler -fpermissive\"" # Interpret .c files as .cu, gate fpermissive behind Xcompiler
+  switches.add " -d:cudnn"
+
+template mkl_threadedSwitches(switches: var string) =
+  switches.add " --d:openmp"
+  switches.add " --stackTrace:off"
+  switches.add " --d:blas=mkl_intel_lp64"
+  switches.add " --d:lapack=mkl_intel_lp64"
+  switches.add " --clibdir:/opt/intel/mkl/lib/intel64"
+  switches.add " --passl:/opt/intel/mkl/lib/intel64/libmkl_intel_lp64.a"
+  switches.add " --passl:-lmkl_core"
+  switches.add " --passl:-lmkl_gnu_thread"
+  switches.add " --passl:-lgomp"
+  switches.add " --dynlibOverride:mkl_intel_lp64"
+
+template mkl_singleSwitches(switches: var string) =
+  switches.add " --d:blas=mkl_intel_lp64"
+  switches.add " --d:lapack=mkl_intel_lp64"
+  switches.add " --clibdir:/opt/intel/mkl/lib/intel64"
+  switches.add " --passl:/opt/intel/mkl/lib/intel64/libmkl_intel_lp64.a"
+  switches.add " --passl:-lmkl_core"
+  switches.add " --passl:-lmkl_sequential"
+  switches.add " --dynlibOverride:mkl_intel_lp64"
+
+template cuda_mkl_openmp(switches: var string) =
+  switches.mkl_threadedSwitches()
+  switches.add " --cincludes:/opt/cuda/include"
+  switches.add " --cc:gcc" # We trick Nim about nvcc being gcc, pending https://github.com/nim-lang/Nim/issues/6372
+  switches.add " --gcc.exe:/opt/cuda/bin/nvcc"
+  switches.add " --gcc.linkerexe:/opt/cuda/bin/nvcc"
+  switches.add " --gcc.cpp.exe:/opt/cuda/bin/nvcc"
+  switches.add " --gcc.cpp.linkerexe:/opt/cuda/bin/nvcc"
+  switches.add " --gcc.options.always:\"-arch=sm_61 --x cu -Xcompiler -fopenmp -Xcompiler -march=native\""
+  switches.add " --gcc.cpp.options.always:\"-arch=sm_61 --x cu -Xcompiler -fopenmp -Xcompiler -march=native\""
 
 ########################################################
 # Optimization
@@ -105,83 +102,86 @@ template cuda_mkl_openmp() =
 ##########################################################################
 ## Testing tasks
 
-proc test(name: string, lang: string = "c") =
+proc test(name, switches = "", split = false, lang = "c") =
   if not dirExists "build":
     mkDir "build"
-  if not dirExists "nimcache":
-    mkDir "nimcache"
-  --run
-  switch("out", ("./build/" & name))
-  setCommand lang, "tests/" & name & ".nim"
+  if not split:
+    exec "nim " & lang & " -o:build/" & name & switches & " -r tests/" & name & ".nim"
+  else:
+    exec "nim " & lang & " -o:build/" & name & switches & " -r tests/_split_tests/" & name & ".nim"
 
 task all_tests, "Run all tests - Intel MKL + OpenMP + Cuda + march=native + release":
-  switch("define","cuda")
-  cuda_mkl_openmp
-  test "full_test_suite", "cpp"
+  var switches = " -d:cuda"
+  switches.cuda_mkl_openmp()
+  test "full_test_suite", switches, split=false, lang="cpp"
 
 task test, "Run all tests - Default BLAS & Lapack":
-  test "tests_cpu"
+  test "tests_tensor_part01", "", split = true
+  test "tests_tensor_part02", "", split = true
+  test "tests_tensor_part03", "", split = true
+  test "tests_tensor_part04", "", split = true
+  test "tests_tensor_part05", "", split = true
+  test "tests_cpu_remainder", "", split = true
 
 task test_no_lapack, "Run all tests - Default BLAS without lapack":
-  switch("define", "no_lapack")
-  test "tests_cpu"
+  let switch = " -d:no_lapack"
+  test "tests_tensor_part01", switch, split = true
+  test "tests_tensor_part02", switch, split = true
+  test "tests_tensor_part03", switch, split = true
+  test "tests_tensor_part04", switch, split = true
+  test "tests_tensor_part05", switch, split = true
+  test "tests_cpu_remainder", switch, split = true
 
 task test_cpp, "Run all tests - Cpp codegen":
-  test "tests_cpu", "cpp"
+  test "tests_cpu", "", split = false, "cpp"
 
 task test_cuda, "Run all tests - Cuda backend with CUBLAS and CuDNN":
-  switch("define","cuda")
-  switch("define","cudnn")
-  cudaSwitches  # Unfortunately the "switch" line doesn't also trigger
-                # the "when defined(cuda)" part of this nimble file
-                # hence the need to call cudaSwitches explicitly
-  test "tests_cuda", "cpp"
+  var switches = " -d:cuda -d:cudnn"
+  switches.cudaSwitches()
+  test "tests_cuda", switches, split = false, "cpp"
 
 task test_opencl, "Run all OpenCL backend tests":
-  switch("define", "opencl")
-  test "tests_opencl"
+  test "tests_opencl", " -d:opencl"
 
 # task test_deprecated, "Run all tests on deprecated procs":
 #  test "tests_cpu_deprecated"
 
 task test_openblas, "Run all tests - OpenBLAS":
-  switch("define","blas=openblas")
-  switch("define","lapack=openblas")
+  var switches = " -d:blas=openblas -d:lapack=openblas"
   when defined(macosx):
     ## Should work but somehow Nim doesn't find libopenblas.dylib on MacOS
-    switch("clibdir", "/usr/local/opt/openblas/lib")
-    switch("cincludes", "/usr/local/opt/openblas/include")
-  test "tests_cpu"
+    switches.add " --clibdir:/usr/local/opt/openblas/lib"
+    switches.add " --cincludes:/usr/local/opt/openblas/include"
+  test "tests_cpu", switches
 
 task test_blis, "Run all tests - BLIS":
-  switch("define","blis")
-  test "tests_cpu"
+  test "tests_cpu", " -d:blis"
 
 task test_native, "Run all tests - march=native":
-  switch("define","native")
-  test "tests_cpu"
+  test "tests_cpu", " -d:native"
 
 task test_openmp, "Run all tests - OpenMP":
-  switch("define","openmp")
-  switch("stackTrace","off") # stacktraces interfere with OpenMP
+  var switches = " -d:openmp"
+  switches.add " --stackTrace:off" # stacktraces interfere with OpenMP
   when defined(macosx): # Default compiler on Mac is clang without OpenMP and gcc is an alias to clang.
                         # Use Homebrew GCC instead for OpenMP support. GCC (v7), must be properly linked via `brew link gcc`
-    switch("cc", "gcc")
-    switch("gcc.exe", "/usr/local/bin/gcc-7")
-    switch("gcc.linkerexe", "/usr/local/bin/gcc-7")
-  test "tests_cpu"
+    switches.add " --cc:gcc"
+    switches.add " --gcc.exe:/usr/local/bin/gcc-7"
+    switches.add " --gcc.linkerexe:/usr/local/bin/gcc-7"
+  test "tests_cpu", switches
 
 task test_mkl, "Run all tests - Intel MKL - single threaded":
-  mkl_singleSwitches
-  test "tests_cpu"
+  var switches: string
+  switches.mkl_singleSwitches()
+  test "tests_cpu", switches
 
 task test_mkl_omp, "Run all tests - Intel MKL + OpenMP":
-  mkl_threadedSwitches
-  test "tests_cpu"
+  var switches: string
+  switches.mkl_threadedSwitches()
+  test "tests_cpu", switches
 
 task test_release, "Run all tests - Release mode":
-  switch("define","release")
-  test "tests_cpu"
+  test "tests_cpu", " -d:release"
 
 task gen_doc, "Generate Arraymancer documentation":
   # TODO: Industrialize: something more robust that only check nim files (and not .DS_Store ...)

tests/_split_tests/tests_cpu_remainder.nim

@@ -0,0 +1,46 @@
+# Copyright 2017 the Arraymancer contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Tests were split to save on memory
+# (https://github.com/mratsim/Arraymancer/issues/359#issuecomment-500107895)
+
+import  ../io/test_csv,
+        ../io/test_numpy,
+        ../datasets/test_mnist,
+        ../datasets/test_imdb,
+        ../nn_primitives/test_nnp_numerical_gradient,
+        ../nn_primitives/test_nnp_convolution,
+        ../nn_primitives/test_nnp_loss,
+        ../nn_primitives/test_nnp_maxpool,
+        ../nn_primitives/test_nnp_gru,
+        ../nn_primitives/test_nnp_embedding,
+        ../autograd/test_gate_basic,
+        ../autograd/test_gate_blas,
+        ../autograd/test_gate_hadamard,
+        ../autograd/test_gate_shapeshifting,
+        ../ml/test_metrics,
+        ../test_bugtracker
+
+when not defined(windows) and not sizeof(int) == 4:
+  # STB image does not work on windows 32-bit, https://github.com/mratsim/Arraymancer/issues/358
+  import ../io/test_image
+
+when not defined(no_lapack):
+  import ../linear_algebra/test_linear_algebra,
+        ../ml/test_dimensionality_reduction,
+        ../ml/test_clustering
+
+import  ../stability_tests/test_stability_openmp,
+        # /end_to_end/examples_compile
+        ../end_to_end/examples_run

tests/_split_tests/tests_tensor_part01.nim

@@ -0,0 +1,23 @@
+# Copyright 2017-Present Mamy André-Ratsimbazafy & the Arraymancer contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Split tests to save on memory
+# (https://github.com/mratsim/Arraymancer/issues/359#issuecomment-500107895)
+
+import
+  ../tensor/test_init,
+  ../tensor/test_operators_comparison,
+  ../tensor/test_accessors,
+  ../tensor/test_accessors_slicer,
+  ../tensor/test_display

tests/_split_tests/tests_tensor_part02.nim

@@ -0,0 +1,19 @@
+# Copyright 2017-Present Mamy André-Ratsimbazafy & the Arraymancer contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Split tests to save on memory
+# (https://github.com/mratsim/Arraymancer/issues/359#issuecomment-500107895)
+
+import
+  ../tensor/test_operators_blas

tests/_split_tests/tests_tensor_part03.nim

@@ -0,0 +1,21 @@
+# Copyright 2017-Present Mamy André-Ratsimbazafy & the Arraymancer contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Split tests to save on memory
+# (https://github.com/mratsim/Arraymancer/issues/359#issuecomment-500107895)
+
+import
+  ../tensor/test_math_functions,
+  ../tensor/test_higherorder,
+  ../tensor/test_aggregate

tests/_split_tests/tests_tensor_part04.nim

@@ -0,0 +1,20 @@
+# Copyright 2017-Present Mamy André-Ratsimbazafy & the Arraymancer contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Split tests to save on memory
+# (https://github.com/mratsim/Arraymancer/issues/359#issuecomment-500107895)
+
+import
+  ../tensor/test_shapeshifting,
+  ../tensor/test_broadcasting