BUG: Travis building on container-based infrastructure

closes #10598

Author: Neil Parley <mail@neilparley.co.uk>

Closes #12946 from nparley/travis and squashes the following commits:

Author: nparley

.travis.yml

@@ -1,14 +1,24 @@
-
+sudo: false
 language: python
 
-env:
+# To turn off cached miniconda, cython files and compiler cache comment out the
+# USE_CACHE=true line for the build in the matrix below. To delete caches go to
+# https://travis-ci.org/OWNER/REPOSITORY/caches or run
+# travis cache --delete inside the project directory from the travis command line client
+# The cash directories will be deleted if anything in ci/ changes in a commit
+cache:
+ directories:
+  - $HOME/miniconda # miniconda cache
+  - $HOME/.cache # cython cache
+  - $HOME/.ccache # compiler cache
 
+env:
   global:
   # scatterci API key
   #- secure: "Bx5umgo6WjuGY+5XFa004xjCiX/vq0CyMZ/ETzcs7EIBI1BE/0fIDXOoWhoxbY9HPfdPGlDnDgB9nGqr5wArO2s+BavyKBWg6osZ3dmkfuJPMOWeyCa92EeP+sfKw8e5HSU5MizW9e319wHWOF/xkzdHR7T67Qd5erhv91x4DnQ="
   # ironcache API key
-  - secure: "e4eEFn9nDQc3Xa5BWYkzfX37jaWVq89XidVX+rcCNEr5OlOImvveeXnF1IzbRXznH4Sv0YsLwUd8RGUWOmyCvkONq/VJeqCHWtTMyfaCIdqSyhIP9Odz8r9ahch+Y0XFepBey92AJHmlnTh+2GjCDgIiqq4fzglojnp56Vg1ojA="
-  - secure: "CjmYmY5qEu3KrvMtel6zWFEtMq8ORBeS1S1odJHnjQpbwT1KY2YFZRVlLphfyDQXSz6svKUdeRrCNp65baBzs3DQNA8lIuXGIBYFeJxqVGtYAZZs6+TzBPfJJK798sGOj5RshrOJkFG2rdlWNuTq/XphI0JOrN3nPUkRrdQRpAw="
+  #- secure: "e4eEFn9nDQc3Xa5BWYkzfX37jaWVq89XidVX+rcCNEr5OlOImvveeXnF1IzbRXznH4Sv0YsLwUd8RGUWOmyCvkONq/VJeqCHWtTMyfaCIdqSyhIP9Odz8r9ahch+Y0XFepBey92AJHmlnTh+2GjCDgIiqq4fzglojnp56Vg1ojA="
+  #- secure: "CjmYmY5qEu3KrvMtel6zWFEtMq8ORBeS1S1odJHnjQpbwT1KY2YFZRVlLphfyDQXSz6svKUdeRrCNp65baBzs3DQNA8lIuXGIBYFeJxqVGtYAZZs6+TzBPfJJK798sGOj5RshrOJkFG2rdlWNuTq/XphI0JOrN3nPUkRrdQRpAw="
   # pandas-docs-bot GH
   - secure: "PCzUFR8CHmw9lH84p4ygnojdF7Z8U5h7YfY0RyT+5K/aiQ1ZTU3ZkDTPI0/rR5FVMxsEEKEQKMcc5fvqW0PeD7Q2wRmluloKgT9w4EVEJ1ppKf7lITPcvZR2QgVOvjv4AfDtibLHFNiaSjzoqyJVjM4igjOu8WTlF3JfZcmOQjQ="
 
@@ -29,112 +39,194 @@ matrix:
         - BUILD_TYPE=conda
         - JOB_TAG=_OSX
         - TRAVIS_PYTHON_VERSION=3.5
+        - CACHE_NAME="35_osx"
+        - USE_CACHE=true
     - python: 2.7
       env:
-      - JOB_NAME: "27_slow_nnet_LOCALE"
-      - NOSE_ARGS="slow and not network and not disabled"
-      - LOCALE_OVERRIDE="zh_CN.GB18030"
-      - FULL_DEPS=true
-      - JOB_TAG=_LOCALE
+        - JOB_NAME: "27_slow_nnet_LOCALE"
+        - NOSE_ARGS="slow and not network and not disabled"
+        - LOCALE_OVERRIDE="zh_CN.UTF-8"
+        - FULL_DEPS=true
+        - JOB_TAG=_LOCALE
+        - CACHE_NAME="27_slow_nnet_LOCALE"
+        - USE_CACHE=true
+      addons:
+        apt:
+          packages:
+          - language-pack-zh-hans
     - python: 2.7
       env:
-      - JOB_NAME: "27_nslow"
-      - NOSE_ARGS="not slow and not disabled"
-      - FULL_DEPS=true
-      - CLIPBOARD_GUI=gtk2
-      - LINT=true
+        - JOB_NAME: "27_nslow"
+        - NOSE_ARGS="not slow and not disabled"
+        - FULL_DEPS=true
+        - CLIPBOARD_GUI=gtk2
+        - LINT=true
+        - CACHE_NAME="27_nslow"
+        - USE_CACHE=true
+      addons:
+        apt:
+          packages:
+          - python-gtk2
     - python: 3.4
       env:
-      - JOB_NAME: "34_nslow"
-      - NOSE_ARGS="not slow and not disabled"
-      - FULL_DEPS=true
-      - CLIPBOARD=xsel
+        - JOB_NAME: "34_nslow"
+        - NOSE_ARGS="not slow and not disabled"
+        - FULL_DEPS=true
+        - CLIPBOARD=xsel
+        - CACHE_NAME="34_nslow"
+        - USE_CACHE=true
+      addons:
+        apt:
+          packages:
+          - xsel
     - python: 3.5
       env:
-      - JOB_NAME: "35_nslow"
-      - NOSE_ARGS="not slow and not network and not disabled"
-      - FULL_DEPS=true
-      - CLIPBOARD=xsel
-      - COVERAGE=true
+        - JOB_NAME: "35_nslow"
+        - NOSE_ARGS="not slow and not network and not disabled"
+        - FULL_DEPS=true
+        - CLIPBOARD=xsel
+        - COVERAGE=true
+        - CACHE_NAME="35_nslow"
+#        - USE_CACHE=true # Don't use cache for 35_nslow
+      addons:
+        apt:
+          packages:
+          - xsel
+#    In allow_failures
     - python: 2.7
       env:
-      - JOB_NAME: "27_slow"
-      - JOB_TAG=_SLOW
-      - NOSE_ARGS="slow and not network and not disabled"
-      - FULL_DEPS=true
+        - JOB_NAME: "27_slow"
+        - JOB_TAG=_SLOW
+        - NOSE_ARGS="slow and not network and not disabled"
+        - FULL_DEPS=true
+        - CACHE_NAME="27_slow"
+        - USE_CACHE=true
+#    In allow_failures
     - python: 3.4
       env:
-      - JOB_NAME: "34_slow"
-      - JOB_TAG=_SLOW
-      - NOSE_ARGS="slow and not network and not disabled"
-      - FULL_DEPS=true
-      - CLIPBOARD=xsel
+        - JOB_NAME: "34_slow"
+        - JOB_TAG=_SLOW
+        - NOSE_ARGS="slow and not network and not disabled"
+        - FULL_DEPS=true
+        - CLIPBOARD=xsel
+        - CACHE_NAME="34_slow"
+        - USE_CACHE=true
+      addons:
+        apt:
+          packages:
+          - xsel
+#    In allow_failures
     - python: 2.7
       env:
-      - JOB_NAME: "27_build_test_conda"
-      - JOB_TAG=_BUILD_TEST
-      - NOSE_ARGS="not slow and not disabled"
-      - FULL_DEPS=true
-      - BUILD_TEST=true
+        - JOB_NAME: "27_build_test_conda"
+        - JOB_TAG=_BUILD_TEST
+        - NOSE_ARGS="not slow and not disabled"
+        - FULL_DEPS=true
+        - BUILD_TEST=true
+        - CACHE_NAME="27_build_test_conda"
+        - USE_CACHE=true
+#    In allow_failures
     - python: 3.5
       env:
-      - JOB_NAME: "35_numpy_dev"
-      - JOB_TAG=_NUMPY_DEV
-      - NOSE_ARGS="not slow and not network and not disabled"
-      - PANDAS_TESTING_MODE="deprecate"
+        - JOB_NAME: "35_numpy_dev"
+        - JOB_TAG=_NUMPY_DEV
+        - NOSE_ARGS="not slow and not network and not disabled"
+        - PANDAS_TESTING_MODE="deprecate"
+        - CACHE_NAME="35_numpy_dev"
+        - USE_CACHE=true
+      addons:
+        apt:
+          packages:
+          - libatlas-base-dev
+          - gfortran
+#    In allow_failures
     - python: 2.7
       env:
-      - JOB_NAME: "27_nslow_nnet_COMPAT"
-      - NOSE_ARGS="not slow and not network and not disabled"
-      - LOCALE_OVERRIDE="it_IT.UTF-8"
-      - INSTALL_TEST=true
-      - JOB_TAG=_COMPAT
+        - JOB_NAME: "27_nslow_nnet_COMPAT"
+        - NOSE_ARGS="not slow and not network and not disabled"
+        - LOCALE_OVERRIDE="it_IT.UTF-8"
+        - INSTALL_TEST=true
+        - JOB_TAG=_COMPAT
+        - CACHE_NAME="27_nslow_nnet_COMPAT"
+        - USE_CACHE=true
+      addons:
+        apt:
+          packages:
+          - language-pack-it
+#    In allow_failures
     - python: 2.7
       env:
-      - JOB_NAME: "doc_build"
-      - FULL_DEPS=true
-      - DOC_BUILD=true # if rst files were changed, build docs in parallel with tests
-      - JOB_TAG=_DOC_BUILD
+        - JOB_NAME: "doc_build"
+        - FULL_DEPS=true
+        - DOC_BUILD=true
+        - JOB_TAG=_DOC_BUILD
+        - CACHE_NAME="doc_build"
+        - USE_CACHE=true
     allow_failures:
       - python: 2.7
         env:
         - JOB_NAME: "27_slow"
         - JOB_TAG=_SLOW
         - NOSE_ARGS="slow and not network and not disabled"
         - FULL_DEPS=true
+        - CACHE_NAME="27_slow"
+        - USE_CACHE=true
       - python: 3.4
         env:
         - JOB_NAME: "34_slow"
         - JOB_TAG=_SLOW
         - NOSE_ARGS="slow and not network and not disabled"
         - FULL_DEPS=true
         - CLIPBOARD=xsel
+        - CACHE_NAME="34_slow"
+        - USE_CACHE=true
+        addons:
+          apt:
+            packages:
+            - xsel
       - python: 2.7
         env:
         - JOB_NAME: "27_build_test_conda"
         - JOB_TAG=_BUILD_TEST
         - NOSE_ARGS="not slow and not disabled"
         - FULL_DEPS=true
         - BUILD_TEST=true
+        - CACHE_NAME="27_build_test_conda"
+        - USE_CACHE=true
       - python: 3.5
         env:
         - JOB_NAME: "35_numpy_dev"
         - JOB_TAG=_NUMPY_DEV
         - NOSE_ARGS="not slow and not network and not disabled"
         - PANDAS_TESTING_MODE="deprecate"
+        - CACHE_NAME="35_numpy_dev"
+        - USE_CACHE=true
+        addons:
+          apt:
+            packages:
+            - libatlas-base-dev
+            - gfortran
       - python: 2.7
         env:
         - JOB_NAME: "27_nslow_nnet_COMPAT"
         - NOSE_ARGS="not slow and not network and not disabled"
         - LOCALE_OVERRIDE="it_IT.UTF-8"
         - INSTALL_TEST=true
         - JOB_TAG=_COMPAT
+        - CACHE_NAME="27_nslow_nnet_COMPAT"
+        - USE_CACHE=true
+        addons:
+          apt:
+            packages:
+            - language-pack-it
       - python: 2.7
         env:
         - JOB_NAME: "doc_build"
         - FULL_DEPS=true
         - DOC_BUILD=true
         - JOB_TAG=_DOC_BUILD
+        - CACHE_NAME="doc_build"
+        - USE_CACHE=true
 
 before_install:
   - echo "before_install"
@@ -153,9 +245,10 @@ before_install:
 
 install:
   - echo "install start"
-  - ci/prep_ccache.sh
+  - ci/check_cache.sh
+  - ci/prep_cython_cache.sh
   - ci/install_travis.sh
-  - ci/submit_ccache.sh
+  - ci/submit_cython_cache.sh
   - echo "install done"
 
 before_script:
@@ -175,6 +268,6 @@ after_success:
 after_script:
   - echo "after_script start"
   - ci/install_test.sh
-  - source activate pandas && ci/print_versions.py
+  - source activate pandas && python -c "import pandas; pandas.show_versions();"
   - ci/print_skipped.py /tmp/nosetests.xml
   - echo "after_script done"

ci/before_install_travis.sh

@@ -9,8 +9,6 @@ echo "inside $0"
 
 # overview
 if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
-   sudo apt-get update $APT_ARGS # run apt-get update for all versions
-
    sh -e /etc/init.d/xvfb start
 fi
 

ci/check_cache.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+if [ "$TRAVIS_PULL_REQUEST" == "false" ]
+then
+    echo "Not a PR: checking for changes in ci/ from last 2 commits"
+    git diff HEAD~2 --numstat | grep -E "ci/"
+    ci_changes=$(git diff HEAD~2 --numstat | grep -E "ci/"| wc -l)
+else
+    echo "PR: checking for changes in ci/ from last 2 commits"
+    git fetch origin pull/${TRAVIS_PULL_REQUEST}/head:PR_HEAD
+    git diff PR_HEAD~2 --numstat | grep -E "ci/"
+    ci_changes=$(git diff PR_HEAD~2 --numstat | grep -E "ci/"| wc -l)
+fi
+
+MINICONDA_DIR="$HOME/miniconda/"
+CACHE_DIR="$HOME/.cache/"
+CCACHE_DIR="$HOME/.ccache/"
+
+if [ $ci_changes -ne 0 ]
+then
+    echo "Files have changed in ci/ deleting all caches"
+    rm -rf "$MINICONDA_DIR"
+    rm -rf "$CACHE_DIR"
+    rm -rf "$CCACHE_DIR"
+fi

ci/install-2.7_NUMPY_DEV.sh

@@ -12,8 +12,6 @@ pip uninstall numpy -y
 # these wheels don't play nice with the conda libgfortran / openblas
 # time conda install -n pandas libgfortran openblas || exit 1
 
-time sudo apt-get $APT_ARGS install libatlas-base-dev gfortran
-
 # install numpy wheel from master
 pip install --pre --upgrade --no-index --timeout=60 --trusted-host travis-dev-wheels.scipy.org -f http://travis-dev-wheels.scipy.org/ numpy
 

ci/install-3.5_NUMPY_DEV.sh

@@ -12,8 +12,6 @@ pip uninstall numpy -y
 # these wheels don't play nice with the conda libgfortran / openblas
 # time conda install -n pandas libgfortran openblas || exit 1
 
-time sudo apt-get $APT_ARGS install libatlas-base-dev gfortran
-
 # install numpy wheel from master
 pip install --pre --upgrade --no-index --timeout=60 --trusted-host travis-dev-wheels.scipy.org -f http://travis-dev-wheels.scipy.org/ numpy scipy