Clean up build scripts, build python 2.7,3.6,3.7 and numpy 1.14,1.15

.travis.yml

@@ -48,9 +48,7 @@ script:
  #- wget -O conda-build-all --quiet https://raw.githubusercontent.com/omnia-md/conda-recipes/master/conda-build-all
  - chmod +x conda-build-all
  # Select upload destination
- - export CHECK_AGAINST="--check-against omnia --check-against omnia-dev --check-against omnia-dev/label/cuda${CUDA_SHORT_VERSION}"
- - export CHECK_AGAINST="$CHECK_AGAINST --check-against omnia-dev/label/beta --check-against omnia-dev/label/betacuda${CUDA_SHORT_VERSION}"
- - export CHECK_AGAISNT="$CHECK_AGAINST --check-against omnia-dev/label/dev --check-against omnia-dev/label/devcuda${CUDA_SHORT_VERSION}"
+ - export CHECK_AGAINST="--check-against omnia-dev omnia-dev/label/cuda${CUDA_SHORT_VERSION}"
  - if [[ "${TRAVIS_PULL_REQUEST}" == "false" && "${TRAVIS_BRANCH}" == "master" ]]; then
  export UPLOAD="$CHECK_AGAINST --upload omnia-dev";
  else

appveyor.yml

@@ -55,7 +55,7 @@ install:
  - conda install -yq conda==4.3.34 conda-build==2.1.17 jinja2 anaconda-client
  - conda config --add channels omnia
  - conda config --add channels conda-forge
- - conda config --add channels omnia/label/dev
+ - conda config --add channels omnia-dev
  - powershell .\\devtools\\appveyor\\missing-headers.ps1
  # conda-build for some inane reason, effectively puts C:\cygwin\bin at the
  # front of the PATH, ahead of everything else, regardless. See
@@ -72,5 +72,5 @@ install:
 build: false
 
 test_script:
- - ps: if (($env:APPVEYOR_PULL_REQUEST_NUMBER -eq $null) -and ($env:APPVEYOR_REPO_BRANCH -eq "master")) { $env:UPLOAD = "--upload omnia --dev --force" } else { $env:UPLOAD = " " }
+ - ps: if (($env:APPVEYOR_PULL_REQUEST_NUMBER -eq $null) -and ($env:APPVEYOR_REPO_BRANCH -eq "master")) { $env:UPLOAD = "--check-against omnia-dev --upload omnia-dev --dev --force" } else { $env:UPLOAD = " " }
  - "%CMD_IN_ENV% python conda-build-all %UPLOAD% --python %CONDA_PY% *"

conda-build-all

@@ -9,7 +9,7 @@ import time
 import tarfile
 import os.path
 import argparse
-from subprocess import call, check_call, CalledProcessError
+from subprocess import call, check_call, check_output, CalledProcessError
 from functools import reduce
 from collections import defaultdict
 from distutils.spawn import find_executable
@@ -218,8 +218,14 @@ def get_bldpkg_path(m, python, numpy):
  cfg = _CondaBuildConfig()
  cfg.CONDA_PY = int(python.replace('.', ''))
  cfg.CONDA_NPY = int(numpy.replace('.', ''))
+
  m.config = cfg
- platform = m.info_index()['platform']
+ try:
+ platform = m.info_index()['platform']
+ except conda.exceptions.InvalidSpecError as e:
+ print('Error processing package: %s' % m.name())
+ raise e
+
  if platform is None:
  platform = 'noarch'
  cfg.platform = platform
@@ -249,7 +255,6 @@ def clean_builds():
 
 def upload_to_binstar(m, python, numpy, username, max_retries=5,
  force=False, dev=False):
- print('Attempting to upload package to anaconda cloud as user %s...' % username)
  filename = get_bldpkg_path(m, python, numpy)
  if not os.path.exists(filename):
  print('ERROR: File does not exist: %s' % filename)
@@ -276,14 +281,22 @@ def upload_to_binstar(m, python, numpy, username, max_retries=5,
  cmd.extend(['--label', 'main'])
 
  err = None
+ import subprocess
  for i in range(max_retries):
  try:
  print("Uploading package '%s' to anaconda user '%s'..." % (filename, username))
- check_call(cmd)
+ #check_call(cmd)
+ check_output(cmd, stderr=subprocess.STDOUT)
  return
  except CalledProcessError as e:
  err = e
  err.cmd = '[not shown]'
+ print(' stdout:\n%s' % err.stdout)
+ print(' stderr:\n%s' % err.stderr)
+ if ((type(err.output) is str) and ('Distribution already exists' in err.output)) or (b'Distribution already exists' in err.output):
+ # Flag in output but don't kill build for all packages
+ print('*** DISTRIBUTION ALREADY EXISTS FOR PACKAGE %s' % m.name())
+ return
  print("Retrying after %s seconds..." % (i+1))
  time.sleep(i+1)
 
@@ -319,6 +332,7 @@ class VersionIter(object):
 
 def required_builds(metadatas, pythons, numpys, channel_urls, verbose,
  force=False):
+ print('Getting package index from urls: %s' % list(channel_urls)) # DEBUG
  index = conda.api.get_index(
  channel_urls=list(channel_urls),
  prepend=False)
@@ -431,7 +445,7 @@ def build_package(m, python, numpy, args):
  else:
  print('Package failed to build (return code %s); will not upload.' % str(retcode))
 
- clean_builds()
+ #clean_builds()
 
  sys.stdout.flush()
  return retcode
@@ -477,8 +491,7 @@ def execute(args, p):
  print()
  sys.stdout.flush()
 
- # Generate list of unique packages to build
- to_build = list(set(required_builds(metadatas, pythons, numpys, channel_urls, verbose=args.verbose, force=args.force)))
+ to_build = list(required_builds(metadatas, pythons, numpys, channel_urls, verbose=args.verbose, force=args.force))
 
  if len(to_build) > 0:
  print('[conda-build-all] Scheduling the following builds (%s):' % platform)

devtools/docker-build.sh

@@ -7,9 +7,8 @@ set -x
 
 conda config --add channels conda-forge
 conda config --add channels omnia
-conda config --add channels omnia/label/dev
 conda config --add channels omnia-dev
-conda config --add channels omnia-dev/label/dev
+conda config --add channels omnia/label/cuda${CUDA_SHORT_VERSION}
 
 #conda install -yq conda\<=4.3.34
 #conda install -yq conda-build==2.1.17 jinja2 anaconda-client
@@ -36,16 +35,6 @@ conda config --add channels omnia-dev/label/dev
 #pythonvers=`python -c 'import sys; print sys.version[:3]'`
 #export PYTHONPATH=/opt/rh/devtoolset-2/root/usr/lib64/python$pythonvers/site-packages:/opt/rh/devtoolset-2/root/usr/lib/python$pythonvers/site-packages${PYTHONPATH:+:${PYTHONPATH}}
 
-# Enable conda
-#source /opt/docker/bin/entrypoint_source
-#for PY_BUILD_VERSION in "27" "35" "36" "37"; do
-
-# Make sure we have the appropriate channel added
-conda config --add channels omnia/label/betacuda${CUDA_SHORT_VERSION};
-conda config --add channels omnia/label/devcuda${CUDA_SHORT_VERSION};
-
-for PY_BUILD_VERSION in "37" "36" "35" "27" ; do
- /io/conda-build-all -vvv --python $PY_BUILD_VERSION --check-against omnia/label/beta --check-against omnia/label/betacuda${CUDA_SHORT_VERSION} --check-against omnia/label/dev --check-against omnia/label/devcuda${CUDA_SHORT_VERSION} --numpy "1.15" $UPLOAD -- /io/*
-done
+/io/conda-build-all -vvv --python "37,36,27" --numpy "1.15" $UPLOAD -- /io/*
 
 #mv /anaconda/conda-bld/linux-64/*tar.bz2 /io/ || true

devtools/osx-build.sh

@@ -4,17 +4,16 @@ export MACOSX_DEPLOYMENT_TARGET="10.9"
 # Clear existing locks
 rm -rf /usr/local/var/homebrew/locks
 # Update homebrew cant disable this yet, -y and --quiet do nothing
-brew update
+brew --quiet update
 
 # Install Miniconda
 curl -s -O https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh;
 bash Miniconda3-latest-MacOSX-x86_64.sh -b -p $HOME/anaconda;
 export PATH=$HOME/anaconda/bin:$PATH;
-conda config --add channels conda-forge
-conda config --add channels omnia
-conda config --add channels omnia/label/dev
-conda config --add channels omnia-dev
-conda config --add channels omnia-dev/label/dev
+conda config --add channels conda-forge;
+conda config --add channels omnia;
+conda config --add channels omnia-dev;
+conda config --add channels omnia-dev/label/cuda${CUDA_SHORT_VERSION};
 conda install -yq conda\<=4.3.34;
 #####################################################################
 # WORKAROUND FOR BUG WITH ruamel_yaml
@@ -79,11 +78,7 @@ fi;
 # Build packages
 export CUDA_SHORT_VERSION
 
-# Make sure we have the appropriate channel added
-conda config --add channels omnia/label/betacuda${CUDA_SHORT_VERSION};
-conda config --add channels omnia/label/devcuda${CUDA_SHORT_VERSION};
+# Build all support Python and numpy versions
+./conda-build-all -vvv --python "37,36,27" --numpy "1.15" $UPLOAD -- *
 
-#for PY_BUILD_VERSION in "27" "35" "36" "37"; do
-for PY_BUILD_VERSION in "37" "36" "35" "27"; do
- ./conda-build-all -vvv --python $PY_BUILD_VERSION --check-against omnia/label/beta --check-against omnia/label/betacuda${CUDA_SHORT_VERSION} --check-against omnia/label/dev --check-against omnia/label/devcuda${CUDA_SHORT_VERSION} --numpy "1.15" $UPLOAD -- *
 done

openmm/build.sh

@@ -81,18 +81,16 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then
  export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/glibc-2.14/lib
 fi
 
-# DEBUG: Needed for latest sphinx
-locale -a
+# Needed for latest sphinx
 export LC_ALL=C
 #export LC_ALL="en_US.UTF-8"
 #export LC_CTYPE="en_US.UTF-8"
-locale -a
 #sudo dpkg-reconfigure locales
 
 # Build PDF manuals
-make -j$CPU_COUNT sphinxpdf
-mv sphinx-docs/userguide/latex/*.pdf $PREFIX/docs/openmm/
-mv sphinx-docs/developerguide/latex/*.pdf $PREFIX/docs/openmm/
+#make -j$CPU_COUNT sphinxpdf
+#mv sphinx-docs/userguide/latex/*.pdf $PREFIX/docs/openmm/
+#mv sphinx-docs/developerguide/latex/*.pdf $PREFIX/docs/openmm/
 
 # Put examples into an appropriate subdirectory.
 mkdir $PREFIX/share/openmm/