Improved the way requirements are handled, ensuring that pip gets all…

… the information it needs.

.travis.yml

@@ -55,14 +55,25 @@ install:
   # Customise the testing environment
   # ---------------------------------
   - conda config --add channels conda-forge
-  - if [[ "$TEST_MINIMAL" == true ]]; then
-      conda install --quiet --file minimal-conda-requirements.txt;
+  - >
+    if [[ "$TEST_MINIMAL" == true ]]; then
+      CONDA_REQS_FILE=conda-requirements.txt
+      if [[ "$TRAVIS_PYTHON_VERSION" == 3* ]]; then
+          python requirements/gen_conda_requirements.py --groups test > ${CONDA_REQS_FILE};
+      else
+          python requirements/gen_conda_requirements.py --py2 --groups test > ${CONDA_REQS_FILE};
+      fi
+      cat ${CONDA_REQS_FILE};
+      conda install --quiet --file ${CONDA_REQS_FILE};
     else
+      CONDA_REQS_FILE=conda-requirements.txt
       if [[ "$TRAVIS_PYTHON_VERSION" == 3* ]]; then
-        sed -e '/esmpy/d' -e 's/#.\+$//' conda-requirements.txt | xargs conda install --quiet;
+          python requirements/gen_conda_requirements.py --groups test full > ${CONDA_REQS_FILE};
       else
-        conda install --quiet --file conda-requirements.txt;
+          python requirements/gen_conda_requirements.py --py2 --groups test full > ${CONDA_REQS_FILE};
       fi
+      cat ${CONDA_REQS_FILE};
+      conda install --quiet --file ${CONDA_REQS_FILE};
     fi
 
   # JUST FOR NOW : Install latest version of iris-grib.

requirements/core.txt

@@ -0,0 +1,12 @@
+# Absolute minimal dependencies for iris
+# --------------------------------------
+
+# Without these, iris won't even import.
+
+cartopy
+matplotlib<1.9
+netcdf4
+numpy
+pyke
+cf_units
+dask>=0.15.0

requirements/full.txt

@@ -0,0 +1,20 @@
+# Dependencies for a feature complete installation
+# ------------------------------------------------
+
+esmpy>=7.0;python_version<"3"  #conda: esmpy>=7.0  (only python=2)
+gdal
+mo_pack
+nc_time_axis
+pandas
+stratify  #conda: python-stratify
+pyugrid
+
+
+# Iris extensions (i.e. key tools that depend on Iris)
+# ----------------------------------------------------
+
+# Note: pip can handle the circularity of these extensions, but conda will
+# struggle. To install these extensions, ensure iris[core] has been installed
+# first.
+
+iris_grib;python_version<"3"  #conda: iris_grib  (only python=2)

requirements/gen_conda_requirements.py

@@ -0,0 +1,72 @@
+import argparse
+import os.path
+
+
+REQS_DIR = os.path.dirname(__file__)
+CONDA_PATTERN = '#conda:'
+
+
+def read_conda_reqs(fname, options):
+    lines = []
+    with open(fname, 'r') as fh:
+        for line in fh:
+            line = line.strip()
+            if '#conda:' in line:
+                line_start = line.index(CONDA_PATTERN) + len(CONDA_PATTERN)
+                line = line[line_start:].strip()
+                if 'only python=2' in line:
+                    if 'python=2' in options:
+                        line = line.replace('(only python=2)', '')
+                        lines.append(line)
+                    else:
+                        continue
+            else:
+                lines.append(line)
+    return lines
+
+
+def compute_requirements(requirement_names=('core', ), options=None):
+    conda_reqs_lines = []
+
+    if 'python=2' in options:
+        conda_reqs_lines.append('python=2.*')
+    else:
+        conda_reqs_lines.append('# Python 3 conda configuration')
+
+    for req_name in requirement_names:
+        fname = os.path.join(REQS_DIR, '{}.txt'.format(req_name))
+        if not os.path.exists(fname):
+            raise RuntimeError('Unable to find the requirements file for {} '
+                               'in {}'.format(req_name, fname))
+        conda_reqs_lines.extend(read_conda_reqs(fname, options))
+        conda_reqs_lines.append('')
+
+    return conda_reqs_lines
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--verbosity", help="increase output verbosity")
+    parser.add_argument(
+            "--groups", nargs='*', default=[],
+            help=("Gather requirements for these given named groups "
+                  "(as found in the requirements/ folder)"))
+    parser.add_argument(
+            "--py2", action="store_true",
+            help="Build the conda requirements for a python 2 installation")
+
+    args = parser.parse_args()
+
+    requirement_names = args.groups
+    requirement_names.insert(0, 'core')
+    requirement_names.insert(0, 'setup')
+
+    options = []
+    if args.py2:
+        options.append('python=2')
+
+    print('\n'.join(compute_requirements(requirement_names, options)))
+
+
+if __name__ == '__main__':
+    main()

requirements/setup.txt

@@ -0,0 +1,5 @@
+# Dependencies necessary to run setup.py of iris
+# ----------------------------------------------
+
+setuptools
+numpy

requirements/test.txt

@@ -0,0 +1,9 @@
+# Dependencies needed to run the iris tests
+#------------------------------------------
+
+mock
+nose
+pep8
+filelock
+imagehash
+requests

setup.py

@@ -62,6 +62,22 @@ def std_name_cmd(target_dir):
     cmd = (sys.executable, script_path, xml_path, module_path)
     return cmd
 
+SETUP_DIR = os.path.dirname(__file__)
+
+def requirements(name):
+    fname = os.path.join(SETUP_DIR, 'requirements', '{}.txt'.format(name))
+    if not os.path.exists(fname):
+        raise RuntimeError('Unable to find the {} requirements file at {}'
+                           ''.format(name, fname))
+    reqs = []
+    with open(fname, 'r') as fh:
+        for line in fh:
+            line = line.strip()
+            if not line or line.startswith('#'):
+                continue
+            reqs.append(line)
+    return reqs
+
 
 class SetupTestRunner(TestRunner, setuptools.Command):
     pass
@@ -182,8 +198,7 @@ def run(self):
 
 def extract_version():
     version = None
-    fdir = os.path.dirname(__file__)
-    fnme = os.path.join(fdir, 'lib', 'iris', '__init__.py')
+    fnme = os.path.join(SETUP_DIR, 'lib', 'iris', '__init__.py')
     with open(fnme) as fd:
         for line in fd:
             if (line.startswith('__version__')):
@@ -193,8 +208,11 @@ def extract_version():
     return version
 
 
+pypi_name = 'scitools-iris'
+
+
 setup(
-    name='Iris',
+    name=pypi_name,
     version=extract_version(),
     url='http://scitools.org.uk/iris/',
     author='UK Met Office',
@@ -208,8 +226,14 @@ def extract_version():
                 ['fileformats/_pyke_rules/*.k?b'] + \
                 ['tests/stock*.npz']
         },
+    setup_requires=requirements('setup'),
+    install_requires=requirements('core'),
+    tests_require=['{}[test]'.format(pypi_name)],
+    extras_require = {
+                      'test': requirements('test'),
+                      'all': requirements('full'),
+                      },
     data_files=[('iris', ['CHANGES', 'COPYING', 'COPYING.LESSER'])],
-    tests_require=['nose'],
     cmdclass={'test': SetupTestRunner, 'build_py': BuildPyWithExtras,
               'std_names': MakeStdNames, 'pyke_rules': MakePykeRules,
               'clean_source': CleanSource},