From 509634f7fe61d29a4c7168b5e081f6fee1d60952 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Wed, 1 Sep 2021 11:18:51 +0100 Subject: [PATCH 01/12] Fix broken ABF link --- docs/src/whatsnew/1.3.rst | 4 ++-- lib/iris/fileformats/abf.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/whatsnew/1.3.rst b/docs/src/whatsnew/1.3.rst index beaa594ab5..1895711379 100644 --- a/docs/src/whatsnew/1.3.rst +++ b/docs/src/whatsnew/1.3.rst @@ -34,8 +34,8 @@ Loading ABF/ABL Files --------------------- Support for the ABF and ABL file formats (as -`defined `_ by the -climate and vegetation research group of Boston University), is +`defined `_ +by the climate and vegetation research group of Boston University), is currently provided under the "experimental" system. As such, ABF/ABL file detection is not automatically enabled. diff --git a/lib/iris/fileformats/abf.py b/lib/iris/fileformats/abf.py index c031bfb415..678d9b04cf 100644 --- a/lib/iris/fileformats/abf.py +++ b/lib/iris/fileformats/abf.py @@ -10,7 +10,7 @@ Including this module adds ABF and ABL loading to the session's capabilities. The documentation for this file format can be found -`here `_. +`here `_. """ From c611e0d41e20ec2fa2e95b54d1eab981c8abcb53 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Thu, 2 Sep 2021 21:20:53 +0100 Subject: [PATCH 02/12] add cartopy downloader utility --- .cirrus.yml | 8 ++- tools/cartopy_feature_download.py | 95 +++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100755 tools/cartopy_feature_download.py diff --git a/.cirrus.yml b/.cirrus.yml index 839ef7fc77..49e5380f32 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -26,7 +26,7 @@ env: # Maximum cache period (in weeks) before forcing a new cache upload. CACHE_PERIOD: "2" # Increment the build number to force new cartopy cache upload. - CARTOPY_CACHE_BUILD: "0" + CARTOPY_CACHE_BUILD: "1" # Increment the build number to force new conda cache upload. CONDA_CACHE_BUILD: "0" # Increment the build number to force new nox cache upload. @@ -72,6 +72,12 @@ linux_task_template: &LINUX_TASK_TEMPLATE fingerprint_script: - echo "${CIRRUS_OS}" - echo "$(date +%Y).$(expr $(date +%U) / ${CACHE_PERIOD}):${CARTOPY_CACHE_BUILD}" + populate_script: + - conda create --quiet --name cartopy-cache cartopy + - source ${HOME}/miniconda/etc/profile.d/conda.sh >/dev/null 2>&1 + - conda activate cartopy-cache >/dev/null 2>&1 + - python tools/cartopy_feature_download.py --output ${HOME}/.local/share/cartopy --nowarn + - conda deactivate >/dev/null 2>&1 nox_cache: folder: ${CIRRUS_WORKING_DIR}/.nox reupload_on_changes: true diff --git a/tools/cartopy_feature_download.py b/tools/cartopy_feature_download.py new file mode 100755 index 0000000000..b996739082 --- /dev/null +++ b/tools/cartopy_feature_download.py @@ -0,0 +1,95 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +A command line utility for downloading cartopy resources e.g., + + > python cartopy_feature_download.py --help + + +""" + +import argparse +import os +import pathlib +import sys +import tempfile +import urllib.request + +import cartopy +from cartopy import config + +FEATURE_DOWNLOAD_URL = f"https://raw.githubusercontent.com/SciTools/cartopy/v{cartopy.__version__}/tools/feature_download.py" +# This will be the (more stable) cartopy resource endpoint from v0.19.0.post1+ +URL_TEMPLATE = "https://naturalearth.s3.amazonaws.com/{resolution}_{category}/ne_{resolution}_{name}.zip" +SHP_NE_SPEC = ("shapefiles", "natural_earth") + + +def main(target_dir, features, dry_run): + target_dir = pathlib.Path(target_dir).expanduser().resolve() + target_dir.mkdir(parents=True, exist_ok=True) + + with tempfile.TemporaryDirectory() as tmpdir: + cwd = pathlib.Path.cwd() + os.chdir(tmpdir) + + # Download cartopy feature_download tool, which is not bundled + # within a cartopy package, and make it importable. + urllib.request.urlretrieve(FEATURE_DOWNLOAD_URL, "feature_download.py") + with open("__init__.py", "w"): + pass + sys.path.append(tmpdir) + + from feature_download import download_features + + # Configure the cartopy resource cache. + config["pre_existing_data_dir"] = str(target_dir) + config["data_dir"] = str(target_dir) + config["repo_data_dir"] = str(target_dir) + config["downloaders"][SHP_NE_SPEC].url_template = URL_TEMPLATE + + # Perform download, or dry-run. + download_features(features, dry_run=dry_run) + os.chdir(cwd) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Download cartopy data for caching." + ) + parser.add_argument( + "--dryrun", + "-d", + action="store_true", + help="perform a dry-run of the download", + ) + parser.add_argument( + "--feature", + "-f", + nargs="+", + default=["physical"], + help="list of one or more features to download", + ) + parser.add_argument( + "--nowarn", + "-n", + action="store_true", + help="ignore cartopy DownloadWarning warnings", + ) + parser.add_argument( + "--output", + "-o", + required=True, + help="save datasets in the specified directory", + ) + + args = parser.parse_args() + + if args.nowarn: + import warnings + + warnings.filterwarnings("ignore", category=cartopy.io.DownloadWarning) + + main(args.output, args.feature, args.dryrun) From 5e72e2bafa67a7d60847ca75527632a2d764b5cd Mon Sep 17 00:00:00 2001 From: Bill Little Date: Thu, 2 Sep 2021 21:36:07 +0100 Subject: [PATCH 03/12] update netcdf4-python link --- lib/iris/fileformats/netcdf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/iris/fileformats/netcdf.py b/lib/iris/fileformats/netcdf.py index 60da5c5e97..9dd8848f7f 100644 --- a/lib/iris/fileformats/netcdf.py +++ b/lib/iris/fileformats/netcdf.py @@ -6,7 +6,7 @@ """ Module to support the loading of a NetCDF file into an Iris cube. -See also: `netCDF4 python `_. +See also: `netCDF4 python `_ Also refer to document 'NetCDF Climate and Forecast (CF) Metadata Conventions'. From 9d057453bdf281e7b1eab106404b88036e4fb013 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Thu, 2 Sep 2021 21:54:46 +0100 Subject: [PATCH 04/12] explicitly specify features to download for ci --- .cirrus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 49e5380f32..bfd0747586 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -76,7 +76,7 @@ linux_task_template: &LINUX_TASK_TEMPLATE - conda create --quiet --name cartopy-cache cartopy - source ${HOME}/miniconda/etc/profile.d/conda.sh >/dev/null 2>&1 - conda activate cartopy-cache >/dev/null 2>&1 - - python tools/cartopy_feature_download.py --output ${HOME}/.local/share/cartopy --nowarn + - python tools/cartopy_feature_download.py --output ${HOME}/.local/share/cartopy --feature physical --nowarn - conda deactivate >/dev/null 2>&1 nox_cache: folder: ${CIRRUS_WORKING_DIR}/.nox From ddcb95f08de3ca5540fdc518aff3c2a6a114942e Mon Sep 17 00:00:00 2001 From: Bill Little Date: Thu, 2 Sep 2021 22:12:37 +0100 Subject: [PATCH 05/12] added whatsnew entry --- docs/src/common_links.inc | 1 + docs/src/whatsnew/latest.rst | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/src/common_links.inc b/docs/src/common_links.inc index d59f98de47..e22b37c87e 100644 --- a/docs/src/common_links.inc +++ b/docs/src/common_links.inc @@ -2,6 +2,7 @@ Common resources in alphabetical order: .. _black: https://black.readthedocs.io/en/stable/ +.. _cartopy: https://github.com/SciTools/cartopy .. _.cirrus.yml: https://github.com/SciTools/iris/blob/main/.cirrus.yml .. _flake8: https://flake8.pycqa.org/en/stable/ .. _.flake8.yml: https://github.com/SciTools/iris/blob/main/.flake8 diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index f85c001453..f96b8f1990 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -31,7 +31,9 @@ This document explains the changes made to Iris for this release ✨ Features =========== -#. N/A +#. `@bjlittle`_ added the `tools/cartopy_feature_download.py` command-line utility + which allows users to easily download feature resources for `cartopy`_. + (:pull:`4304`) 🐛 Bugs Fixed From ac6697ce1d6e3008ee8078eb2b9d725d0f61ce80 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Thu, 2 Sep 2021 22:42:28 +0100 Subject: [PATCH 06/12] update utility help --- tools/cartopy_feature_download.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/cartopy_feature_download.py b/tools/cartopy_feature_download.py index b996739082..9285bf82d2 100755 --- a/tools/cartopy_feature_download.py +++ b/tools/cartopy_feature_download.py @@ -70,7 +70,10 @@ def main(target_dir, features, dry_run): "-f", nargs="+", default=["physical"], - help="list of one or more features to download", + help=( + "specify one or more features to download [cultural|cultural-extra|gshhs|physical], " + 'default is "physical"' + ), ) parser.add_argument( "--nowarn", From 81481ed1ae09189e704a4ac847e098701b2f8c54 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Fri, 3 Sep 2021 09:12:09 +0100 Subject: [PATCH 07/12] utility tidy --- tools/cartopy_feature_download.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) mode change 100755 => 100644 tools/cartopy_feature_download.py diff --git a/tools/cartopy_feature_download.py b/tools/cartopy_feature_download.py old mode 100755 new mode 100644 index 9285bf82d2..ef4799ad97 --- a/tools/cartopy_feature_download.py +++ b/tools/cartopy_feature_download.py @@ -23,6 +23,7 @@ FEATURE_DOWNLOAD_URL = f"https://raw.githubusercontent.com/SciTools/cartopy/v{cartopy.__version__}/tools/feature_download.py" # This will be the (more stable) cartopy resource endpoint from v0.19.0.post1+ +# See https://github.com/SciTools/cartopy/pull/1833 URL_TEMPLATE = "https://naturalearth.s3.amazonaws.com/{resolution}_{category}/ne_{resolution}_{name}.zip" SHP_NE_SPEC = ("shapefiles", "natural_earth") @@ -30,9 +31,9 @@ def main(target_dir, features, dry_run): target_dir = pathlib.Path(target_dir).expanduser().resolve() target_dir.mkdir(parents=True, exist_ok=True) + cwd = pathlib.Path.cwd() with tempfile.TemporaryDirectory() as tmpdir: - cwd = pathlib.Path.cwd() os.chdir(tmpdir) # Download cartopy feature_download tool, which is not bundled @@ -48,10 +49,12 @@ def main(target_dir, features, dry_run): config["pre_existing_data_dir"] = str(target_dir) config["data_dir"] = str(target_dir) config["repo_data_dir"] = str(target_dir) + # Force use of stable endpoint for pre-v0.20 cartopy. config["downloaders"][SHP_NE_SPEC].url_template = URL_TEMPLATE # Perform download, or dry-run. download_features(features, dry_run=dry_run) + os.chdir(cwd) From 6c5ebfc875af3a0d62af8649aa12b426e3efb5cf Mon Sep 17 00:00:00 2001 From: Bill Little Date: Fri, 3 Sep 2021 10:39:31 +0100 Subject: [PATCH 08/12] tidy utility --- tools/cartopy_feature_download.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/cartopy_feature_download.py b/tools/cartopy_feature_download.py index ef4799ad97..8cebc86354 100644 --- a/tools/cartopy_feature_download.py +++ b/tools/cartopy_feature_download.py @@ -39,8 +39,6 @@ def main(target_dir, features, dry_run): # Download cartopy feature_download tool, which is not bundled # within a cartopy package, and make it importable. urllib.request.urlretrieve(FEATURE_DOWNLOAD_URL, "feature_download.py") - with open("__init__.py", "w"): - pass sys.path.append(tmpdir) from feature_download import download_features From f4af47b3633b3e6855f6118c556fcb760ec6f2e6 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Fri, 3 Sep 2021 11:32:37 +0100 Subject: [PATCH 09/12] copy with cartopy script rename --- tools/cartopy_feature_download.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tools/cartopy_feature_download.py b/tools/cartopy_feature_download.py index 8cebc86354..7bd5ec79f0 100644 --- a/tools/cartopy_feature_download.py +++ b/tools/cartopy_feature_download.py @@ -21,7 +21,7 @@ import cartopy from cartopy import config -FEATURE_DOWNLOAD_URL = f"https://raw.githubusercontent.com/SciTools/cartopy/v{cartopy.__version__}/tools/feature_download.py" +FEATURE_DOWNLOAD_URL = f"https://raw.githubusercontent.com/SciTools/cartopy/v{cartopy.__version__}/tools/{{name}}.py" # This will be the (more stable) cartopy resource endpoint from v0.19.0.post1+ # See https://github.com/SciTools/cartopy/pull/1833 URL_TEMPLATE = "https://naturalearth.s3.amazonaws.com/{resolution}_{category}/ne_{resolution}_{name}.zip" @@ -38,10 +38,21 @@ def main(target_dir, features, dry_run): # Download cartopy feature_download tool, which is not bundled # within a cartopy package, and make it importable. - urllib.request.urlretrieve(FEATURE_DOWNLOAD_URL, "feature_download.py") + script = "download.py" + try: + urllib.request.urlretrieve( + FEATURE_DOWNLOAD_URL.format(name="feature_download"), script + ) + except urllib.error.HTTPError: + # See https://github.com/SciTools/cartopy/pull/1602 + urllib.request.urlretrieve( + FEATURE_DOWNLOAD_URL.format(name="cartopy_feature_download"), + script, + ) + + # Add the script to the path to make it importable. sys.path.append(tmpdir) - - from feature_download import download_features + from download import download_features # Configure the cartopy resource cache. config["pre_existing_data_dir"] = str(target_dir) From f120bc3002309962a25b13d06fe90c896f3469a4 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Fri, 3 Sep 2021 12:06:25 +0100 Subject: [PATCH 10/12] update whatsnew to clarify as dev tool --- docs/src/whatsnew/latest.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index f96b8f1990..616ce76f15 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -32,7 +32,7 @@ This document explains the changes made to Iris for this release =========== #. `@bjlittle`_ added the `tools/cartopy_feature_download.py` command-line utility - which allows users to easily download feature resources for `cartopy`_. + which allows developers to easily download feature resources for `cartopy`_. (:pull:`4304`) From 9e1fbc8cb5bec9f21e6d967f710300fe77c4c494 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Fri, 3 Sep 2021 14:40:59 +0100 Subject: [PATCH 11/12] push fix to cartopy --- .cirrus.yml | 6 +- docs/src/whatsnew/latest.rst | 4 +- tools/cartopy_feature_download.py | 110 ------------------------------ 3 files changed, 5 insertions(+), 115 deletions(-) delete mode 100644 tools/cartopy_feature_download.py diff --git a/.cirrus.yml b/.cirrus.yml index bfd0747586..7886d6a5ce 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -26,7 +26,7 @@ env: # Maximum cache period (in weeks) before forcing a new cache upload. CACHE_PERIOD: "2" # Increment the build number to force new cartopy cache upload. - CARTOPY_CACHE_BUILD: "1" + CARTOPY_CACHE_BUILD: "2" # Increment the build number to force new conda cache upload. CONDA_CACHE_BUILD: "0" # Increment the build number to force new nox cache upload. @@ -76,7 +76,9 @@ linux_task_template: &LINUX_TASK_TEMPLATE - conda create --quiet --name cartopy-cache cartopy - source ${HOME}/miniconda/etc/profile.d/conda.sh >/dev/null 2>&1 - conda activate cartopy-cache >/dev/null 2>&1 - - python tools/cartopy_feature_download.py --output ${HOME}/.local/share/cartopy --feature physical --nowarn + - cd $(mktemp -d) + - wget https://raw.githubusercontent.com/bjlittle/cartopy/cartopy-feature-download/tools/cartopy_feature_download.py + - python cartopy_feature_download.py physical --output ${HOME}/.local/share/cartopy --no-warn - conda deactivate >/dev/null 2>&1 nox_cache: folder: ${CIRRUS_WORKING_DIR}/.nox diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index 616ce76f15..f85c001453 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -31,9 +31,7 @@ This document explains the changes made to Iris for this release ✨ Features =========== -#. `@bjlittle`_ added the `tools/cartopy_feature_download.py` command-line utility - which allows developers to easily download feature resources for `cartopy`_. - (:pull:`4304`) +#. N/A 🐛 Bugs Fixed diff --git a/tools/cartopy_feature_download.py b/tools/cartopy_feature_download.py deleted file mode 100644 index 7bd5ec79f0..0000000000 --- a/tools/cartopy_feature_download.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright Iris contributors -# -# This file is part of Iris and is released under the LGPL license. -# See COPYING and COPYING.LESSER in the root of the repository for full -# licensing details. -""" -A command line utility for downloading cartopy resources e.g., - - > python cartopy_feature_download.py --help - - -""" - -import argparse -import os -import pathlib -import sys -import tempfile -import urllib.request - -import cartopy -from cartopy import config - -FEATURE_DOWNLOAD_URL = f"https://raw.githubusercontent.com/SciTools/cartopy/v{cartopy.__version__}/tools/{{name}}.py" -# This will be the (more stable) cartopy resource endpoint from v0.19.0.post1+ -# See https://github.com/SciTools/cartopy/pull/1833 -URL_TEMPLATE = "https://naturalearth.s3.amazonaws.com/{resolution}_{category}/ne_{resolution}_{name}.zip" -SHP_NE_SPEC = ("shapefiles", "natural_earth") - - -def main(target_dir, features, dry_run): - target_dir = pathlib.Path(target_dir).expanduser().resolve() - target_dir.mkdir(parents=True, exist_ok=True) - cwd = pathlib.Path.cwd() - - with tempfile.TemporaryDirectory() as tmpdir: - os.chdir(tmpdir) - - # Download cartopy feature_download tool, which is not bundled - # within a cartopy package, and make it importable. - script = "download.py" - try: - urllib.request.urlretrieve( - FEATURE_DOWNLOAD_URL.format(name="feature_download"), script - ) - except urllib.error.HTTPError: - # See https://github.com/SciTools/cartopy/pull/1602 - urllib.request.urlretrieve( - FEATURE_DOWNLOAD_URL.format(name="cartopy_feature_download"), - script, - ) - - # Add the script to the path to make it importable. - sys.path.append(tmpdir) - from download import download_features - - # Configure the cartopy resource cache. - config["pre_existing_data_dir"] = str(target_dir) - config["data_dir"] = str(target_dir) - config["repo_data_dir"] = str(target_dir) - # Force use of stable endpoint for pre-v0.20 cartopy. - config["downloaders"][SHP_NE_SPEC].url_template = URL_TEMPLATE - - # Perform download, or dry-run. - download_features(features, dry_run=dry_run) - - os.chdir(cwd) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Download cartopy data for caching." - ) - parser.add_argument( - "--dryrun", - "-d", - action="store_true", - help="perform a dry-run of the download", - ) - parser.add_argument( - "--feature", - "-f", - nargs="+", - default=["physical"], - help=( - "specify one or more features to download [cultural|cultural-extra|gshhs|physical], " - 'default is "physical"' - ), - ) - parser.add_argument( - "--nowarn", - "-n", - action="store_true", - help="ignore cartopy DownloadWarning warnings", - ) - parser.add_argument( - "--output", - "-o", - required=True, - help="save datasets in the specified directory", - ) - - args = parser.parse_args() - - if args.nowarn: - import warnings - - warnings.filterwarnings("ignore", category=cartopy.io.DownloadWarning) - - main(args.output, args.feature, args.dryrun) From be51d8d41bb6a350683926b466fff82b48617ebe Mon Sep 17 00:00:00 2001 From: Bill Little Date: Fri, 3 Sep 2021 23:34:03 +0100 Subject: [PATCH 12/12] use cartopy master --- .cirrus.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 7886d6a5ce..3a3989e46c 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -26,7 +26,7 @@ env: # Maximum cache period (in weeks) before forcing a new cache upload. CACHE_PERIOD: "2" # Increment the build number to force new cartopy cache upload. - CARTOPY_CACHE_BUILD: "2" + CARTOPY_CACHE_BUILD: "3" # Increment the build number to force new conda cache upload. CONDA_CACHE_BUILD: "0" # Increment the build number to force new nox cache upload. @@ -77,7 +77,7 @@ linux_task_template: &LINUX_TASK_TEMPLATE - source ${HOME}/miniconda/etc/profile.d/conda.sh >/dev/null 2>&1 - conda activate cartopy-cache >/dev/null 2>&1 - cd $(mktemp -d) - - wget https://raw.githubusercontent.com/bjlittle/cartopy/cartopy-feature-download/tools/cartopy_feature_download.py + - wget --quiet https://raw.githubusercontent.com/SciTools/cartopy/master/tools/cartopy_feature_download.py - python cartopy_feature_download.py physical --output ${HOME}/.local/share/cartopy --no-warn - conda deactivate >/dev/null 2>&1 nox_cache: