From c4d6da14fd8ef627bb933d4ce6ae261c8a1cbaf4 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 25 May 2021 09:43:40 -0400 Subject: [PATCH 01/14] release: update to use image pavics/workflow-tests:210525 --- Jenkinsfile | 2 +- binder/Dockerfile | 2 +- launchcontainer | 2 +- launchnotebook | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a3a77e24..a457f609 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,7 @@ pipeline { // https://jenkins.io/doc/book/pipeline/syntax/ agent { docker { - image "pavics/workflow-tests:210415" + image "pavics/workflow-tests:210525" label 'linux && docker' } } diff --git a/binder/Dockerfile b/binder/Dockerfile index 398c21ac..59a5edb4 100644 --- a/binder/Dockerfile +++ b/binder/Dockerfile @@ -1,4 +1,4 @@ -FROM pavics/workflow-tests:210415 +FROM pavics/workflow-tests:210525 USER root diff --git a/launchcontainer b/launchcontainer index 85e368c1..6f930e24 100755 --- a/launchcontainer +++ b/launchcontainer @@ -1,7 +1,7 @@ #!/bin/sh -x if [ -z "$DOCKER_IMAGE" ]; then - DOCKER_IMAGE="pavics/workflow-tests:210415" + DOCKER_IMAGE="pavics/workflow-tests:210525" fi if [ -z "$CONTAINER_NAME" ]; then diff --git a/launchnotebook b/launchnotebook index a6aca549..a339c539 100755 --- a/launchnotebook +++ b/launchnotebook @@ -7,7 +7,7 @@ if [ -z "$PORT" ]; then fi if [ -z "$DOCKER_IMAGE" ]; then - DOCKER_IMAGE="pavics/workflow-tests:210415" + DOCKER_IMAGE="pavics/workflow-tests:210525" fi if [ -z "$CONTAINER_NAME" ]; then From ad6d514676871d6f3cc85d394921a5da983f8eb0 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 25 May 2021 17:37:49 -0400 Subject: [PATCH 02/14] docker: add new Dockerfile for testing new package Useful to avoid full rebuild just to test adding a new package or upgrading an existing package. Sort of a restore of commit cbfa5d5471fa99d5534b87791a175b8436f09cf1. --- docker/Dockerfile.testing | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 docker/Dockerfile.testing diff --git a/docker/Dockerfile.testing b/docker/Dockerfile.testing new file mode 100644 index 00000000..983dda74 --- /dev/null +++ b/docker/Dockerfile.testing @@ -0,0 +1,17 @@ +# For testing quickly without having to do a full rebuild. + +FROM pavics/workflow-tests:210525 + +USER root + +# Use 'update' for existing and 'install' for new package. +# Keep same channel ordering to not revert anything. +#RUN umask 0000 \ +# && conda update -c zeitsperre -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy clisops xclim \ +# && pip uninstall -y ravenpy \ +# && conda install -c zeitsperre -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy ravenpy aiohttp + +RUN umask 0000 \ + && conda install -c zeitsperre -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy pydantic + +USER jenkins From 8103b74da38ecc59ed7a4a3c40367f2a29354e45 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 25 May 2021 18:08:28 -0400 Subject: [PATCH 03/14] docker: add pydantic, was part of previous build but missing in current build Previous build probably got pydantic via indirect dependency but somehow this has changed so now have to add pydantic explicitly, for Raven notebooks. --- docker/environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/environment.yml b/docker/environment.yml index c72bc230..bf3e0c49 100644 --- a/docker/environment.yml +++ b/docker/environment.yml @@ -49,6 +49,7 @@ dependencies: # for error 'ImportError: HTTPFileSystem requires "requests" and "aiohttp" to # be installed' with call 'fsspec.filesystem('https')' - aiohttp + - pydantic - xclim - ravenpy # Universal Regridder for Geospatial Data From 757c44eb9807b1e01500e9a389e184e9936e9c90 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 25 May 2021 18:43:51 -0400 Subject: [PATCH 04/14] docker: test latest RavenPy to fix Raven notebooks The fix did not make it to RavenPy 0.5.1. To fix error running Raven notebooks: ``` --------------------------------------------------------------------------- ImportError Traceback (most recent call last) in 2 3 from birdy import WPSClient ----> 4 from ravenpy.utilities.testdata import get_file 5 import os 6 import xarray as xr /opt/conda/envs/birdy/lib/python3.7/site-packages/ravenpy/utilities/testdata.py in 14 from xarray import Dataset 15 from xarray import open_dataset as _open_dataset ---> 16 from xarray.tutorial import file_md5_checksum 17 18 _default_cache_dir = Path.home() / ".raven_testing_data" ImportError: cannot import name 'file_md5_checksum' from 'xarray.tutorial' (/opt/conda/envs/birdy/lib/python3.7/site-packages/xarray/tutorial.py) ``` --- docker/Dockerfile.testing | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/Dockerfile.testing b/docker/Dockerfile.testing index 983dda74..ec427ed3 100644 --- a/docker/Dockerfile.testing +++ b/docker/Dockerfile.testing @@ -14,4 +14,7 @@ USER root RUN umask 0000 \ && conda install -c zeitsperre -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy pydantic +RUN umask 0000 \ + && pip install https://github.com/CSHS-CWRA/RavenPy/archive/refs/heads/master.zip + USER jenkins From 155ff691abd9f969a12c4300c1b95f4f95e4fc14 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Wed, 26 May 2021 13:33:43 -0400 Subject: [PATCH 05/14] release: update to use image pavics/workflow-tests:210526 --- Jenkinsfile | 2 +- binder/Dockerfile | 2 +- launchcontainer | 2 +- launchnotebook | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a457f609..6a6a80eb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,7 @@ pipeline { // https://jenkins.io/doc/book/pipeline/syntax/ agent { docker { - image "pavics/workflow-tests:210525" + image "pavics/workflow-tests:210526" label 'linux && docker' } } diff --git a/binder/Dockerfile b/binder/Dockerfile index 59a5edb4..ec27bc8b 100644 --- a/binder/Dockerfile +++ b/binder/Dockerfile @@ -1,4 +1,4 @@ -FROM pavics/workflow-tests:210525 +FROM pavics/workflow-tests:210526 USER root diff --git a/launchcontainer b/launchcontainer index 6f930e24..96c16d0f 100755 --- a/launchcontainer +++ b/launchcontainer @@ -1,7 +1,7 @@ #!/bin/sh -x if [ -z "$DOCKER_IMAGE" ]; then - DOCKER_IMAGE="pavics/workflow-tests:210525" + DOCKER_IMAGE="pavics/workflow-tests:210526" fi if [ -z "$CONTAINER_NAME" ]; then diff --git a/launchnotebook b/launchnotebook index a339c539..faa13ff4 100755 --- a/launchnotebook +++ b/launchnotebook @@ -7,7 +7,7 @@ if [ -z "$PORT" ]; then fi if [ -z "$DOCKER_IMAGE" ]; then - DOCKER_IMAGE="pavics/workflow-tests:210525" + DOCKER_IMAGE="pavics/workflow-tests:210526" fi if [ -z "$CONTAINER_NAME" ]; then From eed790493de8eebd4a733292c0d880aedcfd7a96 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Thu, 27 May 2021 09:54:35 -0400 Subject: [PATCH 06/14] docker: remove conda channel zeitsperre, ravenpy now available from conda-forge --- docker/Dockerfile | 2 +- docker/Dockerfile.testing | 6 +++--- docker/environment.yml | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 431e45cd..713aa9c4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -31,7 +31,7 @@ RUN python -m ipykernel install --name birdy # anything accidentally # this is for debug only, all dependencies should be specified in # environment.yml above -# RUN conda install -c zeitsperre -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy nbdime +# RUN conda install -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy nbdime # build jupyterlab extensions installed by conda, see `jupyter labextension list` RUN jupyter lab build diff --git a/docker/Dockerfile.testing b/docker/Dockerfile.testing index ec427ed3..0e319739 100644 --- a/docker/Dockerfile.testing +++ b/docker/Dockerfile.testing @@ -7,12 +7,12 @@ USER root # Use 'update' for existing and 'install' for new package. # Keep same channel ordering to not revert anything. #RUN umask 0000 \ -# && conda update -c zeitsperre -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy clisops xclim \ +# && conda update -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy clisops xclim \ # && pip uninstall -y ravenpy \ -# && conda install -c zeitsperre -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy ravenpy aiohttp +# && conda install -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy ravenpy aiohttp RUN umask 0000 \ - && conda install -c zeitsperre -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy pydantic + && conda install -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy pydantic RUN umask 0000 \ && pip install https://github.com/CSHS-CWRA/RavenPy/archive/refs/heads/master.zip diff --git a/docker/environment.yml b/docker/environment.yml index bf3e0c49..c66b73ae 100644 --- a/docker/environment.yml +++ b/docker/environment.yml @@ -1,7 +1,6 @@ # conda env create -f environment.yml name: birdy channels: - - zeitsperre # for ravenpy - conda-forge - cdat - bokeh From be6aa30729451b61525798a2d458e0ee66982209 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Thu, 27 May 2021 09:55:38 -0400 Subject: [PATCH 07/14] release: update to use image pavics/workflow-tests:210527 --- Jenkinsfile | 2 +- binder/Dockerfile | 2 +- launchcontainer | 2 +- launchnotebook | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6a6a80eb..0df0b955 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,7 @@ pipeline { // https://jenkins.io/doc/book/pipeline/syntax/ agent { docker { - image "pavics/workflow-tests:210526" + image "pavics/workflow-tests:210527" label 'linux && docker' } } diff --git a/binder/Dockerfile b/binder/Dockerfile index ec27bc8b..b8d97ebb 100644 --- a/binder/Dockerfile +++ b/binder/Dockerfile @@ -1,4 +1,4 @@ -FROM pavics/workflow-tests:210526 +FROM pavics/workflow-tests:210527 USER root diff --git a/launchcontainer b/launchcontainer index 96c16d0f..046cbec4 100755 --- a/launchcontainer +++ b/launchcontainer @@ -1,7 +1,7 @@ #!/bin/sh -x if [ -z "$DOCKER_IMAGE" ]; then - DOCKER_IMAGE="pavics/workflow-tests:210526" + DOCKER_IMAGE="pavics/workflow-tests:210527" fi if [ -z "$CONTAINER_NAME" ]; then diff --git a/launchnotebook b/launchnotebook index faa13ff4..b988641a 100755 --- a/launchnotebook +++ b/launchnotebook @@ -7,7 +7,7 @@ if [ -z "$PORT" ]; then fi if [ -z "$DOCKER_IMAGE" ]; then - DOCKER_IMAGE="pavics/workflow-tests:210526" + DOCKER_IMAGE="pavics/workflow-tests:210527" fi if [ -z "$CONTAINER_NAME" ]; then From 6e84c79aeb10cc1f114e63d4c651323201cb09b8 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Thu, 27 May 2021 13:12:00 -0400 Subject: [PATCH 08/14] docker: pin previous version of cftime due to issue with nc-time-axis --- docker/environment.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/environment.yml b/docker/environment.yml index c66b73ae..409a755f 100644 --- a/docker/environment.yml +++ b/docker/environment.yml @@ -44,6 +44,8 @@ dependencies: # pinning hvplot did not solve the problem with violin plot. - hvplot - nc-time-axis + # Pin cftime because of https://github.com/SciTools/nc-time-axis/issues/58 + - cftime==1.4.1 - statsmodels # for ravenpy # for error 'ImportError: HTTPFileSystem requires "requests" and "aiohttp" to # be installed' with call 'fsspec.filesystem('https')' From 2715a312306cb9acfe54f6cf390883df4e107b29 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Thu, 27 May 2021 13:12:33 -0400 Subject: [PATCH 09/14] release: update to use image pavics/workflow-tests:210527.1 --- Jenkinsfile | 2 +- binder/Dockerfile | 2 +- launchcontainer | 2 +- launchnotebook | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0df0b955..4b97c6e5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,7 @@ pipeline { // https://jenkins.io/doc/book/pipeline/syntax/ agent { docker { - image "pavics/workflow-tests:210527" + image "pavics/workflow-tests:210527.1" label 'linux && docker' } } diff --git a/binder/Dockerfile b/binder/Dockerfile index b8d97ebb..13a52eda 100644 --- a/binder/Dockerfile +++ b/binder/Dockerfile @@ -1,4 +1,4 @@ -FROM pavics/workflow-tests:210527 +FROM pavics/workflow-tests:210527.1 USER root diff --git a/launchcontainer b/launchcontainer index 046cbec4..1e5df802 100755 --- a/launchcontainer +++ b/launchcontainer @@ -1,7 +1,7 @@ #!/bin/sh -x if [ -z "$DOCKER_IMAGE" ]; then - DOCKER_IMAGE="pavics/workflow-tests:210527" + DOCKER_IMAGE="pavics/workflow-tests:210527.1" fi if [ -z "$CONTAINER_NAME" ]; then diff --git a/launchnotebook b/launchnotebook index b988641a..ce1e71d1 100755 --- a/launchnotebook +++ b/launchnotebook @@ -7,7 +7,7 @@ if [ -z "$PORT" ]; then fi if [ -z "$DOCKER_IMAGE" ]; then - DOCKER_IMAGE="pavics/workflow-tests:210527" + DOCKER_IMAGE="pavics/workflow-tests:210527.1" fi if [ -z "$CONTAINER_NAME" ]; then From 61a6ddb839c5f6d660b08e78bc31c1f6b6db3d5c Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 1 Jun 2021 18:34:48 -0400 Subject: [PATCH 10/14] docker: test pin rasterio=1.2.2 to fix Raven Extract_geographical_watershed_properties.ipynb --- docker/Dockerfile.testing | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile.testing b/docker/Dockerfile.testing index 0e319739..4b77b297 100644 --- a/docker/Dockerfile.testing +++ b/docker/Dockerfile.testing @@ -1,6 +1,6 @@ # For testing quickly without having to do a full rebuild. -FROM pavics/workflow-tests:210525 +FROM pavics/workflow-tests:210527.1 USER root @@ -12,9 +12,9 @@ USER root # && conda install -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy ravenpy aiohttp RUN umask 0000 \ - && conda install -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy pydantic + && conda install -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy rasterio=1.2.2 -RUN umask 0000 \ - && pip install https://github.com/CSHS-CWRA/RavenPy/archive/refs/heads/master.zip +#RUN umask 0000 \ +# && pip install https://github.com/CSHS-CWRA/RavenPy/archive/refs/heads/master.zip USER jenkins From 0c82a3ee49e00b8edd989399330887c77f0ab253 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 1 Jun 2021 19:33:05 -0400 Subject: [PATCH 11/14] docker: test pin rasterio=1.2.1 from older base pavics/workflow-tests:210415 to reproduce Raven nb prob --- docker/Dockerfile.testing | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.testing b/docker/Dockerfile.testing index 4b77b297..e4d26131 100644 --- a/docker/Dockerfile.testing +++ b/docker/Dockerfile.testing @@ -1,6 +1,6 @@ # For testing quickly without having to do a full rebuild. -FROM pavics/workflow-tests:210527.1 +FROM pavics/workflow-tests:210415 USER root @@ -12,7 +12,7 @@ USER root # && conda install -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy ravenpy aiohttp RUN umask 0000 \ - && conda install -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy rasterio=1.2.2 + && conda install -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy rasterio=1.2.1 #RUN umask 0000 \ # && pip install https://github.com/CSHS-CWRA/RavenPy/archive/refs/heads/master.zip From e1f5087f86b2cc0576adcb9eafeb3768fd45d5c2 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 15 Jun 2021 12:15:57 -0400 Subject: [PATCH 12/14] docker: update latest xclim 0.27.0 --- docker/Dockerfile.testing | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.testing b/docker/Dockerfile.testing index e4d26131..b4993b6e 100644 --- a/docker/Dockerfile.testing +++ b/docker/Dockerfile.testing @@ -1,6 +1,6 @@ # For testing quickly without having to do a full rebuild. -FROM pavics/workflow-tests:210415 +FROM pavics/workflow-tests:210527.1 USER root @@ -12,7 +12,7 @@ USER root # && conda install -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy ravenpy aiohttp RUN umask 0000 \ - && conda install -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy rasterio=1.2.1 + && conda update -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy xclim #RUN umask 0000 \ # && pip install https://github.com/CSHS-CWRA/RavenPy/archive/refs/heads/master.zip From 08f652cda3b07aa0af328ab08fdcbb85a313e6e5 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 15 Jun 2021 12:22:12 -0400 Subject: [PATCH 13/14] docker: add intake intake-esm gcsfs zarr for Raven notebooks --- docker/Dockerfile.testing | 3 ++- docker/environment.yml | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.testing b/docker/Dockerfile.testing index b4993b6e..5b8ccfce 100644 --- a/docker/Dockerfile.testing +++ b/docker/Dockerfile.testing @@ -12,7 +12,8 @@ USER root # && conda install -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy ravenpy aiohttp RUN umask 0000 \ - && conda update -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy xclim + && conda update -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy xclim \ + && conda install -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy intake intake-esm gcsfs zarr #RUN umask 0000 \ # && pip install https://github.com/CSHS-CWRA/RavenPy/archive/refs/heads/master.zip diff --git a/docker/environment.yml b/docker/environment.yml index 409a755f..7f30f3ae 100644 --- a/docker/environment.yml +++ b/docker/environment.yml @@ -51,6 +51,10 @@ dependencies: # be installed' with call 'fsspec.filesystem('https')' - aiohttp - pydantic + - intake + - intake-esm + - gcsfs + - zarr - xclim - ravenpy # Universal Regridder for Geospatial Data From cfb10a27d97fbbde88ccc194bbfb21838cb84b5a Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 15 Jun 2021 14:50:20 -0400 Subject: [PATCH 14/14] release: update to use image pavics/workflow-tests:210527.1-update20210615 --- Jenkinsfile | 2 +- binder/Dockerfile | 2 +- launchcontainer | 2 +- launchnotebook | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4b97c6e5..621b9d8e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,7 @@ pipeline { // https://jenkins.io/doc/book/pipeline/syntax/ agent { docker { - image "pavics/workflow-tests:210527.1" + image "pavics/workflow-tests:210527.1-update20210615" label 'linux && docker' } } diff --git a/binder/Dockerfile b/binder/Dockerfile index 13a52eda..a71be259 100644 --- a/binder/Dockerfile +++ b/binder/Dockerfile @@ -1,4 +1,4 @@ -FROM pavics/workflow-tests:210527.1 +FROM pavics/workflow-tests:210527.1-update20210615 USER root diff --git a/launchcontainer b/launchcontainer index 1e5df802..06268f9a 100755 --- a/launchcontainer +++ b/launchcontainer @@ -1,7 +1,7 @@ #!/bin/sh -x if [ -z "$DOCKER_IMAGE" ]; then - DOCKER_IMAGE="pavics/workflow-tests:210527.1" + DOCKER_IMAGE="pavics/workflow-tests:210527.1-update20210615" fi if [ -z "$CONTAINER_NAME" ]; then diff --git a/launchnotebook b/launchnotebook index ce1e71d1..84c5837f 100755 --- a/launchnotebook +++ b/launchnotebook @@ -7,7 +7,7 @@ if [ -z "$PORT" ]; then fi if [ -z "$DOCKER_IMAGE" ]; then - DOCKER_IMAGE="pavics/workflow-tests:210527.1" + DOCKER_IMAGE="pavics/workflow-tests:210527.1-update20210615" fi if [ -z "$CONTAINER_NAME" ]; then