From 2bcf6acedce6f59f13f18525e7b58c16df1242fd Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Thu, 28 Mar 2024 17:40:09 -0400 Subject: [PATCH 01/21] Testing disabled garbage collection --- feedstock/requirements.txt | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/feedstock/requirements.txt b/feedstock/requirements.txt index d43b4e0..0ce1ebb 100644 --- a/feedstock/requirements.txt +++ b/feedstock/requirements.txt @@ -1,5 +1,12 @@ -pangeo-forge-esgf==0.1.1 -pangeo-forge-recipes==0.10.6 -dynamic-chunks==0.0.2 +git+https://github.com/ranchodeluxe/xarray@ranchodeluxe-patch-1 +boto3 +requests +git+https://github.com/ranchodeluxe/datatree@main#egg=xarray-datatree +git+https://github.com/pangeo-forge/pangeo-forge-recipes@gc/cached_disabled +git+https://github.com/carbonplan/ndpyramid +apache-beam==2.52.0 +zarr==2.16.1 +numpy +git+https://github.com/ranchodeluxe/rioxarray gcsfs apache-beam[gcp] From 881b6bb8639aed5a67c6fb0c4a2a0f218d5a828b Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Thu, 28 Mar 2024 17:43:23 -0400 Subject: [PATCH 02/21] Update requirements.txt --- feedstock/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feedstock/requirements.txt b/feedstock/requirements.txt index 0ce1ebb..77ecd92 100644 --- a/feedstock/requirements.txt +++ b/feedstock/requirements.txt @@ -2,7 +2,7 @@ git+https://github.com/ranchodeluxe/xarray@ranchodeluxe-patch-1 boto3 requests git+https://github.com/ranchodeluxe/datatree@main#egg=xarray-datatree -git+https://github.com/pangeo-forge/pangeo-forge-recipes@gc/cached_disabled +git+https://github.com/pangeo-forge/pangeo-forge-recipes@gc/cache_disabled git+https://github.com/carbonplan/ndpyramid apache-beam==2.52.0 zarr==2.16.1 From d30876532bd4899774f5cacce909e5bffd7ec22a Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Thu, 28 Mar 2024 18:24:31 -0400 Subject: [PATCH 03/21] Update cmip_mre.py --- feedstock/cmip_mre.py | 50 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/feedstock/cmip_mre.py b/feedstock/cmip_mre.py index e0dff78..bb92170 100644 --- a/feedstock/cmip_mre.py +++ b/feedstock/cmip_mre.py @@ -61,6 +61,44 @@ def expand(self, pcoll: beam.PCollection) -> beam.PCollection: 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_185501010130-185912312230.nc', ] pattern = pattern_from_file_sequence(urls, concat_dim='time') + +urls_full = [ + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_185001010130-185412312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_185501010130-185912312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_186001010130-186412312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_186501010130-186912312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_187001010130-187412312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_187501010130-187912312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_188001010130-188412312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_188501010130-188912312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_189001010130-189412312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_189501010130-189912312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_190001010130-190412312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_190501010130-190912312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_191001010130-191412312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_191501010130-191912312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_192001010130-192412312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_192501010130-192912312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_193001010130-193412312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_193501010130-193912312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_194001010130-194412312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_194501010130-194912312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_195001010130-195412312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_195501010130-195912312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_196001010130-196412312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_196501010130-196912312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_197001010130-197412312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_197501010130-197912312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_198001010130-198412312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_198501010130-198912312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_199001010130-199412312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_199501010130-199912312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_200001010130-200412312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_200501010130-200912312230.nc', + 'https://esgf-data1.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/CMCC/CMCC-ESM2/historical/r1i1p1f1/3hr/pr/gn/v20210114/pr_3hr_CMCC-ESM2_historical_r1i1p1f1_gn_201001010130-201412312230.nc', +] + +pattern_full = pattern_from_file_sequence(urls_full, concat_dim='time') # full example with only time chunking time_only = ( @@ -87,6 +125,18 @@ def expand(self, pcoll: beam.PCollection) -> beam.PCollection: ) ) +lon_only_full = ( + f'Creating {iid}' >> beam.Create(pattern.items()) + | OpenURLWithFSSpec() + | OpenWithXarray(xarray_open_kwargs={'use_cftime': True}) + | Preprocessor() + | StoreToZarr( + store_name=f'{iid}.zarr', + combine_dims=pattern.combine_dim_keys, + target_chunks={'lon': 10, 'time':29200, 'bnds': 2, 'lat': 192} + ) +) + time_only_load = ( f'Creating {iid}' >> beam.Create(pattern.items()) | OpenURLWithFSSpec() From 9fd3ea21894c3abc95cf8fcc35c6c686219d7a84 Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Thu, 28 Mar 2024 18:26:06 -0400 Subject: [PATCH 04/21] Update meta.yaml --- feedstock/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/feedstock/meta.yaml b/feedstock/meta.yaml index 6582d49..2e98a4e 100644 --- a/feedstock/meta.yaml +++ b/feedstock/meta.yaml @@ -10,6 +10,8 @@ recipes: object: "cmip_mre:time_only_load" - id: lon_only_load object: "cmip_mre:lon_only_load" + - id: lon_only_full + object: "cmip_mre:lon_only_full" provenance: providers: - name: "Zenodo" From e408c8d054f4a2df6b625d242d1ebb70f5aca7d9 Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Thu, 28 Mar 2024 18:39:34 -0400 Subject: [PATCH 05/21] Update cmip_mre.py --- feedstock/cmip_mre.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feedstock/cmip_mre.py b/feedstock/cmip_mre.py index bb92170..25476f5 100644 --- a/feedstock/cmip_mre.py +++ b/feedstock/cmip_mre.py @@ -126,7 +126,7 @@ def expand(self, pcoll: beam.PCollection) -> beam.PCollection: ) lon_only_full = ( - f'Creating {iid}' >> beam.Create(pattern.items()) + f'Creating {iid}' >> beam.Create(pattern_full.items()) | OpenURLWithFSSpec() | OpenWithXarray(xarray_open_kwargs={'use_cftime': True}) | Preprocessor() From 986b3e80ceaea59314752168cc76876eec68ff70 Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Thu, 28 Mar 2024 20:13:27 -0400 Subject: [PATCH 06/21] Update requirements.txt --- feedstock/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feedstock/requirements.txt b/feedstock/requirements.txt index 77ecd92..7eb429d 100644 --- a/feedstock/requirements.txt +++ b/feedstock/requirements.txt @@ -2,7 +2,7 @@ git+https://github.com/ranchodeluxe/xarray@ranchodeluxe-patch-1 boto3 requests git+https://github.com/ranchodeluxe/datatree@main#egg=xarray-datatree -git+https://github.com/pangeo-forge/pangeo-forge-recipes@gc/cache_disabled +pangeo-forge-recipes==0.10.7 git+https://github.com/carbonplan/ndpyramid apache-beam==2.52.0 zarr==2.16.1 From e277ba2c9b7779e4d8373a0a078358a79f6f1254 Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Thu, 28 Mar 2024 20:29:07 -0400 Subject: [PATCH 07/21] Update requirements.txt --- feedstock/requirements.txt | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/feedstock/requirements.txt b/feedstock/requirements.txt index 7eb429d..04c76f6 100644 --- a/feedstock/requirements.txt +++ b/feedstock/requirements.txt @@ -1,12 +1,6 @@ -git+https://github.com/ranchodeluxe/xarray@ranchodeluxe-patch-1 -boto3 -requests -git+https://github.com/ranchodeluxe/datatree@main#egg=xarray-datatree +git+https://github.com/ranchodeluxe/xarray@ranchodeluxe-patch-1#egg=xarray-datatree pangeo-forge-recipes==0.10.7 -git+https://github.com/carbonplan/ndpyramid apache-beam==2.52.0 zarr==2.16.1 -numpy -git+https://github.com/ranchodeluxe/rioxarray gcsfs apache-beam[gcp] From a9c3bb2ae574c1c97fe458b1669a033138d93316 Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Thu, 28 Mar 2024 20:45:28 -0400 Subject: [PATCH 08/21] Update requirements.txt --- feedstock/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feedstock/requirements.txt b/feedstock/requirements.txt index 04c76f6..2a1e2d8 100644 --- a/feedstock/requirements.txt +++ b/feedstock/requirements.txt @@ -1,4 +1,4 @@ -git+https://github.com/ranchodeluxe/xarray@ranchodeluxe-patch-1#egg=xarray-datatree +git+https://github.com/ranchodeluxe/xarray@ranchodeluxe-patch-1#egg=xarray pangeo-forge-recipes==0.10.7 apache-beam==2.52.0 zarr==2.16.1 From fe3892a22d86702e79cd2fca56e895d98b914875 Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Thu, 28 Mar 2024 20:53:09 -0400 Subject: [PATCH 09/21] Update requirements.txt --- feedstock/requirements.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/feedstock/requirements.txt b/feedstock/requirements.txt index 2a1e2d8..a80076e 100644 --- a/feedstock/requirements.txt +++ b/feedstock/requirements.txt @@ -1,6 +1,9 @@ git+https://github.com/ranchodeluxe/xarray@ranchodeluxe-patch-1#egg=xarray -pangeo-forge-recipes==0.10.7 +git+https://github.com/ranchodeluxe/rioxarray +git+https://github.com/ranchodeluxe/datatree@main#egg=xarray-datatree +git+https://github.com/pangeo-forge/pangeo-forge-recipes@gc/cached_disabled apache-beam==2.52.0 zarr==2.16.1 gcsfs apache-beam[gcp] + From ae7ac02be470c9b353fe1fc95931a90396199b47 Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Thu, 28 Mar 2024 22:48:37 -0400 Subject: [PATCH 10/21] Update requirements.txt --- feedstock/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feedstock/requirements.txt b/feedstock/requirements.txt index a80076e..ad7b762 100644 --- a/feedstock/requirements.txt +++ b/feedstock/requirements.txt @@ -1,7 +1,7 @@ git+https://github.com/ranchodeluxe/xarray@ranchodeluxe-patch-1#egg=xarray git+https://github.com/ranchodeluxe/rioxarray git+https://github.com/ranchodeluxe/datatree@main#egg=xarray-datatree -git+https://github.com/pangeo-forge/pangeo-forge-recipes@gc/cached_disabled +git+https://github.com/pangeo-forge/pangeo-forge-recipes@jb/xarray-hack #see @gc/cached_disabled but with cache enabled apache-beam==2.52.0 zarr==2.16.1 gcsfs From 1acf91cb99b365e8b4172b04829ed78c8391849d Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Mon, 1 Apr 2024 14:25:15 -0400 Subject: [PATCH 11/21] Update cmip_mre.py --- feedstock/cmip_mre.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feedstock/cmip_mre.py b/feedstock/cmip_mre.py index 25476f5..3a6438e 100644 --- a/feedstock/cmip_mre.py +++ b/feedstock/cmip_mre.py @@ -128,7 +128,7 @@ def expand(self, pcoll: beam.PCollection) -> beam.PCollection: lon_only_full = ( f'Creating {iid}' >> beam.Create(pattern_full.items()) | OpenURLWithFSSpec() - | OpenWithXarray(xarray_open_kwargs={'use_cftime': True}) + | OpenWithXarray(xarray_open_kwargs={'use_cftime': True, copy_to_local=True}) | Preprocessor() | StoreToZarr( store_name=f'{iid}.zarr', From 1f62aaf100d7edd34949dd21116e4b0b0520df29 Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Mon, 1 Apr 2024 14:26:24 -0400 Subject: [PATCH 12/21] Update requirements.txt --- feedstock/requirements.txt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/feedstock/requirements.txt b/feedstock/requirements.txt index ad7b762..f1418ed 100644 --- a/feedstock/requirements.txt +++ b/feedstock/requirements.txt @@ -1,7 +1,5 @@ -git+https://github.com/ranchodeluxe/xarray@ranchodeluxe-patch-1#egg=xarray -git+https://github.com/ranchodeluxe/rioxarray -git+https://github.com/ranchodeluxe/datatree@main#egg=xarray-datatree -git+https://github.com/pangeo-forge/pangeo-forge-recipes@jb/xarray-hack #see @gc/cached_disabled but with cache enabled +xarray +pangeo-forge-recipes@0.10.7 apache-beam==2.52.0 zarr==2.16.1 gcsfs From eeb3ef820a11bdf3e388feb64e40630727a22913 Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Mon, 1 Apr 2024 14:28:57 -0400 Subject: [PATCH 13/21] Update requirements.txt --- feedstock/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feedstock/requirements.txt b/feedstock/requirements.txt index f1418ed..c2bc45e 100644 --- a/feedstock/requirements.txt +++ b/feedstock/requirements.txt @@ -1,5 +1,5 @@ xarray -pangeo-forge-recipes@0.10.7 +pangeo-forge-recipes==0.10.7 apache-beam==2.52.0 zarr==2.16.1 gcsfs From 742396f824a99b29afec10d400cb1336a0f2daab Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Mon, 1 Apr 2024 14:34:39 -0400 Subject: [PATCH 14/21] Update cmip_mre.py --- feedstock/cmip_mre.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feedstock/cmip_mre.py b/feedstock/cmip_mre.py index 3a6438e..e025152 100644 --- a/feedstock/cmip_mre.py +++ b/feedstock/cmip_mre.py @@ -128,7 +128,7 @@ def expand(self, pcoll: beam.PCollection) -> beam.PCollection: lon_only_full = ( f'Creating {iid}' >> beam.Create(pattern_full.items()) | OpenURLWithFSSpec() - | OpenWithXarray(xarray_open_kwargs={'use_cftime': True, copy_to_local=True}) + | OpenWithXarray(xarray_open_kwargs={'use_cftime': True, copy_to_local:True}) | Preprocessor() | StoreToZarr( store_name=f'{iid}.zarr', From e13950bff5584f6efd6ff3a744f1d5a3b58be4e2 Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Mon, 1 Apr 2024 15:03:33 -0400 Subject: [PATCH 15/21] Update cmip_mre.py --- feedstock/cmip_mre.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feedstock/cmip_mre.py b/feedstock/cmip_mre.py index e025152..540a9aa 100644 --- a/feedstock/cmip_mre.py +++ b/feedstock/cmip_mre.py @@ -128,7 +128,7 @@ def expand(self, pcoll: beam.PCollection) -> beam.PCollection: lon_only_full = ( f'Creating {iid}' >> beam.Create(pattern_full.items()) | OpenURLWithFSSpec() - | OpenWithXarray(xarray_open_kwargs={'use_cftime': True, copy_to_local:True}) + | OpenWithXarray(xarray_open_kwargs={'use_cftime': True}, copy_to_local=True) | Preprocessor() | StoreToZarr( store_name=f'{iid}.zarr', From a2c32ffbabfd3d1450cb40955d79eedcce5525e7 Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Mon, 1 Apr 2024 15:51:36 -0400 Subject: [PATCH 16/21] Update cmip_mre.py --- feedstock/cmip_mre.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feedstock/cmip_mre.py b/feedstock/cmip_mre.py index 540a9aa..4111e75 100644 --- a/feedstock/cmip_mre.py +++ b/feedstock/cmip_mre.py @@ -128,7 +128,7 @@ def expand(self, pcoll: beam.PCollection) -> beam.PCollection: lon_only_full = ( f'Creating {iid}' >> beam.Create(pattern_full.items()) | OpenURLWithFSSpec() - | OpenWithXarray(xarray_open_kwargs={'use_cftime': True}, copy_to_local=True) + | OpenWithXarray(xarray_open_kwargs={'use_cftime': True}, copy_to_local=True, load=True) | Preprocessor() | StoreToZarr( store_name=f'{iid}.zarr', From 1ef1dada40d37331465051004f55eb531d5c33ba Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Mon, 1 Apr 2024 16:33:41 -0400 Subject: [PATCH 17/21] Update deploy.yaml --- .github/workflows/deploy.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index c0fcd7d..abb99b1 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -32,7 +32,8 @@ jobs: "use_public_ips": true, "service_account_email": "julius-leap-dataflow@leap-pangeo.iam.gserviceaccount.com", "project_id": "leap-pangeo", - "temp_gcs_location": "gs://leap-scratch/data-library/temp" + "temp_gcs_location": "gs://leap-scratch/data-library/temp", + "machine_type": "n2-highmem-32" }, "TargetStorage": { "fsspec_class": "gcsfs.GCSFileSystem", From 21d17eeeb2c4f0a97fb21efa72cd58b63c7a7db9 Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Mon, 1 Apr 2024 16:55:15 -0400 Subject: [PATCH 18/21] Update cmip_mre.py --- feedstock/cmip_mre.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feedstock/cmip_mre.py b/feedstock/cmip_mre.py index 4111e75..540a9aa 100644 --- a/feedstock/cmip_mre.py +++ b/feedstock/cmip_mre.py @@ -128,7 +128,7 @@ def expand(self, pcoll: beam.PCollection) -> beam.PCollection: lon_only_full = ( f'Creating {iid}' >> beam.Create(pattern_full.items()) | OpenURLWithFSSpec() - | OpenWithXarray(xarray_open_kwargs={'use_cftime': True}, copy_to_local=True, load=True) + | OpenWithXarray(xarray_open_kwargs={'use_cftime': True}, copy_to_local=True) | Preprocessor() | StoreToZarr( store_name=f'{iid}.zarr', From fbf1bb99f0f5c8ffab1ed3c5e99ff36b551e554d Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Mon, 1 Apr 2024 17:38:29 -0400 Subject: [PATCH 19/21] Update deploy.yaml --- .github/workflows/deploy.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index abb99b1..8782582 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -33,7 +33,8 @@ jobs: "service_account_email": "julius-leap-dataflow@leap-pangeo.iam.gserviceaccount.com", "project_id": "leap-pangeo", "temp_gcs_location": "gs://leap-scratch/data-library/temp", - "machine_type": "n2-highmem-32" + "machine_type": "n2-highmem-32", + "disk_size_gb": 250 }, "TargetStorage": { "fsspec_class": "gcsfs.GCSFileSystem", From b517c86cb633ef261aeb7194f9368c6c63bc87a1 Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Mon, 1 Apr 2024 17:43:27 -0400 Subject: [PATCH 20/21] Update deploy.yaml --- .github/workflows/deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 8782582..e278190 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -20,7 +20,7 @@ jobs: with: credentials_json: "${{ secrets.GCP_DATAFLOW_SERVICE_KEY }}" - name: "Deploy recipes" - uses: "pangeo-forge/deploy-recipe-action@add-label-print" + uses: "pangeo-forge/deploy-recipe-action@dev-test-runner-disk_size_gb" with: select_recipe_by_label: true pangeo_forge_runner_config: > From 7060f91dad536ecd6972748b5e93648a3f93ac13 Mon Sep 17 00:00:00 2001 From: Julius Busecke Date: Mon, 1 Apr 2024 18:13:56 -0400 Subject: [PATCH 21/21] Update cmip_mre.py --- feedstock/cmip_mre.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feedstock/cmip_mre.py b/feedstock/cmip_mre.py index 540a9aa..4111e75 100644 --- a/feedstock/cmip_mre.py +++ b/feedstock/cmip_mre.py @@ -128,7 +128,7 @@ def expand(self, pcoll: beam.PCollection) -> beam.PCollection: lon_only_full = ( f'Creating {iid}' >> beam.Create(pattern_full.items()) | OpenURLWithFSSpec() - | OpenWithXarray(xarray_open_kwargs={'use_cftime': True}, copy_to_local=True) + | OpenWithXarray(xarray_open_kwargs={'use_cftime': True}, copy_to_local=True, load=True) | Preprocessor() | StoreToZarr( store_name=f'{iid}.zarr',