From 8e5b1af218a0573205459e70b395112a536ce0c3 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Fri, 1 Mar 2024 13:58:10 -0600 Subject: [PATCH] More testing fixes --- .../integration/generated/directions_chrysalis.md | 2 +- .../generated/test_bundles_chrysalis.cfg | 8 ++++---- .../generated/test_complete_run_chrysalis.cfg | 14 +++++++------- .../integration/generated/test_debug_chrysalis.cfg | 6 +++--- tests/integration/template_directions.md | 2 +- tests/integration/utils.py | 6 +++--- zppy/templates/ilamb.bash | 4 ++-- zppy/templates/ts.bash | 4 +--- zppy/utils.py | 1 - 9 files changed, 22 insertions(+), 25 deletions(-) diff --git a/tests/integration/generated/directions_chrysalis.md b/tests/integration/generated/directions_chrysalis.md index eb486d46..1565026c 100644 --- a/tests/integration/generated/directions_chrysalis.md +++ b/tests/integration/generated/directions_chrysalis.md @@ -12,7 +12,7 @@ NOTE: Actually running the tests (e.g., `python -u -m unittest tests/integration can not be done from two different branches simultaneously (since files in the `zppy` directory rather than an external directory get changed). -## Debug your code with test_debug +### Debug your code with test_debug Follow the directions in the cfg to modify it for your testing needs. diff --git a/tests/integration/generated/test_bundles_chrysalis.cfg b/tests/integration/generated/test_bundles_chrysalis.cfg index 5002c233..0f7bb38a 100644 --- a/tests/integration/generated/test_bundles_chrysalis.cfg +++ b/tests/integration/generated/test_bundles_chrysalis.cfg @@ -7,11 +7,11 @@ input = "/lcrc/group/e3sm/ac.forsyth2/E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_bundles.py -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/unique_id/v2.LR.historical_0201" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/pr-548-20240229/v2.LR.historical_0201" partition = "compute" qos = "regular" walltime = "07:00:00" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/unique_id" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/pr-548-20240229" [bundle] @@ -36,7 +36,7 @@ years = "1850:1854:2", "1850:1854:4", [ts] active = True bundle = "bundle1" -e3sm_to_cmip_environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240223" +e3sm_to_cmip_environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240229" years = "1850:1854:2", [[ atm_monthly_180x360_aave ]] @@ -60,7 +60,7 @@ years = "1850:1854:2", years = "1850:1860:5", [[ land_monthly ]] - e3sm_to_cmip_environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240223" + e3sm_to_cmip_environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240229" extra_vars = "landfrac" frequency = "monthly" input_files = "elm.h0" diff --git a/tests/integration/generated/test_complete_run_chrysalis.cfg b/tests/integration/generated/test_complete_run_chrysalis.cfg index df99a939..adf064f8 100644 --- a/tests/integration/generated/test_complete_run_chrysalis.cfg +++ b/tests/integration/generated/test_complete_run_chrysalis.cfg @@ -7,10 +7,10 @@ input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/pr-548-20240229/v2.LR.historical_0201" partition = "debug" qos = "regular" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/unique_id" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/pr-548-20240229" [climo] active = True @@ -37,7 +37,7 @@ years = "1850:1854:2", "1850:1854:4", [ts] active = True -e3sm_to_cmip_environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240223" +e3sm_to_cmip_environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240229" walltime = "00:30:00" years = "1850:1854:2", @@ -62,7 +62,7 @@ years = "1850:1854:2", years = "1850:1860:5", [[ land_monthly ]] - e3sm_to_cmip_environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240223" + e3sm_to_cmip_environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240229" extra_vars = "landfrac" frequency = "monthly" input_files = "elm.h0" @@ -111,7 +111,7 @@ years = "1850:1854:2", "1850:1854:4", walltime = "5:00:00" [[ atm_monthly_180x360_aave_environment_commands ]] - environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240214" + environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240229" sets = "qbo", ts_subsection = "atm_monthly_180x360_aave" @@ -132,7 +132,7 @@ years = "1850:1854:2", "1850:1854:4", ref_name = "v2.LR.historical_0201" ref_start_yr = 1850 ref_years = "1850-1851", - reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201/post/atm/180x360_aave/clim" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/pr-548-20240229/v2.LR.historical_0201/post/atm/180x360_aave/clim" run_type = "model_vs_model" short_ref_name = "v2.LR.historical_0201" swap_test_ref = False @@ -153,7 +153,7 @@ years = "1850:1854:2", "1850:1854:4", ref_name = "v2.LR.historical_0201" ref_start_yr = 1850 ref_years = "1850-1851", - reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201/post/lnd/180x360_aave/clim" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/pr-548-20240229/v2.LR.historical_0201/post/lnd/180x360_aave/clim" run_type = "model_vs_model" sets = "lat_lon_land", short_ref_name = "same simulation" diff --git a/tests/integration/generated/test_debug_chrysalis.cfg b/tests/integration/generated/test_debug_chrysalis.cfg index 22158126..f5e4d8ad 100644 --- a/tests/integration/generated/test_debug_chrysalis.cfg +++ b/tests/integration/generated/test_debug_chrysalis.cfg @@ -13,10 +13,10 @@ input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/unique_id/v2.LR.historical_0201" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/pr-548-20240229/v2.LR.historical_0201" partition = "debug" qos = "regular" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_debug_www/unique_id" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_debug_www/pr-548-20240229" [ts] active = True @@ -36,7 +36,7 @@ years = "1850:1854:2", vars = "PRECT" [[ land_monthly ]] - e3sm_to_cmip_environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240223" + e3sm_to_cmip_environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240229" extra_vars = "landfrac" frequency = "monthly" input_files = "elm.h0" diff --git a/tests/integration/template_directions.md b/tests/integration/template_directions.md index 602f20d1..e210b24a 100644 --- a/tests/integration/template_directions.md +++ b/tests/integration/template_directions.md @@ -12,7 +12,7 @@ NOTE: Actually running the tests (e.g., `python -u -m unittest tests/integration can not be done from two different branches simultaneously (since files in the `zppy` directory rather than an external directory get changed). -## Debug your code with test_debug +### Debug your code with test_debug Follow the directions in the cfg to modify it for your testing needs. diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 75f5b37a..388346fa 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -7,7 +7,7 @@ from mache import MachineInfo from PIL import Image, ImageChops, ImageDraw -UNIQUE_ID = "unique_id" +UNIQUE_ID = "pr-548-20240229" # Image checking ########################################################## @@ -136,9 +136,9 @@ def get_chyrsalis_expansions(config): "bundles_walltime": "07:00:00", "constraint": "", # To run this test, replace conda environment with your e3sm_diags dev environment - "diags_environment_commands": "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240214", + "diags_environment_commands": "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240229", "diags_walltime": "5:00:00", - "e3sm_to_cmip_environment_commands": "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240223", + "e3sm_to_cmip_environment_commands": "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_to_cmip_20240229", "environment_commands_test": "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.9.0rc16_chrysalis.sh", "expected_dir": "/lcrc/group/e3sm/public_html/zppy_test_resources/", "mpas_analysis_walltime": "00:30:00", diff --git a/zppy/templates/ilamb.bash b/zppy/templates/ilamb.bash index 03d9248e..ade7750a 100644 --- a/zppy/templates/ilamb.bash +++ b/zppy/templates/ilamb.bash @@ -46,8 +46,8 @@ cd ${model_root}/${case} # Create output directory # Create local links to input cmip time-series files -lnd_ts_for_ilamb={{ output }}/post/lnd/180x360_aave/cmip_ts/${Y1}-${Y2}/monthly/ -atm_ts_for_ilamb={{ output }}/post/atm/180x360_aave/cmip_ts/${Y1}-${Y2}/monthly/ +lnd_ts_for_ilamb={{ output }}/post/lnd/180x360_aave/cmip_ts/monthly/ +atm_ts_for_ilamb={{ output }}/post/atm/180x360_aave/cmip_ts/monthly/ cp -s ${lnd_ts_for_ilamb}/*_*_*_*_*_*_${Y1}??-${Y2}??.nc . cp -s ${atm_ts_for_ilamb}/*_*_*_*_*_*_${Y1}??-${Y2}??.nc . cd ../.. diff --git a/zppy/templates/ts.bash b/zppy/templates/ts.bash index 3ecd606a..6c555801 100644 --- a/zppy/templates/ts.bash +++ b/zppy/templates/ts.bash @@ -142,9 +142,7 @@ EOF { export cmortables_dir={{ cmor_tables_prefix }}/cmip6-cmor-tables/Tables input_dir={{ output }}/post/{{ component }}/{{ grid }}/ts/{{ frequency }}/{{ '%dyr' % (ypf) }} - Y1="{{ '%04d' % (yr_start) }}" - Y2="{{ '%04d' % (yr_end) }}" - dest_cmip={{ output }}/post/{{ component }}/{{ grid }}/cmip_ts/${Y1}-${Y2}/{{ frequency }} + dest_cmip={{ output }}/post/{{ component }}/{{ grid }}/cmip_ts/{{ frequency }} mkdir -p ${dest_cmip} {{ e3sm_to_cmip_environment_commands }} srun -N 1 e3sm_to_cmip \ diff --git a/zppy/utils.py b/zppy/utils.py index cda764f8..5a34ce98 100644 --- a/zppy/utils.py +++ b/zppy/utils.py @@ -192,7 +192,6 @@ def submitScript(scriptFile, statusFile, export, job_ids_file, dependFiles=[]): # Handle dependencies dependIds = [] - for dependFile in dependFiles: if os.path.isfile(dependFile): with open(dependFile, "r") as f: