From fa06f593ea454130aa43af3930c7e8ae61108834 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 21 Feb 2024 14:54:14 -0600 Subject: [PATCH 1/3] Testing updates --- .../generated/directions_chrysalis.md | 23 +++++++- .../generated/test_bundles_chrysalis.cfg | 4 +- .../generated/test_complete_run_chrysalis.cfg | 4 +- .../generated/test_debug_chrysalis.cfg | 56 +++++++++++++++++++ tests/integration/template_debug.cfg | 56 +++++++++++++++++++ tests/integration/template_directions.md | 23 +++++++- tests/integration/test_bundles.py | 6 +- tests/integration/utils.py | 4 +- 8 files changed, 163 insertions(+), 13 deletions(-) create mode 100644 tests/integration/generated/test_debug_chrysalis.cfg create mode 100644 tests/integration/template_debug.cfg diff --git a/tests/integration/generated/directions_chrysalis.md b/tests/integration/generated/directions_chrysalis.md index 7a7e96b2..eb486d46 100644 --- a/tests/integration/generated/directions_chrysalis.md +++ b/tests/integration/generated/directions_chrysalis.md @@ -12,6 +12,25 @@ NOTE: Actually running the tests (e.g., `python -u -m unittest tests/integration can not be done from two different branches simultaneously (since files in the `zppy` directory rather than an external directory get changed). +## Debug your code with test_debug + +Follow the directions in the cfg to modify it for your testing needs. + +``` +rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_debug_www//v2.LR.historical_0201 +rm -rf /lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output//v2.LR.historical_0201/post +# Generate cfg +python tests/integration/utils.py + +# Run jobs: +zppy -c tests/integration/generated/test_debug_chrysalis.cfg +# After they finish, check the results: +cd /lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output//v2.LR.historical_0201/post/scripts +grep -v "OK" *status +# Nothing should print +cd - +``` + ### test_bundles ``` @@ -87,7 +106,7 @@ mkdir -p /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundl cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output//v2.LR.historical_0201/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files cd /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles # Remove the image check failures, so they don't end up in the expected files. -rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/forsyth/zppy_test_bundles_www//v2.LR.historical_0201/image_check_failures +rm -rf image_check_failures # This file will list all the expected images. find . -type f -name '*.png' > ../image_list_expected_bundles.txt cd @@ -119,7 +138,7 @@ rm -rf /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www//v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run cd /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run # Remove the image check failures, so they don't end up in the expected files. -rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/forsyth/zppy_test_complete_run_www//v2.LR.historical_0201/image_check_failures +rm -rf image_check_failures # This file will list all the expected images. find . -type f -name '*.png' > ../image_list_expected_complete_run.txt cd diff --git a/tests/integration/generated/test_bundles_chrysalis.cfg b/tests/integration/generated/test_bundles_chrysalis.cfg index 95068a36..7e9a61bb 100644 --- a/tests/integration/generated/test_bundles_chrysalis.cfg +++ b/tests/integration/generated/test_bundles_chrysalis.cfg @@ -7,11 +7,11 @@ input = "/lcrc/group/e3sm/ac.forsyth2/E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_bundles.py -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/unique_id/v2.LR.historical_0201" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/testing-updates/v2.LR.historical_0201" partition = "compute" qos = "regular" walltime = "07:00:00" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/unique_id" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/testing-updates" [bundle] diff --git a/tests/integration/generated/test_complete_run_chrysalis.cfg b/tests/integration/generated/test_complete_run_chrysalis.cfg index bb222b8d..e712723c 100644 --- a/tests/integration/generated/test_complete_run_chrysalis.cfg +++ b/tests/integration/generated/test_complete_run_chrysalis.cfg @@ -7,10 +7,10 @@ input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/testing-updates/v2.LR.historical_0201" partition = "debug" qos = "regular" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/unique_id" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/testing-updates" [climo] active = True diff --git a/tests/integration/generated/test_debug_chrysalis.cfg b/tests/integration/generated/test_debug_chrysalis.cfg new file mode 100644 index 00000000..cff15475 --- /dev/null +++ b/tests/integration/generated/test_debug_chrysalis.cfg @@ -0,0 +1,56 @@ +# Use this cfg to test a pull request before merging. +# 1. Copy over the contents of template_complete_run.cfg. +# 2. Delete parts that aren't needed. `output` and `www` should include `zppy_test_debug` rather than `zppy_test_complete_run`. +# 3. Iteratively debug/develop with this cfg. +# 4. Once it's working as expected, copy important changes to template_complete_run.cfg. + +[default] +case = v2.LR.historical_0201 +constraint = "" +dry_run = "False" +environment_commands = "" +input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201" +input_subdir = archive/atm/hist +mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" +# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/testing-updates/v2.LR.historical_0201" +partition = "debug" +qos = "regular" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_debug_www/testing-updates" + +[ts] +active = True +walltime = "00:30:00" +years = "1850:1854:2", + + [[ atm_monthly_180x360_aave ]] + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + ts_fmt = "cmip" + + [[ atm_daily_180x360_aave ]] + frequency = "daily" + input_files = "eam.h1" + input_subdir = "archive/atm/hist" + vars = "PRECT" + + [[ land_monthly ]] + extra_vars = "landfrac" + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + #vars = "LAISHA,LAISUN" + vars = "FSH,RH2M" + ts_fmt = "cmip" + +[ilamb] +active = True +grids = '180x360_aave' +nodes = 8 +partition = "compute" +short_name = 'v2.LR.historical_0201' +ts_num_years = 2 +years = "1850:1854:2", + + [[ land_monthly ]] diff --git a/tests/integration/template_debug.cfg b/tests/integration/template_debug.cfg new file mode 100644 index 00000000..11fce44d --- /dev/null +++ b/tests/integration/template_debug.cfg @@ -0,0 +1,56 @@ +# Use this cfg to test a pull request before merging. +# 1. Copy over the contents of template_complete_run.cfg. +# 2. Delete parts that aren't needed. `output` and `www` should include `zppy_test_debug` rather than `zppy_test_complete_run`. +# 3. Iteratively debug/develop with this cfg. +# 4. Once it's working as expected, copy important changes to template_complete_run.cfg. + +[default] +case = v2.LR.historical_0201 +constraint = "#expand constraint#" +dry_run = "#expand dry_run#" +environment_commands = "#expand environment_commands#" +input = "#expand user_input#/E3SMv2/v2.LR.historical_0201" +input_subdir = archive/atm/hist +mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" +# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py +output = "#expand user_output#zppy_test_debug_output/#expand unique_id#/v2.LR.historical_0201" +partition = "#expand partition_short#" +qos = "#expand qos_short#" +www = "#expand user_www#zppy_debug_www/#expand unique_id#" + +[ts] +active = True +walltime = "00:30:00" +years = "1850:1854:2", + + [[ atm_monthly_180x360_aave ]] + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + ts_fmt = "cmip" + + [[ atm_daily_180x360_aave ]] + frequency = "daily" + input_files = "eam.h1" + input_subdir = "archive/atm/hist" + vars = "PRECT" + + [[ land_monthly ]] + extra_vars = "landfrac" + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + #vars = "LAISHA,LAISUN" + vars = "FSH,RH2M" + ts_fmt = "cmip" + +[ilamb] +active = True +grids = '180x360_aave' +nodes = 8 +partition = "#expand partition_long#" +short_name = 'v2.LR.historical_0201' +ts_num_years = 2 +years = "1850:1854:2", + + [[ land_monthly ]] diff --git a/tests/integration/template_directions.md b/tests/integration/template_directions.md index f7887e00..602f20d1 100644 --- a/tests/integration/template_directions.md +++ b/tests/integration/template_directions.md @@ -12,6 +12,25 @@ NOTE: Actually running the tests (e.g., `python -u -m unittest tests/integration can not be done from two different branches simultaneously (since files in the `zppy` directory rather than an external directory get changed). +## Debug your code with test_debug + +Follow the directions in the cfg to modify it for your testing needs. + +``` +rm -rf #expand user_www#zppy_test_debug_www//v2.LR.historical_0201 +rm -rf #expand user_output#zppy_test_debug_output//v2.LR.historical_0201/post +# Generate cfg +python tests/integration/utils.py + +# Run jobs: +zppy -c tests/integration/generated/test_debug_#expand machine#.cfg +# After they finish, check the results: +cd #expand user_output#zppy_test_debug_output//v2.LR.historical_0201/post/scripts +grep -v "OK" *status +# Nothing should print +cd - +``` + ### test_bundles ``` @@ -87,7 +106,7 @@ mkdir -p #expand expected_dir#expected_bundles/bundle_files cp -r #expand user_output#zppy_test_bundles_output//v2.LR.historical_0201/post/scripts/bundle*.bash #expand expected_dir#expected_bundles/bundle_files cd #expand expected_dir#expected_bundles # Remove the image check failures, so they don't end up in the expected files. -rm -rf #expand user_www#forsyth/zppy_test_bundles_www//v2.LR.historical_0201/image_check_failures +rm -rf image_check_failures # This file will list all the expected images. find . -type f -name '*.png' > ../image_list_expected_bundles.txt cd @@ -119,7 +138,7 @@ rm -rf #expand expected_dir#expected_complete_run cp -r #expand user_www#zppy_test_complete_run_www//v2.LR.historical_0201 #expand expected_dir#expected_complete_run cd #expand expected_dir#expected_complete_run # Remove the image check failures, so they don't end up in the expected files. -rm -rf #expand user_www#forsyth/zppy_test_complete_run_www//v2.LR.historical_0201/image_check_failures +rm -rf image_check_failures # This file will list all the expected images. find . -type f -name '*.png' > ../image_list_expected_complete_run.txt cd diff --git a/tests/integration/test_bundles.py b/tests/integration/test_bundles.py index faeeb1b1..26e5a03b 100644 --- a/tests/integration/test_bundles.py +++ b/tests/integration/test_bundles.py @@ -60,19 +60,19 @@ def test_bundles_bash_file_content(self): # Check that bundle files are correct self.assertEqual( os.system( - f"diff -bu {actual_directory}/bundle1.bash {expected_directory}/bundle1.bash" + f"diff -bu -I 'zppy_test_bundles_output/' {actual_directory}/bundle1.bash {expected_directory}/bundle1.bash" ), 0, ) self.assertEqual( os.system( - f"diff -bu {actual_directory}/bundle2.bash {expected_directory}/bundle2.bash" + f"diff -bu -I 'zppy_test_bundles_output/' {actual_directory}/bundle2.bash {expected_directory}/bundle2.bash" ), 0, ) self.assertEqual( os.system( - f"diff -bu {actual_directory}/bundle3.bash {expected_directory}/bundle3.bash" + f"diff -bu -I 'zppy_test_bundles_output/' {actual_directory}/bundle3.bash {expected_directory}/bundle3.bash" ), 0, ) diff --git a/tests/integration/utils.py b/tests/integration/utils.py index b52c5790..a526119a 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -7,7 +7,7 @@ from mache import MachineInfo from PIL import Image, ImageChops, ImageDraw -UNIQUE_ID = "unique_id" +UNIQUE_ID = "testing-updates" # Image checking ########################################################## @@ -257,7 +257,7 @@ def generate_cfgs(unified_testing=False, dry_run=False): else: expansions["dry_run"] = "False" - cfg_names = ["bundles", "complete_run"] + cfg_names = ["bundles", "complete_run", "debug"] for cfg_name in cfg_names: cfg_template = f"{git_top_level}/tests/integration/template_{cfg_name}.cfg" cfg_generated = ( From f8b0c6d4dc40ef174dd1d9d2571529087f2dc376 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 21 Feb 2024 16:04:44 -0600 Subject: [PATCH 2/3] Change unique id --- tests/integration/generated/test_bundles_chrysalis.cfg | 4 ++-- tests/integration/generated/test_complete_run_chrysalis.cfg | 4 ++-- tests/integration/generated/test_debug_chrysalis.cfg | 4 ++-- tests/integration/utils.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/integration/generated/test_bundles_chrysalis.cfg b/tests/integration/generated/test_bundles_chrysalis.cfg index 7e9a61bb..8ed7309e 100644 --- a/tests/integration/generated/test_bundles_chrysalis.cfg +++ b/tests/integration/generated/test_bundles_chrysalis.cfg @@ -7,11 +7,11 @@ input = "/lcrc/group/e3sm/ac.forsyth2/E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_bundles.py -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/testing-updates/v2.LR.historical_0201" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/testing-updates2/v2.LR.historical_0201" partition = "compute" qos = "regular" walltime = "07:00:00" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/testing-updates" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/testing-updates2" [bundle] diff --git a/tests/integration/generated/test_complete_run_chrysalis.cfg b/tests/integration/generated/test_complete_run_chrysalis.cfg index e712723c..f32880fa 100644 --- a/tests/integration/generated/test_complete_run_chrysalis.cfg +++ b/tests/integration/generated/test_complete_run_chrysalis.cfg @@ -7,10 +7,10 @@ input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/testing-updates/v2.LR.historical_0201" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/testing-updates2/v2.LR.historical_0201" partition = "debug" qos = "regular" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/testing-updates" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/testing-updates2" [climo] active = True diff --git a/tests/integration/generated/test_debug_chrysalis.cfg b/tests/integration/generated/test_debug_chrysalis.cfg index cff15475..080786de 100644 --- a/tests/integration/generated/test_debug_chrysalis.cfg +++ b/tests/integration/generated/test_debug_chrysalis.cfg @@ -13,10 +13,10 @@ input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/testing-updates/v2.LR.historical_0201" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/testing-updates2/v2.LR.historical_0201" partition = "debug" qos = "regular" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_debug_www/testing-updates" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_debug_www/testing-updates2" [ts] active = True diff --git a/tests/integration/utils.py b/tests/integration/utils.py index a526119a..55ea4940 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -7,7 +7,7 @@ from mache import MachineInfo from PIL import Image, ImageChops, ImageDraw -UNIQUE_ID = "testing-updates" +UNIQUE_ID = "testing-updates2" # Image checking ########################################################## From 1e97fd2d98496685fd519513c2e2aa17b5733e29 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 22 Feb 2024 16:22:40 -0600 Subject: [PATCH 3/3] Restore unique_id --- tests/integration/generated/test_bundles_chrysalis.cfg | 4 ++-- tests/integration/generated/test_complete_run_chrysalis.cfg | 4 ++-- tests/integration/generated/test_debug_chrysalis.cfg | 4 ++-- tests/integration/utils.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/integration/generated/test_bundles_chrysalis.cfg b/tests/integration/generated/test_bundles_chrysalis.cfg index 8ed7309e..95068a36 100644 --- a/tests/integration/generated/test_bundles_chrysalis.cfg +++ b/tests/integration/generated/test_bundles_chrysalis.cfg @@ -7,11 +7,11 @@ input = "/lcrc/group/e3sm/ac.forsyth2/E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_bundles.py -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/testing-updates2/v2.LR.historical_0201" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/unique_id/v2.LR.historical_0201" partition = "compute" qos = "regular" walltime = "07:00:00" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/testing-updates2" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/unique_id" [bundle] diff --git a/tests/integration/generated/test_complete_run_chrysalis.cfg b/tests/integration/generated/test_complete_run_chrysalis.cfg index f32880fa..bb222b8d 100644 --- a/tests/integration/generated/test_complete_run_chrysalis.cfg +++ b/tests/integration/generated/test_complete_run_chrysalis.cfg @@ -7,10 +7,10 @@ input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/testing-updates2/v2.LR.historical_0201" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201" partition = "debug" qos = "regular" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/testing-updates2" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/unique_id" [climo] active = True diff --git a/tests/integration/generated/test_debug_chrysalis.cfg b/tests/integration/generated/test_debug_chrysalis.cfg index 080786de..ae2777fe 100644 --- a/tests/integration/generated/test_debug_chrysalis.cfg +++ b/tests/integration/generated/test_debug_chrysalis.cfg @@ -13,10 +13,10 @@ input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/testing-updates2/v2.LR.historical_0201" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/unique_id/v2.LR.historical_0201" partition = "debug" qos = "regular" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_debug_www/testing-updates2" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_debug_www/unique_id" [ts] active = True diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 55ea4940..3cbad951 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -7,7 +7,7 @@ from mache import MachineInfo from PIL import Image, ImageChops, ImageDraw -UNIQUE_ID = "testing-updates2" +UNIQUE_ID = "unique_id" # Image checking ##########################################################