From 483344ba8bab0d2577e68912251cccaee74d6de6 Mon Sep 17 00:00:00 2001 From: mike-gangl Date: Tue, 26 Apr 2022 13:06:29 -0700 Subject: [PATCH 1/2] added programmatice regression testing. currently relies on a valid .netrc file, refactoring might be needed to manually add a user/password to the CMR/TEA downloads --- pyproject.toml | 4 ++ subscriber/podaac_data_downloader.py | 8 ++-- subscriber/podaac_data_subscriber.py | 10 ++-- tests/MANUAL.md | 8 ++-- tests/test_downloader_regression.py | 35 ++++++++++++++ tests/test_subscriber_regression.py | 71 ++++++++++++++++++++++++++++ 6 files changed, 124 insertions(+), 12 deletions(-) create mode 100644 tests/test_downloader_regression.py create mode 100644 tests/test_subscriber_regression.py diff --git a/pyproject.toml b/pyproject.toml index 374b58c..aed08bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,3 +4,7 @@ requires = [ "wheel" ] build-backend = "setuptools.build_meta" +[tool.pytest.ini_options] +markers = [ + "regression: marks a test as a regression, requires netrc file (deselect with '-m \"not regresion\"')" +] diff --git a/subscriber/podaac_data_downloader.py b/subscriber/podaac_data_downloader.py index f2c1420..19c15ab 100644 --- a/subscriber/podaac_data_downloader.py +++ b/subscriber/podaac_data_downloader.py @@ -106,9 +106,10 @@ def create_parser(): return parser -def run(): - parser = create_parser() - args = parser.parse_args() +def run(args=None): + if args is None: + parser = create_parser() + args = parser.parse_args() try: pa.validate(args) @@ -274,7 +275,6 @@ def run(): logging.info("Files Failed to download:" + str(failure_cnt) + "\n") pa.delete_token(token_url, token) logging.info("END \n\n") - exit(0) def main(): diff --git a/subscriber/podaac_data_subscriber.py b/subscriber/podaac_data_subscriber.py index ad3549e..0de7bc8 100755 --- a/subscriber/podaac_data_subscriber.py +++ b/subscriber/podaac_data_subscriber.py @@ -188,9 +188,11 @@ def make_checksum(file_path, algorithm): return hash.hexdigest() -def run(): - parser = create_parser() - args = parser.parse_args() +def run(args=None): + if args is None: + parser = create_parser() + args = parser.parse_args() + try: pa.validate(args) @@ -405,7 +407,7 @@ def run(): logging.info("Skipped Files: " + str(skip_cnt)) pa.delete_token(token_url, token) logging.info("END\n\n") - exit(0) + #exit(0) def main(): diff --git a/tests/MANUAL.md b/tests/MANUAL.md index 52a1e1e..6fe9608 100644 --- a/tests/MANUAL.md +++ b/tests/MANUAL.md @@ -3,7 +3,7 @@ ## Subscriber -### Test 1 +### Test 1 - added to test_regression.py use to test: * download to `this` directory. * download using only 'enddate' @@ -29,7 +29,7 @@ ls -rth .update__ECCO_L4_ATM_STATE_05DEG_DAILY_V4R4 .update__ECCO_L4_ATM_STATE_05DEG_DAILY_V4R4 ``` -### Test 2 +### Test 2 - added to regression test use to test: * cycle based directory layouts * Bounding box limiting search results @@ -54,7 +54,7 @@ JASON_CS_S6A_L2_ALT_LR_STD_OST_NRT_F/ ``` -### Test 3 +### Test 3 -- added to regression, but not the .update file log message portion use to test: * offset Usage * start/end date is working @@ -137,7 +137,7 @@ MUR25-JPL-L4-GLOB-v04.2/ 4 directories, 2 files ``` - +### Test 1 Download by cycle ``` rm -r JASON_CS_S6A_L2_ALT_LR_STD_OST_NRT_F diff --git a/tests/test_downloader_regression.py b/tests/test_downloader_regression.py new file mode 100644 index 0000000..a97941a --- /dev/null +++ b/tests/test_downloader_regression.py @@ -0,0 +1,35 @@ +import pytest +import os +from os.path import exists +from subscriber import podaac_data_downloader as pdd +import shutil + +# REGRESSION TEST CURRENTLY REQUIRES A .NETRC file for CMR/Data Download + +def create_downloader_args(args): + parser = pdd.create_parser() + args2 = parser.parse_args(args) + return args2 + +#Test the downlaoder on MUR25 data for start/stop/, yyyy/mmm/dd dir structure, +# and offset. Running it a second time to ensure it downlaods the files again- +# the downloader doesn't care about updates. +@pytest.mark.regression +def test_downloader_MUR(): + shutil.rmtree('./MUR25-JPL-L4-GLOB-v04.2', ignore_errors=True) + args2 = create_downloader_args('-c MUR25-JPL-L4-GLOB-v04.2 -d ./MUR25-JPL-L4-GLOB-v04.2 -sd 2020-01-01T00:00:00Z -ed 2020-01-02T00:00:00Z -dymd --offset 4'.split()) + pdd.run(args2) + assert exists('./MUR25-JPL-L4-GLOB-v04.2/2020/01/01/20200101090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc') + assert exists('./MUR25-JPL-L4-GLOB-v04.2/2020/01/02/20200102090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc') + t1 = os.path.getmtime('./MUR25-JPL-L4-GLOB-v04.2/2020/01/01/20200101090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc') + t2 = os.path.getmtime('./MUR25-JPL-L4-GLOB-v04.2/2020/01/02/20200102090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc') + + # this part of the test should not re-download the files unless the --force + # option is used. Currently that's not implemented in Downloader, so we'll + # have to update this when that is implemented (that is, the t1/t2 should + # be equal to the gettime of the file) + pdd.run(args2) + assert t1 != os.path.getmtime('./MUR25-JPL-L4-GLOB-v04.2/2020/01/01/20200101090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc') + assert t2 != os.path.getmtime('./MUR25-JPL-L4-GLOB-v04.2/2020/01/02/20200102090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc') + + shutil.rmtree('./MUR25-JPL-L4-GLOB-v04.2') diff --git a/tests/test_subscriber_regression.py b/tests/test_subscriber_regression.py new file mode 100644 index 0000000..07dc2f8 --- /dev/null +++ b/tests/test_subscriber_regression.py @@ -0,0 +1,71 @@ +import pytest +import os +from os.path import exists +from subscriber import podaac_data_subscriber as pds +from subscriber import podaac_data_downloader as pdd +import shutil + +# REGRESSION TEST CURRENTLY REQUIRES A .NETRC file for CMR/Data Download +# +def create_args(args): + parser = pds.create_parser() + args2 = parser.parse_args(args) + return args2 + +# Test to download ECCO data by start/stop date and put it in the year/doy dir +# structure. +@pytest.mark.regression +def test_subscriber_ecco_only_enddate(): + args2 = create_args('-c ECCO_L4_ATM_STATE_05DEG_DAILY_V4R4 -ed 1992-01-03T00:00:00Z -d ./ECCO_L4_ATM_STATE_05DEG_DAILY_V4R4 -dydoy'.split()) + pds.run(args2) + assert exists('./ECCO_L4_ATM_STATE_05DEG_DAILY_V4R4/1992/001/ATM_SURFACE_TEMP_HUM_WIND_PRES_day_mean_1992-01-01_ECCO_V4r4_latlon_0p50deg.nc') + assert exists('./ECCO_L4_ATM_STATE_05DEG_DAILY_V4R4/1992/002/ATM_SURFACE_TEMP_HUM_WIND_PRES_day_mean_1992-01-02_ECCO_V4r4_latlon_0p50deg.nc') + assert exists('./ECCO_L4_ATM_STATE_05DEG_DAILY_V4R4/1992/003/ATM_SURFACE_TEMP_HUM_WIND_PRES_day_mean_1992-01-03_ECCO_V4r4_latlon_0p50deg.nc') + shutil.rmtree('./ECCO_L4_ATM_STATE_05DEG_DAILY_V4R4') + +# test to download S6 data by start/stop time, and bbox, and put it in the +# cycle based directory structure +@pytest.mark.regression +def test_subscriber_cycle_bbox(): + args2 = create_args('-c JASON_CS_S6A_L2_ALT_LR_STD_OST_NRT_F -d ./JASON_CS_S6A_L2_ALT_LR_STD_OST_NRT_F -dc -sd 2022-01-01T00:00:00Z -ed 2022-01-02T00:00:00Z -b=-20,-20,20,20'.split()) + pds.run(args2) + assert exists('./JASON_CS_S6A_L2_ALT_LR_STD_OST_NRT_F/c0042/S6A_P4_2__LR_STD__NR_042_071_20211231T232728_20220101T012144_F04.nc') + assert exists('./JASON_CS_S6A_L2_ALT_LR_STD_OST_NRT_F/c0042/S6A_P4_2__LR_STD__NR_042_082_20220101T090557_20220101T104242_F04.nc') + assert exists('./JASON_CS_S6A_L2_ALT_LR_STD_OST_NRT_F/c0042/S6A_P4_2__LR_STD__NR_042_083_20220101T104242_20220101T123506_F04.nc') + assert exists('./JASON_CS_S6A_L2_ALT_LR_STD_OST_NRT_F/c0042/S6A_P4_2__LR_STD__NR_042_095_20220101T215702_20220101T234905_F04.nc') + assert exists('./JASON_CS_S6A_L2_ALT_LR_STD_OST_NRT_F/c0042/S6A_P4_2__LR_STD__NR_042_097_20220101T234905_20220102T014431_F04.nc') + assert exists('./JASON_CS_S6A_L2_ALT_LR_STD_OST_NRT_F/.update__JASON_CS_S6A_L2_ALT_LR_STD_OST_NRT_F') + shutil.rmtree('./JASON_CS_S6A_L2_ALT_LR_STD_OST_NRT_F') + +# Test to download MUR25 data by start/stop, put it in yyyy/mm/dd dir structure, +# using the offset so it aligns with the right day in the filename. +# +# Test will run it again, to ensure that the files are not re-downlaoded, that +# is, they have the same modified time before/after the second run +@pytest.mark.regression +def test_subscriber_MUR_update_file_no_redownload(): + try: + os.remove('MUR25-JPL-L4-GLOB-v04.2/.update') + except OSError as e: + print("Expecting this...") + try: + os.remove('MUR25-JPL-L4-GLOB-v04.2/..update__MUR25-JPL-L4-GLOB-v04.2') + except OSError as e: + print("Expecting this...") + + args2 = create_args('-c MUR25-JPL-L4-GLOB-v04.2 -d ./MUR25-JPL-L4-GLOB-v04.2 -sd 2020-01-01T00:00:00Z -ed 2020-01-02T00:00:00Z -dymd --offset 4'.split()) + pds.run(args2) + assert exists('./MUR25-JPL-L4-GLOB-v04.2/2020/01/01/20200101090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc') + assert exists('./MUR25-JPL-L4-GLOB-v04.2/2020/01/02/20200102090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc') + assert exists('./MUR25-JPL-L4-GLOB-v04.2/.update__MUR25-JPL-L4-GLOB-v04.2') + t1 = os.path.getmtime('./MUR25-JPL-L4-GLOB-v04.2/2020/01/01/20200101090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc') + t2 = os.path.getmtime('./MUR25-JPL-L4-GLOB-v04.2/2020/01/02/20200102090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc') + + # Compare another run to existing times to ensure it didn't redownload the file + pds.run(args2) + assert exists('./MUR25-JPL-L4-GLOB-v04.2/2020/01/01/20200101090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc') + assert exists('./MUR25-JPL-L4-GLOB-v04.2/2020/01/02/20200102090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc') + assert exists('./MUR25-JPL-L4-GLOB-v04.2/.update__MUR25-JPL-L4-GLOB-v04.2') + assert t1 == os.path.getmtime('./MUR25-JPL-L4-GLOB-v04.2/2020/01/01/20200101090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc') + assert t2 == os.path.getmtime('./MUR25-JPL-L4-GLOB-v04.2/2020/01/02/20200102090000-JPL-L4_GHRSST-SSTfnd-MUR25-GLOB-v02.0-fv04.2.nc') + shutil.rmtree('./MUR25-JPL-L4-GLOB-v04.2') From 62b715d5f21fef05ce15deeb860a231889fe385b Mon Sep 17 00:00:00 2001 From: mike-gangl <59702631+mike-gangl@users.noreply.github.com> Date: Tue, 26 Apr 2022 13:11:01 -0700 Subject: [PATCH 2/2] Update python-app.yml --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 8b214f6..01e158f 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -33,4 +33,4 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest run: | - pytest + pytest -m "not regression"