Merge pull request #90 from podaac/develop

Develop in main. closes #89 and closes #62 and closes #62

CHANGELOG.md

@@ -3,6 +3,11 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
+## [1.10.2]
+### Fixed
+- Fixed an issue where using a default global bounding box prevented download of data that didn't use the horizontal spatial domain [87](https://github.com/podaac/data-subscriber/issues/87)
+- Fixed limit option not being respected. [86](https://github.com/podaac/data-subscriber/issues/86)
+
 ## [1.10.1]
 ### Fixed
 - Support for SHA-256 and SHA-512 checksums

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "podaac-data-subscriber"
-version = "1.10.1"
+version = "1.10.2"
 description = "PO.DAAC Data Subscriber Command Line Tool"
 authors = ["PO.DAAC <podaac@podaac.jpl.nasa.gov>"]
 readme = "README.md"

subscriber/podaac_access.py

@@ -21,7 +21,7 @@
 import tenacity
 from datetime import datetime
 
-__version__ = "1.10.1"
+__version__ = "1.10.2"
 extensions = [".nc", ".h5", ".zip", ".tar.gz"]
 edl = "urs.earthdata.nasa.gov"
 cmr = "cmr.earthdata.nasa.gov"
@@ -129,16 +129,17 @@ def refresh_token(old_token: str, client_id: str):
 
 
 def validate(args):
- bounds = args.bbox.split(',')
- if len(bounds) != 4:
- raise ValueError(
- "Error parsing '--bounds': " + args.bbox + ". Format is W Longitude,S Latitude,E Longitude,N Latitude without spaces ") # noqa E501
- for b in bounds:
- try:
- float(b)
- except ValueError:
+ if args.bbox is not None:
+ bounds = args.bbox.split(',')
+ if len(bounds) != 4:
  raise ValueError(
  "Error parsing '--bounds': " + args.bbox + ". Format is W Longitude,S Latitude,E Longitude,N Latitude without spaces ") # noqa E501
+ for b in bounds:
+ try:
+ float(b)
+ except ValueError:
+ raise ValueError(
+ "Error parsing '--bounds': " + args.bbox + ". Format is W Longitude,S Latitude,E Longitude,N Latitude without spaces ") # noqa E501
 
  if args.startDate:
  try:

subscriber/podaac_data_downloader.py

@@ -14,16 +14,13 @@
 __version__ = pa.__version__
 
 page_size = 2000
-
 edl = pa.edl
 cmr = pa.cmr
 token_url = pa.token_url
 
-
 # The lines below are to get the IP address. You can make this static and
 # assign a fixed value to the IPAddr variable
 
-
 def parse_cycles(cycle_input):
  # if cycle_input is None:
  # return None
@@ -66,14 +63,14 @@ def create_parser():
  help="The ISO date time before which data should be retrieved. For Example, --start-date 2021-01-14T00:00:00Z") # noqa E501
  parser.add_argument("-ed", "--end-date", required=False, dest="endDate",
  help="The ISO date time after which data should be retrieved. For Example, --end-date 2021-01-14T00:00:00Z") # noqa E501
- 
+
  # Adding optional arguments
  parser.add_argument("-f", "--force", dest="force", action="store_true", help = "Flag to force downloading files that are listed in CMR query, even if the file exists and checksum matches") # noqa E501
 
  # spatiotemporal arguments
  parser.add_argument("-b", "--bounds", dest="bbox",
  help="The bounding rectangle to filter result in. Format is W Longitude,S Latitude,E Longitude,N Latitude without spaces. Due to an issue with parsing arguments, to use this command, please use the -b=\"-180,-90,180,90\" syntax when calling from the command line. Default: \"-180,-90,180,90\".",
- default="-180,-90,180,90") # noqa E501
+ default=None) # noqa E501
 
  # Arguments for how data are stored locally - much processing is based on
  # the underlying directory structure (e.g. year/Day-of-year)
@@ -101,9 +98,8 @@ def create_parser():
  parser.add_argument("-p", "--provider", dest="provider", default='POCLOUD',
  help="Specify a provider for collection search. Default is POCLOUD.") # noqa E501
 
- parser.add_argument("--limit", dest="limit", default='2000', type=int,
- help="Integer limit for number of granules to download. Useful in testing. Defaults to " + str(
- page_size)) # noqa E501
+ parser.add_argument("--limit", dest="limit", default=None, type=int,
+ help="Integer limit for number of granules to download. Useful in testing. Defaults to no limit.") # noqa E501
 
  return parser
 
@@ -138,8 +134,9 @@ def run(args=None):
  process_cmd = args.process_cmd
  data_path = args.outputDirectory
 
- if args.limit is not None:
- page_size = args.limit
+ download_limit = None
+ if args.limit is not None and args.limit > 0:
+ download_limit = args.limit
 
  if args.offset:
  ts_shift = timedelta(hours=int(args.offset))
@@ -158,9 +155,6 @@ def run(args=None):
  logging.info("NOTE: Making new data directory at " + data_path + "(This is the first run.)")
  makedirs(data_path, exist_ok=True)
 
- # Change this to whatever extent you need. Format is W Longitude,S Latitude,E Longitude,N Latitude
- bounding_extent = args.bbox
-
  if search_cycles is not None:
  cmr_cycles = search_cycles
  params = [
@@ -169,7 +163,6 @@ def run(args=None):
  ('provider', provider),
  ('ShortName', short_name),
  ('token', token),
- ('bounding_box', bounding_extent),
  ]
  for v in cmr_cycles:
  params.append(("cycle[]", v))
@@ -179,20 +172,20 @@ def run(args=None):
  else:
  temporal_range = pa.get_temporal_range(start_date_time, end_date_time,
  datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")) # noqa E501
- params = {
- 'page_size': page_size,
- 'sort_key': "-start_date",
- 'provider': provider,
- 'ShortName': short_name,
- 'temporal': temporal_range,
- 'token': token,
- 'bounding_box': bounding_extent,
- }
+ params = [
+ ('page_size', page_size),
+ ('sort_key', "-start_date"),
+ ('provider', provider),
+ ('ShortName', short_name),
+ ('temporal', temporal_range),
+ ]
  if args.verbose:
  logging.info("Temporal Range: " + temporal_range)
 
  if args.verbose:
  logging.info("Provider: " + provider)
+ if args.bbox is not None:
+ params.append(('bounding_box', args.bbox))
 
  # If 401 is raised, refresh token and try one more time
  try:
@@ -247,6 +240,8 @@ def run(args=None):
  # Make this a non-verbose message
  # if args.verbose:
  logging.info("Found " + str(len(downloads)) + " total files to download")
+ if download_limit:
+ logging.info("Limiting downloads to " + str(args.limit) + " total files")
  if args.verbose:
  logging.info("Downloading files with extensions: " + str(extensions))
 
@@ -277,6 +272,11 @@ def run(args=None):
  pa.process_file(process_cmd, output_path, args)
  logging.info(str(datetime.now()) + " SUCCESS: " + f)
  success_cnt = success_cnt + 1
+
+ #if limit is set and we're at or over it, stop downloading
+ if download_limit and success_cnt >= download_limit:
+ break
+
  except Exception:
  logging.warning(str(datetime.now()) + " FAILURE: " + f, exc_info=True)
  failure_cnt = failure_cnt + 1

subscriber/podaac_data_subscriber.py

@@ -73,7 +73,7 @@ def create_parser():
  default=False) # noqa E501
  parser.add_argument("-b", "--bounds", dest="bbox",
  help="The bounding rectangle to filter result in. Format is W Longitude,S Latitude,E Longitude,N Latitude without spaces. Due to an issue with parsing arguments, to use this command, please use the -b=\"-180,-90,180,90\" syntax when calling from the command line. Default: \"-180,-90,180,90\".",
- default="-180,-90,180,90") # noqa E501
+ default=None) # noqa E501
 
  # Arguments for how data are stored locally - much processing is based on
  # the underlying directory structure (e.g. year/Day-of-year)
@@ -178,46 +178,36 @@ def run(args=None):
  else:
  logging.warning("No .update__" + short_name + " in the data directory. (Is this the first run?)")
 
- # Change this to whatever extent you need. Format is W Longitude,S Latitude,E Longitude,N Latitude
- bounding_extent = args.bbox
-
- # There are several ways to query for CMR updates that occured during a given timeframe. Read on in the CMR Search documentation:
- # * https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html#c-with-new-granules (Collections)
- # * https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html#c-with-revised-granules (Collections)
- # * https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html#g-production-date (Granules)
- # * https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html#g-created-at (Granules)
- # The `created_at` parameter works for our purposes. It's a granule search parameter that returns the records ingested since the input timestamp.
-
  if defined_time_range:
  # if(data_since):
  temporal_range = pa.get_temporal_range(start_date_time, end_date_time,
  datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")) # noqa E501
 
- params = {
- 'page_size': page_size,
- 'sort_key': "-start_date",
- 'provider': provider,
- 'ShortName': short_name,
- 'updated_since': data_within_last_timestamp,
- 'token': token,
- 'bounding_box': bounding_extent,
- }
+ params = [
+ ('page_size',page_size),
+ ('sort_key', "-start_date"),
+ ('provider', provider),
+ ('ShortName', short_name),
+ ('updated_since', data_within_last_timestamp),
+ ('token', token),
+ ]
 
  if defined_time_range:
- params = {
- 'page_size': page_size,
- 'sort_key': "-start_date",
- 'provider': provider,
- 'updated_since': data_within_last_timestamp,
- 'ShortName': short_name,
- 'temporal': temporal_range,
- 'token': token,
- 'bounding_box': bounding_extent,
- }
-
+ params = [
+ ('page_size', page_size),
+ ('sort_key', "-start_date"),
+ ('provider', provider),
+ ('updated_since', data_within_last_timestamp),
+ ('ShortName', short_name),
+ ('temporal', temporal_range),
+ ('token', token),
+ ]
  if args.verbose:
  logging.info("Temporal Range: " + temporal_range)
 
+ if args.bbox is not None:
+ params.append(('bounding_box', args.bbox))
+
  if args.verbose:
  logging.info("Provider: " + provider)
  logging.info("Updated Since: " + data_within_last_timestamp)

tests/test_downloader_regression.py

@@ -12,6 +12,19 @@ def create_downloader_args(args):
  args2 = parser.parse_args(args)
  return args2
 
+
+#Test the downlaoder on MUR25 data for start/stop/, yyyy/mmm/dd dir structure,
+# and offset. Running it a second time to ensure it downlaods the files again-
+# the downloader doesn't care about updates.
+@pytest.mark.regression
+def test_downloader_limit_MUR():
+ shutil.rmtree('./MUR25-JPL-L4-GLOB-v04.2', ignore_errors=True)
+ args2 = create_downloader_args('-c MUR25-JPL-L4-GLOB-v04.2 -d ./MUR25-JPL-L4-GLOB-v04.2 -sd 2020-01-01T00:00:00Z -ed 2020-01-30T00:00:00Z --limit 1'.split())
+ pdd.run(args2)
+ # count number of files downloaded...
+ assert len([name for name in os.listdir('./MUR25-JPL-L4-GLOB-v04.2') if os.path.isfile('./MUR25-JPL-L4-GLOB-v04.2/' + name)])==1
+ shutil.rmtree('./MUR25-JPL-L4-GLOB-v04.2')
+
 #Test the downlaoder on MUR25 data for start/stop/, yyyy/mmm/dd dir structure,
 # and offset. Running it a second time to ensure it downlaods the files again-
 # the downloader doesn't care about updates.