Merge pull request #127 from ASFHyP3/develop

Release: Container Methods

.github/workflows/notify-downstream.yml

@@ -0,0 +1,28 @@
+name: Notify Downstream of New Release
+
+on:
+  release:
+    types:
+      - released
+
+jobs:
+  update-docs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Bump SDK version in HyP3 docs
+        uses: benc-uk/workflow-dispatch@v1.1
+        with:
+          workflow: Update HyP3 SDK version
+          token: ${{ secrets.TOOLS_BOT_PAK }}
+          repo: ASFHyP3/hyp3-docs
+          ref: main
+          inputs: '{"sdk_version": "${{ github.event.release.tag_name }}"}'
+
+      - name: Tweet release notes
+        uses: benc-uk/workflow-dispatch@v1.1
+        with:
+          workflow: Propose Tweet
+          token: ${{ secrets.TOOLS_BOT_PAK }}
+          repo: ASFHyP3/hyp3-docs
+          ref: develop
+          inputs: '{"message": "## ${{ github.event.release.name }}\n${{ github.event.release.body }}"}'

.github/workflows/release.yml

@@ -15,25 +15,11 @@ jobs:
           token: ${{ secrets.TOOLS_BOT_PAK }}
 
       - name: Create Release
-        uses: docker://antonyurchenko/git-release:latest
+        uses: docker://antonyurchenko/git-release:v3.5.0
         env:
           GITHUB_TOKEN: ${{ secrets.TOOLS_BOT_PAK }}
-          ALLOW_TAG_PREFIX: "true"
           RELEASE_NAME_PREFIX: "HyP3 SDK "
-
-      - name: Get SDK version number
-        id: get_version
-        run: |
-          echo ::set-output name=version_tag::${GITHUB_REF#refs/tags/}
-
-      - name: Bump SDK version in HyP3 docs
-        uses: benc-uk/workflow-dispatch@v1.1
-        with:
-          workflow: Update HyP3 SDK version
-          token: ${{ secrets.TOOLS_BOT_PAK }}
-          repo: ASFHyP3/hyp3-docs
-          ref: main
-          inputs: '{"sdk_version": "${{ steps.get_version.outputs.version_tag }}"}'
+          ALLOW_TAG_PREFIX: true
 
       - name: Attempt fast-forward develop from main
         run: |

CHANGELOG.md

@@ -6,6 +6,25 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) 
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.2.0](https://github.com/ASFHyP3/hyp3-sdk/compare/v1.1.3...v1.2.0)
+
+### Added
+- `Job` class now has a `logs` attribute containing links to job log files
+- Added missing [container methods](https://docs.python.org/3/reference/datamodel.html#emulating-container-types) 
+  - batches are now subscriptable: `batch[0]`
+  - jobs can be searched for in batches:`job in batch`
+  - jobs can be deleted from batches: `del batch[0]`
+  - batches can be reversed now using the `reversed()` function
+- `find_jobs()` now accepts datetimes with no timezone info and defaults to UTC.
+
+### Removed
+- `FoundZeroJobs` warning from `find_jobs()`
+
+### Fixed
+- [#92](https://github.com/ASFHyP3/hyp3-sdk/issues/92) -- `ImportError` being
+  raised when showing a progress bar because `ipywidgets` may not always be
+  installed when running in a Jupyter kernel
+
 ## [1.1.3](https://github.com/ASFHyP3/hyp3-sdk/compare/v1.1.2...v1.1.3)
 
 ### Added

hyp3_sdk/hyp3.py

@@ -1,18 +1,15 @@
 import math
 import time
-import warnings
 from datetime import datetime, timezone
 from functools import singledispatchmethod
 from getpass import getpass
 from typing import List, Literal, Optional, Union
 from urllib.parse import urljoin
 
-from tqdm.auto import tqdm
-
 import hyp3_sdk
 from hyp3_sdk.exceptions import HyP3Error, _raise_for_hyp3_status
 from hyp3_sdk.jobs import Batch, Job
-from hyp3_sdk.util import get_authenticated_session
+from hyp3_sdk.util import get_authenticated_session, get_tqdm_progress_bar
 
 HYP3_PROD = 'https://hyp3-api.asf.alaska.edu'
 HYP3_TEST = 'https://hyp3-test-api.asf.alaska.edu'
@@ -64,7 +61,7 @@ def find_jobs(self, start: Optional[datetime] = None, end: Optional[datetime] =
             if param_value is not None:
                 if isinstance(param_value, datetime):
                     if param_value.tzinfo is None:
-                        param_value.replace(tzinfo=timezone.utc)
+                        param_value = param_value.replace(tzinfo=timezone.utc)
                     param_value = param_value.isoformat(timespec='seconds')
 
                 params[param_name] = param_value
@@ -79,8 +76,6 @@ def find_jobs(self, start: Optional[datetime] = None, end: Optional[datetime] =
             _raise_for_hyp3_status(response)
             jobs.extend([Job.from_dict(job) for job in response.json()['jobs']])
 
-        if not jobs:
-            warnings.warn('Found zero jobs', UserWarning)
         return Batch(jobs)
 
     def get_job_by_id(self, job_id: str) -> Job:
@@ -114,6 +109,7 @@ def watch(self, job_or_batch: Union[Batch, Job], timeout: int = 10800,
 
     @watch.register
     def _watch_batch(self, batch: Batch, timeout: int = 10800, interval: Union[int, float] = 60) -> Batch:
+        tqdm = get_tqdm_progress_bar()
         iterations_until_timeout = math.ceil(timeout / interval)
         bar_format = '{l_bar}{bar}| {n_fmt}/{total_fmt} [{postfix[0]}]'
         with tqdm(total=len(batch), bar_format=bar_format, postfix=[f'timeout in {timeout} s']) as progress_bar:
@@ -135,6 +131,7 @@ def _watch_batch(self, batch: Batch, timeout: int = 10800, interval: Union[int,
 
     @watch.register
     def _watch_job(self, job: Job, timeout: int = 10800, interval: Union[int, float] = 60) -> Job:
+        tqdm = get_tqdm_progress_bar()
         iterations_until_timeout = math.ceil(timeout / interval)
         bar_format = '{n_fmt}/{total_fmt} [{postfix[0]}]'
         with tqdm(total=1, bar_format=bar_format, postfix=[f'timeout in {timeout} s']) as progress_bar:

hyp3_sdk/jobs.py

@@ -6,15 +6,16 @@
 from dateutil import tz
 from dateutil.parser import parse as parse_date
 from requests import HTTPError
-from tqdm.auto import tqdm
 
 from hyp3_sdk.exceptions import HyP3SDKError
-from hyp3_sdk.util import download_file
+from hyp3_sdk.util import download_file, get_tqdm_progress_bar
 
 
 # TODO: actually looks like a good candidate for a dataclass (python 3.7+)
 #       https://docs.python.org/3/library/dataclasses.html
 class Job:
+    _attributes_for_resubmit = {'name', 'job_parameters', 'job_type'}
+
     def __init__(
             self,
             job_type: str,
@@ -25,6 +26,7 @@ def __init__(
             name: Optional[str] = None,
             job_parameters: Optional[dict] = None,
             files: Optional[List] = None,
+            logs: Optional[List] = None,
             browse_images: Optional[List] = None,
             thumbnail_images: Optional[List] = None,
             expiration_time: Optional[datetime] = None
@@ -37,6 +39,7 @@ def __init__(
         self.name = name
         self.job_parameters = job_parameters
         self.files = files
+        self.logs = logs
         self.browse_images = browse_images
         self.thumbnail_images = thumbnail_images
         self.expiration_time = expiration_time
@@ -62,30 +65,27 @@ def from_dict(input_dict: dict):
             name=input_dict.get('name'),
             job_parameters=input_dict.get('job_parameters'),
             files=input_dict.get('files'),
+            logs=input_dict.get('logs'),
             browse_images=input_dict.get('browse_images'),
             thumbnail_images=input_dict.get('thumbnail_images'),
             expiration_time=expiration_time
         )
 
     def to_dict(self, for_resubmit: bool = False):
-        job_dict = {
-            'job_type': self.job_type,
-        }
+        job_dict = {}
+        if for_resubmit:
+            keys_to_process = Job._attributes_for_resubmit
+        else:
+            keys_to_process = vars(self).keys()
 
-        for key in ['name', 'job_parameters']:
+        for key in keys_to_process:
             value = self.__getattribute__(key)
             if value is not None:
-                job_dict[key] = value
-
-        if not for_resubmit:
-            for key in ['files', 'browse_images', 'thumbnail_images', 'job_id', 'status_code', 'user_id',
-                        'expiration_time', 'request_time']:
-                value = self.__getattribute__(key)
-                if value is not None:
-                    if isinstance(value, datetime):
-                        job_dict[key] = value.isoformat(timespec='seconds')
-                    else:
-                        job_dict[key] = value
+                if isinstance(value, datetime):
+                    job_dict[key] = value.isoformat(timespec='seconds')
+                else:
+                    job_dict[key] = value
+
         return job_dict
 
     def succeeded(self) -> bool:
@@ -152,12 +152,39 @@ def __add__(self, other: Union[Job, 'Batch']):
         else:
             raise TypeError(f"unsupported operand type(s) for +: '{type(self)}' and '{type(other)}'")
 
+    def __iadd__(self, other: Union[Job, 'Batch']):
+        if isinstance(other, Batch):
+            self.jobs += other.jobs
+        elif isinstance(other, Job):
+            self.jobs += [other]
+        else:
+            raise TypeError(f"unsupported operand type(s) for +=: '{type(self)}' and '{type(other)}'")
+        return self
+
     def __iter__(self):
         return iter(self.jobs)
 
     def __len__(self):
         return len(self.jobs)
 
+    def __contains__(self, job: Job):
+        return job in self.jobs
+
+    def __delitem__(self, job: Job):
+        self.jobs.pop(job)
+        return self
+
+    def __getitem__(self, index: int):
+        return self.jobs[index]
+
+    def __setitem__(self, index: int, job: Job):
+        self.jobs[index] = job
+        return self
+
+    def __reverse__(self):
+        for job in self.jobs[::-1]:
+            yield job
+
     def __repr__(self):
         reprs = ", ".join([job.__repr__() for job in self.jobs])
         return f'Batch([{reprs}])'
@@ -200,6 +227,7 @@ def download_files(self, location: Union[Path, str] = '.', create: bool = True)
         Returns: list of Path objects to downloaded files
         """
         downloaded_files = []
+        tqdm = get_tqdm_progress_bar()
         for job in tqdm(self.jobs):
             try:
                 downloaded_files.extend(job.download_files(location, create))

hyp3_sdk/util.py

@@ -4,7 +4,6 @@
 
 import requests
 from requests.adapters import HTTPAdapter
-from tqdm.auto import tqdm
 from urllib3.util.retry import Retry
 
 import hyp3_sdk
@@ -14,6 +13,16 @@
            '&redirect_uri=https://auth.asf.alaska.edu/login&app_type=401'
 
 
+def get_tqdm_progress_bar():
+    try:
+        # https://github.com/ASFHyP3/hyp3-sdk/issues/92
+        import ipywidgets  # noqa: F401
+        from tqdm.auto import tqdm
+    except ImportError:
+        from tqdm.std import tqdm
+    return tqdm
+
+
 def get_authenticated_session(username: str, password: str) -> requests.Session:
     """Log into HyP3 using credentials for `urs.earthdata.nasa.gov` from either the provided
      credentials or a `.netrc` file.
@@ -65,6 +74,7 @@ def download_file(url: str, filepath: Union[Path, str], chunk_size=None, retries
 
     with session.get(url, stream=True) as s:
         s.raise_for_status()
+        tqdm = get_tqdm_progress_bar()
         with tqdm.wrapattr(open(filepath, "wb"), 'write', miniters=1, desc=filepath.name,
                            total=int(s.headers.get('content-length', 0))) as f:
             for chunk in s.iter_content(chunk_size=chunk_size):

tests/test_hyp3.py

@@ -1,7 +1,6 @@
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from urllib.parse import urljoin
 
-import pytest
 import responses
 
 import hyp3_sdk
@@ -42,8 +41,7 @@ def test_find_jobs(get_mock_job):
     batch = api.find_jobs()
     assert len(batch) == 3
 
-    with pytest.warns(UserWarning):
-        batch = api.find_jobs()
+    batch = api.find_jobs()
     assert len(batch) == 0
 
 
@@ -72,6 +70,32 @@ def test_find_jobs_paging(get_mock_job):
     assert 'next' in responses.calls[1].request.params
 
 
+@responses.activate
+def test_find_jobs_start():
+    api = HyP3()
+    responses.add(responses.GET, urljoin(api.url, '/jobs?start=2021-01-01T00%3A00%3A00%2B00%3A00'),
+                  json={'jobs': []}, match_querystring=True)
+
+    batch = api.find_jobs(start=datetime(2021, 1, 1))
+    assert len(batch) == 0
+
+    batch = api.find_jobs(start=datetime(2021, 1, 1, tzinfo=timezone.utc))
+    assert len(batch) == 0
+
+
+@responses.activate
+def test_find_jobs_end():
+    api = HyP3()
+    responses.add(responses.GET, urljoin(api.url, '/jobs?end=2021-01-02T00%3A00%3A00%2B00%3A00'),
+                  json={'jobs': []}, match_querystring=True)
+
+    batch = api.find_jobs(end=datetime(2021, 1, 2))
+    assert len(batch) == 0
+
+    batch = api.find_jobs(end=datetime(2021, 1, 2, tzinfo=timezone.utc))
+    assert len(batch) == 0
+
+
 @responses.activate
 def test_get_job_by_id(get_mock_job):
     job = get_mock_job()