From 21337c1afd1e43aec3b209770bc90211ef3dccf6 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 09:25:20 -0500 Subject: [PATCH 01/42] remove Found zero jobs warning --- hyp3_sdk/hyp3.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/hyp3_sdk/hyp3.py b/hyp3_sdk/hyp3.py index 59c0a7b..84fa2b2 100644 --- a/hyp3_sdk/hyp3.py +++ b/hyp3_sdk/hyp3.py @@ -79,8 +79,6 @@ def find_jobs(self, start: Optional[datetime] = None, end: Optional[datetime] = _raise_for_hyp3_status(response) jobs.extend([Job.from_dict(job) for job in response.json()['jobs']]) - if not jobs: - warnings.warn('Found zero jobs', UserWarning) return Batch(jobs) def get_job_by_id(self, job_id: str) -> Job: From f66b6ffd01e32dcc8a262acb58382a1746ffc56c Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 10:32:06 -0500 Subject: [PATCH 02/42] added new magic methods to batch class --- hyp3_sdk/jobs.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/hyp3_sdk/jobs.py b/hyp3_sdk/jobs.py index 51e9b57..1f2cee5 100644 --- a/hyp3_sdk/jobs.py +++ b/hyp3_sdk/jobs.py @@ -152,12 +152,36 @@ def __add__(self, other: Union[Job, 'Batch']): else: raise TypeError(f"unsupported operand type(s) for +: '{type(self)}' and '{type(other)}'") + def __iadd__(self, other: Union[Job, 'Batch']): + if isinstance(other, Batch): + self.jobs += other.jobs + elif isinstance(other, Job): + self.jobs += [other] + else: + raise TypeError(f"unsupported operand type(s) for +: '{type(self)}' and '{type(other)}'") + def __iter__(self): return iter(self.jobs) def __len__(self): return len(self.jobs) + def __contains__(self, job: Job): + return job in self.jobs + + def __delitem__(self, job: Job): + self.jobs.remove(job) + + def __getitem__(self, index: int): + return self.jobs[index] + + def __setitem__(self, index: int, job: Job): + self.jobs[index] = job + + def __reverse__(self): + for job in self.jobs[::-1]: + yield job + def __repr__(self): reprs = ", ".join([job.__repr__() for job in self.jobs]) return f'Batch([{reprs}])' @@ -246,3 +270,4 @@ def filter_jobs( filtered_jobs.append(job) return Batch(filtered_jobs) + \ No newline at end of file From 90d8250c68c68a743a04625ebf2ca145db671146 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 10:51:01 -0500 Subject: [PATCH 03/42] added return self to necessary magic methods --- hyp3_sdk/jobs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hyp3_sdk/jobs.py b/hyp3_sdk/jobs.py index 1f2cee5..e89ae75 100644 --- a/hyp3_sdk/jobs.py +++ b/hyp3_sdk/jobs.py @@ -159,6 +159,7 @@ def __iadd__(self, other: Union[Job, 'Batch']): self.jobs += [other] else: raise TypeError(f"unsupported operand type(s) for +: '{type(self)}' and '{type(other)}'") + return self def __iter__(self): return iter(self.jobs) @@ -171,12 +172,14 @@ def __contains__(self, job: Job): def __delitem__(self, job: Job): self.jobs.remove(job) + return self def __getitem__(self, index: int): return self.jobs[index] def __setitem__(self, index: int, job: Job): self.jobs[index] = job + return self def __reverse__(self): for job in self.jobs[::-1]: @@ -270,4 +273,3 @@ def filter_jobs( filtered_jobs.append(job) return Batch(filtered_jobs) - \ No newline at end of file From b8079a63d68e09e1b20e0b4a0a49c7cd59e88e36 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 10:51:17 -0500 Subject: [PATCH 04/42] added test for __iadd__ --- tests/test_jobs.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 9b7c49a..714db8e 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -155,6 +155,25 @@ def test_batch_add(): assert d.jobs[2].running() +def test_batch_iadd(): + print("Testing iAdd") + a = Batch([Job.from_dict(SUCCEEDED_JOB)]) + b = Batch([Job.from_dict(FAILED_JOB)]) + j = Job.from_dict(SUCCEEDED_JOB) + j.status_code = 'RUNNING' + + a += b + assert len(a) == 2 + assert a.jobs[0].succeeded() + assert a.jobs[1].failed() + + a += j + assert len(a) == 3 + assert a.jobs[0].succeeded() + assert a.jobs[1].failed() + assert a.jobs[2].running() + + def test_batch_iter(): defined_jobs = [Job.from_dict(SUCCEEDED_JOB), Job.from_dict(FAILED_JOB)] batch = Batch(defined_jobs) From 03bf2ba8adcde5006912726ff13ffce2fd427c7b Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 10:52:48 -0500 Subject: [PATCH 05/42] added test for __contains__ --- tests/test_jobs.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 714db8e..8db3321 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -192,6 +192,20 @@ def test_batch_len(): assert len(batch) == 2 +def test_contains(get_mock_job): + unexpired_time = (datetime.now(tz=tz.UTC) + timedelta(days=7)).isoformat(timespec='seconds') + j1 = Job.from_dict(SUCCEEDED_JOB) + j2 = Job.from_dict(FAILED_JOB) + j3 = get_mock_job(status_code='SUCCEEDED', expiration_time=unexpired_time, + files=[{'url': 'https://foo.com/file', 'size': 0, 'filename': 'file'}]) + + a = Batch([j1, j2]) + + assert j1 in a + assert j2 in a + assert j3 not in a + + def test_batch_complete_succeeded(): batch = Batch([Job.from_dict(SUCCEEDED_JOB), Job.from_dict(SUCCEEDED_JOB)]) assert batch.complete() From ad9c6d2dd3ff8ef022158af9e7a85ec2557dd5b6 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 11:01:12 -0500 Subject: [PATCH 06/42] pop needed instead of remove in __delitem__ --- hyp3_sdk/jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hyp3_sdk/jobs.py b/hyp3_sdk/jobs.py index e89ae75..704bcb6 100644 --- a/hyp3_sdk/jobs.py +++ b/hyp3_sdk/jobs.py @@ -171,7 +171,7 @@ def __contains__(self, job: Job): return job in self.jobs def __delitem__(self, job: Job): - self.jobs.remove(job) + self.jobs.pop(job) return self def __getitem__(self, index: int): From 77eae21db63dcd8c8fb4800d4fd943144df4f898 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 11:01:37 -0500 Subject: [PATCH 07/42] added test for __delitem__ --- tests/test_jobs.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 8db3321..17ecb9f 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -206,6 +206,26 @@ def test_contains(get_mock_job): assert j3 not in a +def test_delitem(): + j1 = Job.from_dict(SUCCEEDED_JOB) + j2 = Job.from_dict(FAILED_JOB) + batch = Batch([j1, j2]) + + assert j1 in batch + assert j2 in batch + + del batch[1] + + assert j1 in batch + assert j2 not in batch + + batch += j2 + del batch[0] + + assert j1 not in batch + assert j2 in batch + + def test_batch_complete_succeeded(): batch = Batch([Job.from_dict(SUCCEEDED_JOB), Job.from_dict(SUCCEEDED_JOB)]) assert batch.complete() From 8508fad53fdfad1494c359237f095a84b0e49bb1 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 11:04:11 -0500 Subject: [PATCH 08/42] added test for __getitem__ --- tests/test_jobs.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 17ecb9f..f5431cd 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -226,6 +226,15 @@ def test_delitem(): assert j2 in batch +def test_getitem(): + j1 = Job.from_dict(SUCCEEDED_JOB) + j2 = Job.from_dict(FAILED_JOB) + batch = Batch([j1, j2]) + + assert j1 == batch[0] + assert j2 == batch[1] + + def test_batch_complete_succeeded(): batch = Batch([Job.from_dict(SUCCEEDED_JOB), Job.from_dict(SUCCEEDED_JOB)]) assert batch.complete() From 26c00cc69a8b44ed9026e41ab364621c8adac348 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 11:08:34 -0500 Subject: [PATCH 09/42] added test for __setitem__ --- tests/test_jobs.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_jobs.py b/tests/test_jobs.py index f5431cd..0be297b 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -235,6 +235,18 @@ def test_getitem(): assert j2 == batch[1] +def test_setitem(get_mock_job): + unexpired_time = (datetime.now(tz=tz.UTC) + timedelta(days=7)).isoformat(timespec='seconds') + j1 = Job.from_dict(SUCCEEDED_JOB) + j2 = Job.from_dict(FAILED_JOB) + j3 = get_mock_job(status_code='SUCCEEDED', expiration_time=unexpired_time, + files=[{'url': 'https://foo.com/file', 'size': 0, 'filename': 'file'}]) + batch = Batch([j1, j2]) + + batch[1] = j3 + assert batch[1] == j3 + + def test_batch_complete_succeeded(): batch = Batch([Job.from_dict(SUCCEEDED_JOB), Job.from_dict(SUCCEEDED_JOB)]) assert batch.complete() From 384af7f8e1b3f9e12647fa848d3739072dd35ad9 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 11:13:57 -0500 Subject: [PATCH 10/42] added test for __reverse__ --- tests/test_jobs.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 0be297b..009bd94 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -247,6 +247,22 @@ def test_setitem(get_mock_job): assert batch[1] == j3 +def test_reverse(get_mock_job): + unexpired_time = (datetime.now(tz=tz.UTC) + timedelta(days=7)).isoformat(timespec='seconds') + j1 = Job.from_dict(SUCCEEDED_JOB) + j2 = Job.from_dict(FAILED_JOB) + j3 = get_mock_job(status_code='SUCCEEDED', expiration_time=unexpired_time, + files=[{'url': 'https://foo.com/file', 'size': 0, 'filename': 'file'}]) + + batch = Batch([j1, j2, j3]) + + batch_reversed = list(reversed(batch)) + + assert batch_reversed[0] == j3 + assert batch_reversed[1] == j2 + assert batch_reversed[2] == j1 + + def test_batch_complete_succeeded(): batch = Batch([Job.from_dict(SUCCEEDED_JOB), Job.from_dict(SUCCEEDED_JOB)]) assert batch.complete() From 46956b5d1b1d17c533c2fe5382ca681d2f54ff6e Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 11:14:28 -0500 Subject: [PATCH 11/42] removed print from test_iadd --- tests/test_jobs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 009bd94..1bd3699 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -156,7 +156,6 @@ def test_batch_add(): def test_batch_iadd(): - print("Testing iAdd") a = Batch([Job.from_dict(SUCCEEDED_JOB)]) b = Batch([Job.from_dict(FAILED_JOB)]) j = Job.from_dict(SUCCEEDED_JOB) From 7bbcb0b3d9366defdee352394173fec5d45a7669 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 11:18:27 -0500 Subject: [PATCH 12/42] removed test for warning --- tests/test_hyp3.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_hyp3.py b/tests/test_hyp3.py index 1c465a2..dbf9afa 100644 --- a/tests/test_hyp3.py +++ b/tests/test_hyp3.py @@ -42,8 +42,7 @@ def test_find_jobs(get_mock_job): batch = api.find_jobs() assert len(batch) == 3 - with pytest.warns(UserWarning): - batch = api.find_jobs() + batch = api.find_jobs() assert len(batch) == 0 From 58dabaaec13dde7b35d301c3c962894c4a225d4a Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 11:21:09 -0500 Subject: [PATCH 13/42] removed unused warnings import --- hyp3_sdk/hyp3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hyp3_sdk/hyp3.py b/hyp3_sdk/hyp3.py index 84fa2b2..1ebc3bb 100644 --- a/hyp3_sdk/hyp3.py +++ b/hyp3_sdk/hyp3.py @@ -1,6 +1,5 @@ import math import time -import warnings from datetime import datetime, timezone from functools import singledispatchmethod from getpass import getpass From 5a02198bd0e91a388ddc2d2d7ecf6e9abc717fa9 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 11:24:06 -0500 Subject: [PATCH 14/42] removed unnecessary pytest import --- tests/test_hyp3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_hyp3.py b/tests/test_hyp3.py index dbf9afa..dea718d 100644 --- a/tests/test_hyp3.py +++ b/tests/test_hyp3.py @@ -1,7 +1,6 @@ from datetime import datetime, timedelta from urllib.parse import urljoin -import pytest import responses import hyp3_sdk From 7ff5eea1bf7cb717dbbb5056dfeaf24e116061d9 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 11:31:24 -0500 Subject: [PATCH 15/42] making flake8 happy --- hyp3_sdk/jobs.py | 2 +- tests/test_jobs.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/hyp3_sdk/jobs.py b/hyp3_sdk/jobs.py index 704bcb6..fd7f165 100644 --- a/hyp3_sdk/jobs.py +++ b/hyp3_sdk/jobs.py @@ -169,7 +169,7 @@ def __len__(self): def __contains__(self, job: Job): return job in self.jobs - + def __delitem__(self, job: Job): self.jobs.pop(job) return self diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 1bd3699..1f2a821 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -196,10 +196,10 @@ def test_contains(get_mock_job): j1 = Job.from_dict(SUCCEEDED_JOB) j2 = Job.from_dict(FAILED_JOB) j3 = get_mock_job(status_code='SUCCEEDED', expiration_time=unexpired_time, - files=[{'url': 'https://foo.com/file', 'size': 0, 'filename': 'file'}]) + files=[{'url': 'https://foo.com/file', 'size': 0, 'filename': 'file'}]) a = Batch([j1, j2]) - + assert j1 in a assert j2 in a assert j3 not in a @@ -239,7 +239,7 @@ def test_setitem(get_mock_job): j1 = Job.from_dict(SUCCEEDED_JOB) j2 = Job.from_dict(FAILED_JOB) j3 = get_mock_job(status_code='SUCCEEDED', expiration_time=unexpired_time, - files=[{'url': 'https://foo.com/file', 'size': 0, 'filename': 'file'}]) + files=[{'url': 'https://foo.com/file', 'size': 0, 'filename': 'file'}]) batch = Batch([j1, j2]) batch[1] = j3 @@ -251,12 +251,12 @@ def test_reverse(get_mock_job): j1 = Job.from_dict(SUCCEEDED_JOB) j2 = Job.from_dict(FAILED_JOB) j3 = get_mock_job(status_code='SUCCEEDED', expiration_time=unexpired_time, - files=[{'url': 'https://foo.com/file', 'size': 0, 'filename': 'file'}]) - + files=[{'url': 'https://foo.com/file', 'size': 0, 'filename': 'file'}]) + batch = Batch([j1, j2, j3]) batch_reversed = list(reversed(batch)) - + assert batch_reversed[0] == j3 assert batch_reversed[1] == j2 assert batch_reversed[2] == j1 From b15fcb88ec78452dac26b6fe76470d255bba4f0a Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 12:46:48 -0500 Subject: [PATCH 16/42] Updated changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32a262e..59bd8bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.2.0](https://github.com/ASFHyP3/hyp3-sdk/compare/v1.1.2...v1.1.3) + +### Added +- Added missing container methods: https://docs.python.org/3/reference/datamodel.html#emulating-container-types + - For example, batches are now subscriptable i.e. `batch[0]` may be done instead of `batch.jobs[0]` + ## [1.1.3](https://github.com/ASFHyP3/hyp3-sdk/compare/v1.1.2...v1.1.3) ### Added From 3c31d4dd9b5c26e4f86e14f55751f03a79ea5fcc Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 12:47:56 -0500 Subject: [PATCH 17/42] Fixed typo in error Co-authored-by: Joseph H Kennedy --- hyp3_sdk/jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hyp3_sdk/jobs.py b/hyp3_sdk/jobs.py index fd7f165..7c6376d 100644 --- a/hyp3_sdk/jobs.py +++ b/hyp3_sdk/jobs.py @@ -158,7 +158,7 @@ def __iadd__(self, other: Union[Job, 'Batch']): elif isinstance(other, Job): self.jobs += [other] else: - raise TypeError(f"unsupported operand type(s) for +: '{type(self)}' and '{type(other)}'") + raise TypeError(f"unsupported operand type(s) for +=: '{type(self)}' and '{type(other)}'") return self def __iter__(self): From ee35fd574fec0a1b2fd6477b64bdef6d72e7d27d Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 12:51:14 -0500 Subject: [PATCH 18/42] Update CHANGELOG.md Co-authored-by: Joseph H Kennedy --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59bd8bf..865d5ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,8 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [1.2.0](https://github.com/ASFHyP3/hyp3-sdk/compare/v1.1.2...v1.1.3) ### Added -- Added missing container methods: https://docs.python.org/3/reference/datamodel.html#emulating-container-types - - For example, batches are now subscriptable i.e. `batch[0]` may be done instead of `batch.jobs[0]` +- Added missing [container methods](https://docs.python.org/3/reference/datamodel.html#emulating-container-types) + - batches are now subscriptable like `batch[0]` ## [1.1.3](https://github.com/ASFHyP3/hyp3-sdk/compare/v1.1.2...v1.1.3) From c0802c4759e2d43d625bcee3d6e2d588b4fed814 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 12:56:10 -0500 Subject: [PATCH 19/42] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 865d5ed..dc91801 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added - Added missing [container methods](https://docs.python.org/3/reference/datamodel.html#emulating-container-types) - batches are now subscriptable like `batch[0]` + - jobs can be searched for in batches like`job in batch` + - jobs can be deleted from batches e.g. `del batch[0]` + - batches can be reversed now using the `reversed()` function ## [1.1.3](https://github.com/ASFHyP3/hyp3-sdk/compare/v1.1.2...v1.1.3) From e7184175997d84b3007d2db7b27e9f0415971719 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 12:58:04 -0500 Subject: [PATCH 20/42] Update Changelog --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc91801..f0780ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,9 +10,9 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added - Added missing [container methods](https://docs.python.org/3/reference/datamodel.html#emulating-container-types) - - batches are now subscriptable like `batch[0]` - - jobs can be searched for in batches like`job in batch` - - jobs can be deleted from batches e.g. `del batch[0]` + - batches are now subscriptable: `batch[0]` + - jobs can be searched for in batches:`job in batch` + - jobs can be deleted from batches: `del batch[0]` - batches can be reversed now using the `reversed()` function ## [1.1.3](https://github.com/ASFHyP3/hyp3-sdk/compare/v1.1.2...v1.1.3) From 8c372558229ad9f80cdfde9e96dbcced927e42a6 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 12:59:43 -0500 Subject: [PATCH 21/42] Timezone info defaults to UTC if none is provided --- hyp3_sdk/hyp3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hyp3_sdk/hyp3.py b/hyp3_sdk/hyp3.py index 1ebc3bb..573df54 100644 --- a/hyp3_sdk/hyp3.py +++ b/hyp3_sdk/hyp3.py @@ -63,7 +63,7 @@ def find_jobs(self, start: Optional[datetime] = None, end: Optional[datetime] = if param_value is not None: if isinstance(param_value, datetime): if param_value.tzinfo is None: - param_value.replace(tzinfo=timezone.utc) + param_value = param_value.replace(tzinfo=timezone.utc) param_value = param_value.isoformat(timespec='seconds') params[param_name] = param_value From 1c26efbf89fca00671a69e8116a509ed74c6bef8 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 13:23:12 -0500 Subject: [PATCH 22/42] Updated Changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0780ee..89df77c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - jobs can be searched for in batches:`job in batch` - jobs can be deleted from batches: `del batch[0]` - batches can be reversed now using the `reversed()` function +- `find_jobs()` now accepts datetimes with no timezone info: defaults to UTC. +### Removed +- `FoundZeroJobs` warning from `find_jobs()` ## [1.1.3](https://github.com/ASFHyP3/hyp3-sdk/compare/v1.1.2...v1.1.3) From 9e933228b46bd4f9f0070cc8f80b505ce11862df Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 13:23:58 -0500 Subject: [PATCH 23/42] Update Changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89df77c..590e002 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - jobs can be searched for in batches:`job in batch` - jobs can be deleted from batches: `del batch[0]` - batches can be reversed now using the `reversed()` function -- `find_jobs()` now accepts datetimes with no timezone info: defaults to UTC. +- `find_jobs()` now accepts datetimes with no timezone info and defaults to UTC. ### Removed - `FoundZeroJobs` warning from `find_jobs()` From a6f1368537ec1d273fc4a06d8ebd6c132d038944 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 24 Jun 2021 13:30:49 -0500 Subject: [PATCH 24/42] Update CHANGELOG.md Co-authored-by: Joseph H Kennedy --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 590e002..e326ae5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - jobs can be deleted from batches: `del batch[0]` - batches can be reversed now using the `reversed()` function - `find_jobs()` now accepts datetimes with no timezone info and defaults to UTC. + ### Removed - `FoundZeroJobs` warning from `find_jobs()` From 890fd445ed7aaf7d6cbf1aaba5825acb2817d1f0 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 24 Jun 2021 10:42:57 -0800 Subject: [PATCH 25/42] Gaurd against ipywidgest failure --- hyp3_sdk/hyp3.py | 6 +++--- hyp3_sdk/jobs.py | 4 ++-- hyp3_sdk/util.py | 12 +++++++++++- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/hyp3_sdk/hyp3.py b/hyp3_sdk/hyp3.py index 573df54..adf3428 100644 --- a/hyp3_sdk/hyp3.py +++ b/hyp3_sdk/hyp3.py @@ -6,12 +6,10 @@ from typing import List, Literal, Optional, Union from urllib.parse import urljoin -from tqdm.auto import tqdm - import hyp3_sdk from hyp3_sdk.exceptions import HyP3Error, _raise_for_hyp3_status from hyp3_sdk.jobs import Batch, Job -from hyp3_sdk.util import get_authenticated_session +from hyp3_sdk.util import get_authenticated_session, get_tqdm_progress_bar HYP3_PROD = 'https://hyp3-api.asf.alaska.edu' HYP3_TEST = 'https://hyp3-test-api.asf.alaska.edu' @@ -111,6 +109,7 @@ def watch(self, job_or_batch: Union[Batch, Job], timeout: int = 10800, @watch.register def _watch_batch(self, batch: Batch, timeout: int = 10800, interval: Union[int, float] = 60) -> Batch: + tqdm = get_tqdm_progress_bar() iterations_until_timeout = math.ceil(timeout / interval) bar_format = '{l_bar}{bar}| {n_fmt}/{total_fmt} [{postfix[0]}]' with tqdm(total=len(batch), bar_format=bar_format, postfix=[f'timeout in {timeout} s']) as progress_bar: @@ -132,6 +131,7 @@ def _watch_batch(self, batch: Batch, timeout: int = 10800, interval: Union[int, @watch.register def _watch_job(self, job: Job, timeout: int = 10800, interval: Union[int, float] = 60) -> Job: + tqdm = get_tqdm_progress_bar() iterations_until_timeout = math.ceil(timeout / interval) bar_format = '{n_fmt}/{total_fmt} [{postfix[0]}]' with tqdm(total=1, bar_format=bar_format, postfix=[f'timeout in {timeout} s']) as progress_bar: diff --git a/hyp3_sdk/jobs.py b/hyp3_sdk/jobs.py index 7c6376d..0902cdf 100644 --- a/hyp3_sdk/jobs.py +++ b/hyp3_sdk/jobs.py @@ -6,10 +6,9 @@ from dateutil import tz from dateutil.parser import parse as parse_date from requests import HTTPError -from tqdm.auto import tqdm from hyp3_sdk.exceptions import HyP3SDKError -from hyp3_sdk.util import download_file +from hyp3_sdk.util import download_file, get_tqdm_progress_bar # TODO: actually looks like a good candidate for a dataclass (python 3.7+) @@ -227,6 +226,7 @@ def download_files(self, location: Union[Path, str] = '.', create: bool = True) Returns: list of Path objects to downloaded files """ downloaded_files = [] + tqdm = get_tqdm_progress_bar() for job in tqdm(self.jobs): try: downloaded_files.extend(job.download_files(location, create)) diff --git a/hyp3_sdk/util.py b/hyp3_sdk/util.py index a656b1e..6de3ca0 100644 --- a/hyp3_sdk/util.py +++ b/hyp3_sdk/util.py @@ -4,7 +4,6 @@ import requests from requests.adapters import HTTPAdapter -from tqdm.auto import tqdm from urllib3.util.retry import Retry import hyp3_sdk @@ -14,6 +13,16 @@ '&redirect_uri=https://auth.asf.alaska.edu/login&app_type=401' +def get_tqdm_progress_bar(): + try: + # https://github.com/ASFHyP3/hyp3-sdk/issues/92 + import ipywidgets # noqa: F401 + from tqdm.auto import tqdm + except ImportError: + from tqdm.std import tqdm + return tqdm + + def get_authenticated_session(username: str, password: str) -> requests.Session: """Log into HyP3 using credentials for `urs.earthdata.nasa.gov` from either the provided credentials or a `.netrc` file. @@ -65,6 +74,7 @@ def download_file(url: str, filepath: Union[Path, str], chunk_size=None, retries with session.get(url, stream=True) as s: s.raise_for_status() + tqdm = get_tqdm_progress_bar() with tqdm.wrapattr(open(filepath, "wb"), 'write', miniters=1, desc=filepath.name, total=int(s.headers.get('content-length', 0))) as f: for chunk in s.iter_content(chunk_size=chunk_size): From 8990afef3b780edad003009e459d87f188492443 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 24 Jun 2021 10:49:11 -0800 Subject: [PATCH 26/42] update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e326ae5..ab82118 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,11 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Removed - `FoundZeroJobs` warning from `find_jobs()` + +### Fixed +- [#92](https://github.com/ASFHyP3/hyp3-sdk/issues/92) -- `ImportError` being + raised when showing a progress bar because `ipywidgets` may not always be + installed when running in a Jupyter kernel ## [1.1.3](https://github.com/ASFHyP3/hyp3-sdk/compare/v1.1.2...v1.1.3) From e6ca5d20c71f89aca5db796c1da24d3dcf148f72 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 25 Jun 2021 13:28:29 -0800 Subject: [PATCH 27/42] Update downstream notification --- .github/workflows/notify-downstream.yml | 28 +++++++++++++++++++++++++ .github/workflows/release.yml | 14 ------------- 2 files changed, 28 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/notify-downstream.yml diff --git a/.github/workflows/notify-downstream.yml b/.github/workflows/notify-downstream.yml new file mode 100644 index 0000000..0b4cb7f --- /dev/null +++ b/.github/workflows/notify-downstream.yml @@ -0,0 +1,28 @@ +name: Notify Downstream of New Release + +on: + release: + types: + - released + +jobs: + update-docs: + runs-on: ubuntu-latest + steps: + - name: Bump SDK version in HyP3 docs + uses: benc-uk/workflow-dispatch@v1.1 + with: + workflow: Update HyP3 SDK version + token: ${{ secrets.TOOLS_BOT_PAK }} + repo: ASFHyP3/hyp3-docs + ref: main + inputs: '{"sdk_version": "${{ github.event.release.tag_name }}"}' + + - name: Tweet release notes + uses: benc-uk/workflow-dispatch@v1.1 + with: + workflow: Propose Tweet + token: ${{ secrets.TOOLS_BOT_PAK }} + repo: ASFHyP3/hyp3-docs + ref: develop + inputs: '{"message": "## ${{ github.event.release.name }}\n${{ github.event.release.body }}"}' \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2b99567..1f14007 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,20 +21,6 @@ jobs: ALLOW_TAG_PREFIX: "true" RELEASE_NAME_PREFIX: "HyP3 SDK " - - name: Get SDK version number - id: get_version - run: | - echo ::set-output name=version_tag::${GITHUB_REF#refs/tags/} - - - name: Bump SDK version in HyP3 docs - uses: benc-uk/workflow-dispatch@v1.1 - with: - workflow: Update HyP3 SDK version - token: ${{ secrets.TOOLS_BOT_PAK }} - repo: ASFHyP3/hyp3-docs - ref: main - inputs: '{"sdk_version": "${{ steps.get_version.outputs.version_tag }}"}' - - name: Attempt fast-forward develop from main run: | git fetch --prune From 441cb01c3551467b39b1312325b918b9d2080aac Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 25 Jun 2021 13:51:54 -0800 Subject: [PATCH 28/42] Update release workflow --- .../{asssociated-pr.query.yml => associated-pr.query.yml} | 0 .github/workflows/release.yml | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename .github/queries/{asssociated-pr.query.yml => associated-pr.query.yml} (100%) diff --git a/.github/queries/asssociated-pr.query.yml b/.github/queries/associated-pr.query.yml similarity index 100% rename from .github/queries/asssociated-pr.query.yml rename to .github/queries/associated-pr.query.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1f14007..c797dec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,11 +15,11 @@ jobs: token: ${{ secrets.TOOLS_BOT_PAK }} - name: Create Release - uses: docker://antonyurchenko/git-release:latest + uses: docker://antonyurchenko/git-release:v3.5.0 env: GITHUB_TOKEN: ${{ secrets.TOOLS_BOT_PAK }} - ALLOW_TAG_PREFIX: "true" RELEASE_NAME_PREFIX: "HyP3 SDK " + ALLOW_TAG_PREFIX: true - name: Attempt fast-forward develop from main run: | From c22ed402361f5b03ee0f54f84422f7518202f477 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 25 Jun 2021 14:40:44 -0800 Subject: [PATCH 29/42] whitespace --- .github/workflows/notify-downstream.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/notify-downstream.yml b/.github/workflows/notify-downstream.yml index 0b4cb7f..9a7300b 100644 --- a/.github/workflows/notify-downstream.yml +++ b/.github/workflows/notify-downstream.yml @@ -25,4 +25,4 @@ jobs: token: ${{ secrets.TOOLS_BOT_PAK }} repo: ASFHyP3/hyp3-docs ref: develop - inputs: '{"message": "## ${{ github.event.release.name }}\n${{ github.event.release.body }}"}' \ No newline at end of file + inputs: '{"message": "## ${{ github.event.release.name }}\n${{ github.event.release.body }}"}' From a60deacc9d0511315c2819bc012b21276334a85c Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Tue, 29 Jun 2021 13:27:42 -0500 Subject: [PATCH 30/42] Update CHANGELOG.md Co-authored-by: Andrew Johnston --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab82118..decdeb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [1.2.0](https://github.com/ASFHyP3/hyp3-sdk/compare/v1.1.2...v1.1.3) +## [1.2.0](https://github.com/ASFHyP3/hyp3-sdk/compare/v1.1.3...v1.2.0) ### Added - Added missing [container methods](https://docs.python.org/3/reference/datamodel.html#emulating-container-types) From 652dc2f20c9d1dcf79b0b2bfcb3932e0acf97667 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 29 Jun 2021 11:26:06 -0800 Subject: [PATCH 31/42] Add logs attribute to Job class --- hyp3_sdk/jobs.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hyp3_sdk/jobs.py b/hyp3_sdk/jobs.py index 0902cdf..1a64271 100644 --- a/hyp3_sdk/jobs.py +++ b/hyp3_sdk/jobs.py @@ -24,6 +24,7 @@ def __init__( name: Optional[str] = None, job_parameters: Optional[dict] = None, files: Optional[List] = None, + logs: Optional[List] = None, browse_images: Optional[List] = None, thumbnail_images: Optional[List] = None, expiration_time: Optional[datetime] = None @@ -36,6 +37,7 @@ def __init__( self.name = name self.job_parameters = job_parameters self.files = files + self.logs = logs self.browse_images = browse_images self.thumbnail_images = thumbnail_images self.expiration_time = expiration_time @@ -61,6 +63,7 @@ def from_dict(input_dict: dict): name=input_dict.get('name'), job_parameters=input_dict.get('job_parameters'), files=input_dict.get('files'), + logs=input_dict.get('logs'), browse_images=input_dict.get('browse_images'), thumbnail_images=input_dict.get('thumbnail_images'), expiration_time=expiration_time @@ -77,7 +80,7 @@ def to_dict(self, for_resubmit: bool = False): job_dict[key] = value if not for_resubmit: - for key in ['files', 'browse_images', 'thumbnail_images', 'job_id', 'status_code', 'user_id', + for key in ['files', 'logs', 'browse_images', 'thumbnail_images', 'job_id', 'status_code', 'user_id', 'expiration_time', 'request_time']: value = self.__getattribute__(key) if value is not None: From 9bff4c431df2ba4ae34b30892a365cfc0c54c91f Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 29 Jun 2021 11:57:43 -0800 Subject: [PATCH 32/42] Add test for job attributes --- hyp3_sdk/jobs.py | 6 ++++-- tests/test_jobs.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/hyp3_sdk/jobs.py b/hyp3_sdk/jobs.py index 1a64271..fd788be 100644 --- a/hyp3_sdk/jobs.py +++ b/hyp3_sdk/jobs.py @@ -14,6 +14,9 @@ # TODO: actually looks like a good candidate for a dataclass (python 3.7+) # https://docs.python.org/3/library/dataclasses.html class Job: + _finished_job_keys = {'job_id', 'logs', 'request_time', 'status_code', 'user_id'} + _successful_job_keys = _finished_job_keys.union({'browse_images', 'expiration_time', 'files', 'thumbnail_images'}) + def __init__( self, job_type: str, @@ -80,8 +83,7 @@ def to_dict(self, for_resubmit: bool = False): job_dict[key] = value if not for_resubmit: - for key in ['files', 'logs', 'browse_images', 'thumbnail_images', 'job_id', 'status_code', 'user_id', - 'expiration_time', 'request_time']: + for key in self._successful_job_keys: value = self.__getattribute__(key) if value is not None: if isinstance(value, datetime): diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 1f2a821..875a94f 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -12,6 +12,7 @@ "browse_images": ["https://PAIR_PROCESS.png"], "expiration_time": "2020-10-08T00:00:00+00:00", "files": [{"filename": "PAIR_PROCESS.nc", "size": 5949932, "url": "https://PAIR_PROCESS.nc"}], + "logs": ["https://d1c05104-b455-4f35-a95a-84155d63f855.log"], "job_id": "d1c05104-b455-4f35-a95a-84155d63f855", "job_parameters": {"granules": [ "S1A_IW_SLC__1SDH_20180511T204719_20180511T204746_021862_025C12_6F77", @@ -26,6 +27,7 @@ } FAILED_JOB = { + "logs": ["https://281b2087-9e7d-4d17-a9b3-aebeb2ad23c6.log"], "job_id": "281b2087-9e7d-4d17-a9b3-aebeb2ad23c6", "job_parameters": { "granules": [ @@ -41,6 +43,20 @@ } +def test_job_attributes(): + job = Job.from_dict(SUCCEEDED_JOB) + for key in Job._successful_job_keys: + assert job.__getattribute__(key) + + job = Job.from_dict(FAILED_JOB) + for key in Job._finished_job_keys: + assert job.__getattribute__(key) + + job = Job.from_dict(FAILED_JOB) + for key in Job._successful_job_keys.difference(Job._finished_job_keys): + assert job.__getattribute__(key) is None + + def test_job_dict_transforms(): job = Job.from_dict(SUCCEEDED_JOB) assert job.to_dict() == SUCCEEDED_JOB @@ -48,6 +64,18 @@ def test_job_dict_transforms(): job = Job.from_dict(FAILED_JOB) assert job.to_dict() == FAILED_JOB + job = Job.from_dict(SUCCEEDED_JOB) + retry = job.to_dict(for_resubmit=True) + for key in Job._successful_job_keys: + assert job.__getattribute__(key) + assert not retry.get(key, False) + + job = Job.from_dict(FAILED_JOB) + retry = job.to_dict(for_resubmit=True) + for key in Job._finished_job_keys: + assert job.__getattribute__(key) + assert not retry.get(key, False) + def test_job_complete_succeeded_failed_running(): job = Job.from_dict(SUCCEEDED_JOB) From 0d996ee49153519243d1980d3c4339bc7afa2023 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Tue, 29 Jun 2021 15:11:41 -0500 Subject: [PATCH 33/42] added tests for is114 fix --- tests/test_hyp3.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/test_hyp3.py b/tests/test_hyp3.py index dea718d..a1a9884 100644 --- a/tests/test_hyp3.py +++ b/tests/test_hyp3.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from urllib.parse import urljoin import responses @@ -70,6 +70,30 @@ def test_find_jobs_paging(get_mock_job): assert 'next' in responses.calls[1].request.params +@responses.activate +def test_find_jobs_start(get_mock_job): + api = HyP3() + responses.add(responses.GET, urljoin(api.url, '/jobs?start=2021-01-01T00%3A00%3A00%2B00%3A00'), json={'jobs': []}, match_querystring=True) + + batch = api.find_jobs(start=datetime(2021, 1, 1)) + assert len(batch) == 0 + + batch = api.find_jobs(start=datetime(2021, 1, 1, tzinfo=timezone.utc)) + assert len(batch) == 0 + + +@responses.activate +def test_find_jobs_end(get_mock_job): + api = HyP3() + responses.add(responses.GET, urljoin(api.url, '/jobs?end=2021-01-02T00%3A00%3A00%2B00%3A00'), json={'jobs': []}, match_querystring=True) + + batch = api.find_jobs(end=datetime(2021, 1, 2)) + assert len(batch) == 0 + + batch = api.find_jobs(end=datetime(2021, 1, 2, tzinfo=timezone.utc)) + assert len(batch) == 0 + + @responses.activate def test_get_job_by_id(get_mock_job): job = get_mock_job() From bf3cfff27da195531cb43fce5f99deacd4955884 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Tue, 29 Jun 2021 15:15:41 -0500 Subject: [PATCH 34/42] making flake8 happy --- tests/test_hyp3.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_hyp3.py b/tests/test_hyp3.py index a1a9884..518d38b 100644 --- a/tests/test_hyp3.py +++ b/tests/test_hyp3.py @@ -73,7 +73,8 @@ def test_find_jobs_paging(get_mock_job): @responses.activate def test_find_jobs_start(get_mock_job): api = HyP3() - responses.add(responses.GET, urljoin(api.url, '/jobs?start=2021-01-01T00%3A00%3A00%2B00%3A00'), json={'jobs': []}, match_querystring=True) + responses.add(responses.GET, urljoin(api.url, '/jobs?start=2021-01-01T00%3A00%3A00%2B00%3A00'), + json={'jobs': []}, match_querystring=True) batch = api.find_jobs(start=datetime(2021, 1, 1)) assert len(batch) == 0 @@ -85,11 +86,12 @@ def test_find_jobs_start(get_mock_job): @responses.activate def test_find_jobs_end(get_mock_job): api = HyP3() - responses.add(responses.GET, urljoin(api.url, '/jobs?end=2021-01-02T00%3A00%3A00%2B00%3A00'), json={'jobs': []}, match_querystring=True) + responses.add(responses.GET, urljoin(api.url, '/jobs?end=2021-01-02T00%3A00%3A00%2B00%3A00'), + json={'jobs': []}, match_querystring=True) batch = api.find_jobs(end=datetime(2021, 1, 2)) assert len(batch) == 0 - + batch = api.find_jobs(end=datetime(2021, 1, 2, tzinfo=timezone.utc)) assert len(batch) == 0 From add7e9652d21a89524fb29e32736ad7975c0451c Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Tue, 29 Jun 2021 15:20:45 -0500 Subject: [PATCH 35/42] removed unnecessary get_mock_jobs parameter --- tests/test_hyp3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_hyp3.py b/tests/test_hyp3.py index 518d38b..6b9f25e 100644 --- a/tests/test_hyp3.py +++ b/tests/test_hyp3.py @@ -71,7 +71,7 @@ def test_find_jobs_paging(get_mock_job): @responses.activate -def test_find_jobs_start(get_mock_job): +def test_find_jobs_start(): api = HyP3() responses.add(responses.GET, urljoin(api.url, '/jobs?start=2021-01-01T00%3A00%3A00%2B00%3A00'), json={'jobs': []}, match_querystring=True) @@ -84,7 +84,7 @@ def test_find_jobs_start(get_mock_job): @responses.activate -def test_find_jobs_end(get_mock_job): +def test_find_jobs_end(): api = HyP3() responses.add(responses.GET, urljoin(api.url, '/jobs?end=2021-01-02T00%3A00%3A00%2B00%3A00'), json={'jobs': []}, match_querystring=True) From 0deb1fd426b050f8e2be19219d34acd964f49f15 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 29 Jun 2021 13:47:25 -0800 Subject: [PATCH 36/42] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index decdeb3..4e6b27d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [1.2.0](https://github.com/ASFHyP3/hyp3-sdk/compare/v1.1.3...v1.2.0) ### Added +- `Job` class now has a `logs` attribute containing links to job log files - Added missing [container methods](https://docs.python.org/3/reference/datamodel.html#emulating-container-types) - batches are now subscriptable: `batch[0]` - jobs can be searched for in batches:`job in batch` From b05e53e4fe6229784822fabf6b08edd0debc6a33 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 29 Jun 2021 15:00:43 -0800 Subject: [PATCH 37/42] refactor for resubmit --- CHANGELOG.md | 2 +- hyp3_sdk/jobs.py | 12 ++++-------- tests/test_jobs.py | 10 +++++----- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e6b27d..89ffc73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixed - [#92](https://github.com/ASFHyP3/hyp3-sdk/issues/92) -- `ImportError` being raised when showing a progress bar because `ipywidgets` may not always be - installed when running in a Jupyter kernel + installed when running in a Jupyter kernel ## [1.1.3](https://github.com/ASFHyP3/hyp3-sdk/compare/v1.1.2...v1.1.3) diff --git a/hyp3_sdk/jobs.py b/hyp3_sdk/jobs.py index fd788be..83d84bf 100644 --- a/hyp3_sdk/jobs.py +++ b/hyp3_sdk/jobs.py @@ -14,8 +14,7 @@ # TODO: actually looks like a good candidate for a dataclass (python 3.7+) # https://docs.python.org/3/library/dataclasses.html class Job: - _finished_job_keys = {'job_id', 'logs', 'request_time', 'status_code', 'user_id'} - _successful_job_keys = _finished_job_keys.union({'browse_images', 'expiration_time', 'files', 'thumbnail_images'}) + _attributes_for_resubmit = {'name', 'job_parameters', 'job_type'} def __init__( self, @@ -73,17 +72,14 @@ def from_dict(input_dict: dict): ) def to_dict(self, for_resubmit: bool = False): - job_dict = { - 'job_type': self.job_type, - } - - for key in ['name', 'job_parameters']: + job_dict = {} + for key in Job._attributes_for_resubmit: value = self.__getattribute__(key) if value is not None: job_dict[key] = value if not for_resubmit: - for key in self._successful_job_keys: + for key in set(vars(self).keys()) - Job._attributes_for_resubmit: value = self.__getattribute__(key) if value is not None: if isinstance(value, datetime): diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 875a94f..285551f 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -45,15 +45,15 @@ def test_job_attributes(): job = Job.from_dict(SUCCEEDED_JOB) - for key in Job._successful_job_keys: + for key in SUCCEEDED_JOB.keys(): assert job.__getattribute__(key) job = Job.from_dict(FAILED_JOB) - for key in Job._finished_job_keys: + for key in FAILED_JOB.keys(): assert job.__getattribute__(key) job = Job.from_dict(FAILED_JOB) - for key in Job._successful_job_keys.difference(Job._finished_job_keys): + for key in set(vars(job).keys()) - set(FAILED_JOB.keys()): assert job.__getattribute__(key) is None @@ -66,13 +66,13 @@ def test_job_dict_transforms(): job = Job.from_dict(SUCCEEDED_JOB) retry = job.to_dict(for_resubmit=True) - for key in Job._successful_job_keys: + for key in set(SUCCEEDED_JOB.keys()) - Job._attributes_for_resubmit: assert job.__getattribute__(key) assert not retry.get(key, False) job = Job.from_dict(FAILED_JOB) retry = job.to_dict(for_resubmit=True) - for key in Job._finished_job_keys: + for key in set(FAILED_JOB.keys()) - Job._attributes_for_resubmit: assert job.__getattribute__(key) assert not retry.get(key, False) From e0b7abcb8336270ef88f8f49d9d28b9fe21c660a Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 29 Jun 2021 15:08:04 -0800 Subject: [PATCH 38/42] more refactoring --- hyp3_sdk/jobs.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/hyp3_sdk/jobs.py b/hyp3_sdk/jobs.py index 83d84bf..6e3a579 100644 --- a/hyp3_sdk/jobs.py +++ b/hyp3_sdk/jobs.py @@ -71,21 +71,26 @@ def from_dict(input_dict: dict): expiration_time=expiration_time ) + def _attribute_value(self, attribute_name): + value = self.__getattribute__(attribute_name) + if isinstance(value, datetime): + return value.isoformat(timespec='seconds') + else: + return value + def to_dict(self, for_resubmit: bool = False): job_dict = {} - for key in Job._attributes_for_resubmit: - value = self.__getattribute__(key) - if value is not None: - job_dict[key] = value - - if not for_resubmit: - for key in set(vars(self).keys()) - Job._attributes_for_resubmit: - value = self.__getattribute__(key) + if for_resubmit: + for key in Job._attributes_for_resubmit: + value = self._attribute_value(key) if value is not None: - if isinstance(value, datetime): - job_dict[key] = value.isoformat(timespec='seconds') - else: - job_dict[key] = value + job_dict[key] = value + else: + for key in vars(self).keys(): + value = self._attribute_value(key) + if value is not None: + job_dict[key] = value + return job_dict def succeeded(self) -> bool: From 33ec09f1032c5a3c83ae7f35fba10d56991cbb15 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 29 Jun 2021 15:11:02 -0800 Subject: [PATCH 39/42] Moar refactor! --- hyp3_sdk/jobs.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/hyp3_sdk/jobs.py b/hyp3_sdk/jobs.py index 6e3a579..fbe31e8 100644 --- a/hyp3_sdk/jobs.py +++ b/hyp3_sdk/jobs.py @@ -71,24 +71,19 @@ def from_dict(input_dict: dict): expiration_time=expiration_time ) - def _attribute_value(self, attribute_name): - value = self.__getattribute__(attribute_name) - if isinstance(value, datetime): - return value.isoformat(timespec='seconds') - else: - return value - def to_dict(self, for_resubmit: bool = False): job_dict = {} if for_resubmit: - for key in Job._attributes_for_resubmit: - value = self._attribute_value(key) - if value is not None: - job_dict[key] = value + keys_to_process = Job._attributes_for_resubmit else: - for key in vars(self).keys(): - value = self._attribute_value(key) - if value is not None: + keys_to_process = vars(self).keys() + + for key in keys_to_process: + value = self.__getattribute__(key) + if value is not None: + if isinstance(value, datetime): + job_dict[key] = value.isoformat(timespec='seconds') + else: job_dict[key] = value return job_dict From 9f09fd95d53f98580a67d37593fe18e347c79255 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 29 Jun 2021 15:17:21 -0800 Subject: [PATCH 40/42] Better test names/flow --- tests/test_jobs.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 285551f..a9a7579 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -52,8 +52,8 @@ def test_job_attributes(): for key in FAILED_JOB.keys(): assert job.__getattribute__(key) - job = Job.from_dict(FAILED_JOB) - for key in set(vars(job).keys()) - set(FAILED_JOB.keys()): + unprovided_attributes = set(vars(job).keys()) - set(FAILED_JOB.keys()) + for key in unprovided_attributes: assert job.__getattribute__(key) is None @@ -61,16 +61,14 @@ def test_job_dict_transforms(): job = Job.from_dict(SUCCEEDED_JOB) assert job.to_dict() == SUCCEEDED_JOB - job = Job.from_dict(FAILED_JOB) - assert job.to_dict() == FAILED_JOB - - job = Job.from_dict(SUCCEEDED_JOB) retry = job.to_dict(for_resubmit=True) for key in set(SUCCEEDED_JOB.keys()) - Job._attributes_for_resubmit: assert job.__getattribute__(key) assert not retry.get(key, False) job = Job.from_dict(FAILED_JOB) + assert job.to_dict() == FAILED_JOB + retry = job.to_dict(for_resubmit=True) for key in set(FAILED_JOB.keys()) - Job._attributes_for_resubmit: assert job.__getattribute__(key) From 40d8980dbe701bd665c42b20e93d9ae13f1cceed Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 29 Jun 2021 15:25:40 -0800 Subject: [PATCH 41/42] test resubmit keys in retry dict --- tests/test_jobs.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_jobs.py b/tests/test_jobs.py index a9a7579..839dc79 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -62,6 +62,9 @@ def test_job_dict_transforms(): assert job.to_dict() == SUCCEEDED_JOB retry = job.to_dict(for_resubmit=True) + for key in Job._attributes_for_resubmit: + _ = retry[key] + for key in set(SUCCEEDED_JOB.keys()) - Job._attributes_for_resubmit: assert job.__getattribute__(key) assert not retry.get(key, False) @@ -70,6 +73,9 @@ def test_job_dict_transforms(): assert job.to_dict() == FAILED_JOB retry = job.to_dict(for_resubmit=True) + for key in Job._attributes_for_resubmit: + _ = retry[key] + for key in set(FAILED_JOB.keys()) - Job._attributes_for_resubmit: assert job.__getattribute__(key) assert not retry.get(key, False) From 65b43e47b20bae0146d751e074e89c10214813b0 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 29 Jun 2021 15:28:54 -0800 Subject: [PATCH 42/42] simplify retry test --- tests/test_jobs.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 839dc79..35c409d 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -62,23 +62,13 @@ def test_job_dict_transforms(): assert job.to_dict() == SUCCEEDED_JOB retry = job.to_dict(for_resubmit=True) - for key in Job._attributes_for_resubmit: - _ = retry[key] - - for key in set(SUCCEEDED_JOB.keys()) - Job._attributes_for_resubmit: - assert job.__getattribute__(key) - assert not retry.get(key, False) + assert retry.keys() == Job._attributes_for_resubmit job = Job.from_dict(FAILED_JOB) assert job.to_dict() == FAILED_JOB retry = job.to_dict(for_resubmit=True) - for key in Job._attributes_for_resubmit: - _ = retry[key] - - for key in set(FAILED_JOB.keys()) - Job._attributes_for_resubmit: - assert job.__getattribute__(key) - assert not retry.get(key, False) + assert retry.keys() == Job._attributes_for_resubmit def test_job_complete_succeeded_failed_running():