From bc524d79e5831c40791017597e44dd6581256c1b Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Tue, 17 Sep 2024 09:00:32 -0400 Subject: [PATCH 1/6] rcal-910 Add target to asn_from_list, code cleanup --- romancal/associations/asn_from_list.py | 12 ++++++++++++ romancal/associations/association.py | 18 +++++++++++------- .../associations/lib/asn_schema_jw_level3.json | 9 ++++++++- romancal/associations/lib/dms_base.py | 6 ++++++ romancal/associations/lib/rules_elpp_base.py | 15 ++++++++++----- .../associations/tests/test_asn_from_list.py | 12 ++++++++++++ 6 files changed, 59 insertions(+), 13 deletions(-) diff --git a/romancal/associations/asn_from_list.py b/romancal/associations/asn_from_list.py index 5e9a65284..1509eef68 100644 --- a/romancal/associations/asn_from_list.py +++ b/romancal/associations/asn_from_list.py @@ -37,6 +37,9 @@ def asn_from_list(items, rule=DMS_ELPP_Base, **kwargs): """ asn = rule() asn._add_items(items, **kwargs) + if 'target' in kwargs.keys(): + target = kwargs['target'] + asn['target'] = target return asn @@ -110,6 +113,14 @@ def __init__(self, args=None): help='The association candidate id to use. Default: "%(default)s"', dest="acid", ) + parser.add_argument( + "-t", + "--target", + type=str, + default="None", + help='The target name for the association. Default: "%(default)s"', + dest="target", + ) parser.add_argument( "filelist", @@ -130,6 +141,7 @@ def __init__(self, args=None): rule=rule, product_name=parsed.product_name, acid=parsed.acid, + target=parsed.target ) _, serialized = asn.dump(format=parsed.format) outfile.write(serialized) diff --git a/romancal/associations/association.py b/romancal/associations/association.py index b676c49a8..b25d1d941 100644 --- a/romancal/associations/association.py +++ b/romancal/associations/association.py @@ -86,20 +86,22 @@ class Association(MutableMapping): def __init__( self, - version_id=None, + version_id=None, + target=None, ): - self.data = dict() + self.data = {} self.run_init_hook = True self.meta = {} self.version_id = version_id - + self.target = target self.data.update( { "asn_type": "None", "asn_rule": self.asn_rule, "version_id": self.version_id, "code_version": __version__, + "target": self.target, } ) @@ -135,6 +137,7 @@ def create(cls, item, version_id=None): - [ProcessList[, ...]]: List of items to process again. """ asn = cls(version_id=version_id) + matches, reprocess = asn.add(item) if not matches: return None, reprocess @@ -227,8 +230,8 @@ def dump(self, format="json", **kwargs): """ if self.is_valid: return self.ioregistry[format].dump(self, **kwargs) - else: - raise AssociationNotValidError(f"Association {self} is not valid") + + raise AssociationNotValidError(f"Association {self} is not valid") @classmethod def load(cls, serialized, format=None, validate=True, **kwargs): @@ -430,8 +433,8 @@ def finalize(self): """ if self.is_valid: return [self] - else: - return None + + return None def is_item_member(self, item): """Check if item is already a member of this association @@ -534,6 +537,7 @@ def finalize(asns): def make_timestamp(): + """Generate a timestamp based on runtime """ timestamp = datetime.utcnow().strftime(_TIMESTAMP_TEMPLATE) return timestamp diff --git a/romancal/associations/lib/asn_schema_jw_level3.json b/romancal/associations/lib/asn_schema_jw_level3.json index 750b237fc..bbf1f754f 100644 --- a/romancal/associations/lib/asn_schema_jw_level3.json +++ b/romancal/associations/lib/asn_schema_jw_level3.json @@ -23,7 +23,14 @@ }, "target": { "description": "Canonical name of the astronomical object being observed.", - "type": "string" + "anyOf":[ + { + "type": "string" + }, + { + "type": "null" + } + ] }, "asn_pool": { "description": "Name of the Association Pool from which this association was generated.", diff --git a/romancal/associations/lib/dms_base.py b/romancal/associations/lib/dms_base.py index 88bde16e3..98090c637 100644 --- a/romancal/associations/lib/dms_base.py +++ b/romancal/associations/lib/dms_base.py @@ -189,6 +189,7 @@ def asn_name(self): version_id = self.version_id asn_type = self.data["asn_type"] sequence = self.sequence + target = self.target if version_id: name = _ASN_NAME_TEMPLATE_STAMP.format( @@ -197,6 +198,7 @@ def asn_name(self): stamp=version_id, type=asn_type, sequence=sequence, + target=target, ) else: name = _ASN_NAME_TEMPLATE.format( @@ -204,6 +206,7 @@ def asn_name(self): acid=self.acid.id, type=asn_type, sequence=sequence, + target=target, ) return name.lower() @@ -402,16 +405,19 @@ def update_degraded_status(self): break def update_validity(self, entry): + """ Check/Update the validity of the association""" for test in self.validity.values(): if not test["validated"]: test["validated"] = test["check"](entry) @classmethod def reset_sequence(cls): + """Reset the sequence counter to 1 """ cls._sequence = Counter(start=1) @classmethod def validate(cls, asn): + """Validate the association generated""" super().validate(asn) if isinstance(asn, DMSBaseMixin): diff --git a/romancal/associations/lib/rules_elpp_base.py b/romancal/associations/lib/rules_elpp_base.py index cf5917ea7..e6351c2d7 100644 --- a/romancal/associations/lib/rules_elpp_base.py +++ b/romancal/associations/lib/rules_elpp_base.py @@ -356,8 +356,8 @@ def make_fov_asn(self): if asn.is_valid: results.append(asn) return results - else: - return None + + return None def _init_hook(self, item): """Post-check and pre-add initialization""" @@ -394,6 +394,11 @@ def _add(self, item): # Update meta info self.update_asn(item=item, member=member) + def update_target(self, target): + """Update association target field""" + if self.data['target'] is None: + self.data['target'] = target + def _add_items(self, items, product_name=None, with_exptype=False, **kwargs): """Force adding items to the association @@ -439,7 +444,7 @@ def _add_items(self, items, product_name=None, with_exptype=False, **kwargs): def __repr__(self): # flake8: noqa: F821 try: - file_name, json_repr = self.ioregistry["json"].dump(self) + _ , json_repr = self.ioregistry["json"].dump(self) except KeyValueRegistryError: return str(self.__class__) return json_repr @@ -1031,8 +1036,8 @@ def finalize(self): """ if self.is_valid: return self.make_fov_asn() - else: - return None + + return None class AsnMixin_Lv2Image: diff --git a/romancal/associations/tests/test_asn_from_list.py b/romancal/associations/tests/test_asn_from_list.py index 3105f685e..766ff8206 100644 --- a/romancal/associations/tests/test_asn_from_list.py +++ b/romancal/associations/tests/test_asn_from_list.py @@ -27,6 +27,18 @@ def test_base_roundtrip(): assert asn["members"] == reloaded["members"] +def test_association_target(): + """Create the simple associations with target set""" + items = ["a", "b", "c"] + target_name = "r274dp63x32y80" + product_name = "l3_target" + rule_name = "Association" + asn = asn_from_list(items, rule=Association, product_name=product_name, target=target_name, version_id='c55') + assert asn["asn_rule"] == rule_name + assert asn["asn_type"] == "None" + assert asn["members"] == items + assert asn["target"] == target_name + def test_default_simple(): """Default ELPP association""" product_name = "test_product" From cac3e27285441a4979e48791aa5ccf2a562598fc Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Tue, 17 Sep 2024 09:39:11 -0400 Subject: [PATCH 2/6] rcal-910 update asn_from_list docs --- CHANGES.rst | 6 ++++++ docs/roman/associations/asn_from_list.rst | 17 +++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index e73e53971..a15d7fe6e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,12 @@ 0.16.3 (2024-08-29) =================== +associations +------------ + +- Add target option for asn_from_list command [#] + + mosaic_pipeline --------------- diff --git a/docs/roman/associations/asn_from_list.rst b/docs/roman/associations/asn_from_list.rst index 15c06d6d6..9f5c54013 100644 --- a/docs/roman/associations/asn_from_list.rst +++ b/docs/roman/associations/asn_from_list.rst @@ -9,13 +9,12 @@ Create an association using either the command line tool :func:`romancal.associations.asn_from_list.asn_from_list` -Level2 Associations -^^^^^^^^^^^^^^^^^^^ +Associations +^^^^^^^^^^^^ -Refer to TBD for a full description of Level2 -associations. +Refer to TBD for a full description of associations. -To create a Level2 association, use the following command: +To create an association, use the following command: .. code-block:: python @@ -43,6 +42,12 @@ To create a association with all the detectors for a given exposure from the com .. code-block:: python - asn_from_list -o detector_asn.json --product-name r0000101001001001001_01101_0001_WFI_cal.asdf data/*_cal.asdf + asn_from_list -o detector_asn.json --product-name r0000101001001001001_01101_0001_WFI data/*_cal.asdf where the individual calibrated detector files are in a data subdirectory. + +In addition you can also specify a target for the association file, + +.. code-block:: python + + asn_from_list -o level3_mosaic_asn.json --product-name level3_mosaic --target r274dp63x32y80 data/*_cal.asdf From 86e6486234ed7290efd8cd7cccce61177e430fc8 Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Tue, 17 Sep 2024 09:47:35 -0400 Subject: [PATCH 3/6] rcal-910 update PR number --- CHANGES.rst | 2 +- changes/1411.association.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changes/1411.association.rst diff --git a/CHANGES.rst b/CHANGES.rst index a15d7fe6e..ba9640dc8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,7 @@ associations ------------ -- Add target option for asn_from_list command [#] +- Add target option for asn_from_list command [#1411] mosaic_pipeline diff --git a/changes/1411.association.rst b/changes/1411.association.rst new file mode 100644 index 000000000..4215481a9 --- /dev/null +++ b/changes/1411.association.rst @@ -0,0 +1 @@ +add target to asn_from_list command From 7a5bbaf205ec8d1240c769033a0390c61c7574ec Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Tue, 17 Sep 2024 09:53:33 -0400 Subject: [PATCH 4/6] spelling --- changes/{1411.association.rst => 1411.associations.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changes/{1411.association.rst => 1411.associations.rst} (100%) diff --git a/changes/1411.association.rst b/changes/1411.associations.rst similarity index 100% rename from changes/1411.association.rst rename to changes/1411.associations.rst From 6e9ec5c2b29da377c9fe6fbe2bb967a9de75a9d1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Sep 2024 13:53:46 +0000 Subject: [PATCH 5/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- romancal/associations/asn_from_list.py | 8 ++++---- romancal/associations/association.py | 6 +++--- romancal/associations/lib/dms_base.py | 4 ++-- romancal/associations/lib/rules_elpp_base.py | 6 +++--- romancal/associations/tests/test_asn_from_list.py | 9 ++++++++- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/romancal/associations/asn_from_list.py b/romancal/associations/asn_from_list.py index 1509eef68..03c877c71 100644 --- a/romancal/associations/asn_from_list.py +++ b/romancal/associations/asn_from_list.py @@ -37,9 +37,9 @@ def asn_from_list(items, rule=DMS_ELPP_Base, **kwargs): """ asn = rule() asn._add_items(items, **kwargs) - if 'target' in kwargs.keys(): - target = kwargs['target'] - asn['target'] = target + if "target" in kwargs.keys(): + target = kwargs["target"] + asn["target"] = target return asn @@ -141,7 +141,7 @@ def __init__(self, args=None): rule=rule, product_name=parsed.product_name, acid=parsed.acid, - target=parsed.target + target=parsed.target, ) _, serialized = asn.dump(format=parsed.format) outfile.write(serialized) diff --git a/romancal/associations/association.py b/romancal/associations/association.py index b25d1d941..b9bb7425d 100644 --- a/romancal/associations/association.py +++ b/romancal/associations/association.py @@ -86,8 +86,8 @@ class Association(MutableMapping): def __init__( self, - version_id=None, - target=None, + version_id=None, + target=None, ): self.data = {} self.run_init_hook = True @@ -537,7 +537,7 @@ def finalize(asns): def make_timestamp(): - """Generate a timestamp based on runtime """ + """Generate a timestamp based on runtime""" timestamp = datetime.utcnow().strftime(_TIMESTAMP_TEMPLATE) return timestamp diff --git a/romancal/associations/lib/dms_base.py b/romancal/associations/lib/dms_base.py index 98090c637..660b30804 100644 --- a/romancal/associations/lib/dms_base.py +++ b/romancal/associations/lib/dms_base.py @@ -405,14 +405,14 @@ def update_degraded_status(self): break def update_validity(self, entry): - """ Check/Update the validity of the association""" + """Check/Update the validity of the association""" for test in self.validity.values(): if not test["validated"]: test["validated"] = test["check"](entry) @classmethod def reset_sequence(cls): - """Reset the sequence counter to 1 """ + """Reset the sequence counter to 1""" cls._sequence = Counter(start=1) @classmethod diff --git a/romancal/associations/lib/rules_elpp_base.py b/romancal/associations/lib/rules_elpp_base.py index e6351c2d7..ff4d2f360 100644 --- a/romancal/associations/lib/rules_elpp_base.py +++ b/romancal/associations/lib/rules_elpp_base.py @@ -396,8 +396,8 @@ def _add(self, item): def update_target(self, target): """Update association target field""" - if self.data['target'] is None: - self.data['target'] = target + if self.data["target"] is None: + self.data["target"] = target def _add_items(self, items, product_name=None, with_exptype=False, **kwargs): """Force adding items to the association @@ -444,7 +444,7 @@ def _add_items(self, items, product_name=None, with_exptype=False, **kwargs): def __repr__(self): # flake8: noqa: F821 try: - _ , json_repr = self.ioregistry["json"].dump(self) + _, json_repr = self.ioregistry["json"].dump(self) except KeyValueRegistryError: return str(self.__class__) return json_repr diff --git a/romancal/associations/tests/test_asn_from_list.py b/romancal/associations/tests/test_asn_from_list.py index 766ff8206..f258e47cf 100644 --- a/romancal/associations/tests/test_asn_from_list.py +++ b/romancal/associations/tests/test_asn_from_list.py @@ -33,12 +33,19 @@ def test_association_target(): target_name = "r274dp63x32y80" product_name = "l3_target" rule_name = "Association" - asn = asn_from_list(items, rule=Association, product_name=product_name, target=target_name, version_id='c55') + asn = asn_from_list( + items, + rule=Association, + product_name=product_name, + target=target_name, + version_id="c55", + ) assert asn["asn_rule"] == rule_name assert asn["asn_type"] == "None" assert asn["members"] == items assert asn["target"] == target_name + def test_default_simple(): """Default ELPP association""" product_name = "test_product" From 016db76753af03f2484f529237028c40e85be017 Mon Sep 17 00:00:00 2001 From: Brett Graham Date: Tue, 17 Sep 2024 13:51:32 -0400 Subject: [PATCH 6/6] Remove CHANGES.rst edits --- CHANGES.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index ba9640dc8..e73e53971 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,12 +1,6 @@ 0.16.3 (2024-08-29) =================== -associations ------------- - -- Add target option for asn_from_list command [#1411] - - mosaic_pipeline ---------------