From 3f619a54114762e7c471447e77163c5b36e53c84 Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 6 Mar 2023 12:42:32 -0500 Subject: [PATCH 01/13] disable asdf_extension entry point a few tests required updating as now several types are being converted by asdf-astropy instead of astropy.io.misc.asdf --- asdf/_tests/test_deprecated.py | 7 ----- asdf/_tests/test_entry_points.py | 14 +++------- asdf/_tests/test_versioning.py | 5 ++++ asdf/entry_points.py | 44 ++++---------------------------- 4 files changed, 14 insertions(+), 56 deletions(-) diff --git a/asdf/_tests/test_deprecated.py b/asdf/_tests/test_deprecated.py index c84bf142b..78ae2243b 100644 --- a/asdf/_tests/test_deprecated.py +++ b/asdf/_tests/test_deprecated.py @@ -6,7 +6,6 @@ import asdf._types import asdf.extension import asdf.testing.helpers -from asdf import entry_points from asdf._tests._helpers import assert_extension_correctness from asdf._tests.objects import CustomExtension from asdf._types import CustomType @@ -123,12 +122,6 @@ def test_top_level_asdf_extension_deprecation(): asdf.AsdfExtension -def test_deprecated_entry_point(mock_entry_points): # noqa: F811 - mock_entry_points.append(("asdf_extensions", "legacy", "asdf.tests.test_entry_points:LegacyExtension")) - with pytest.warns(AsdfDeprecationWarning, match=".* uses the deprecated entry point asdf_extensions"): - entry_points.get_extensions() - - def test_asdf_tests_helpers_deprecation(): with pytest.warns(AsdfDeprecationWarning, match="asdf.tests.helpers is deprecated"): if "asdf.tests.helpers" in sys.modules: diff --git a/asdf/_tests/test_entry_points.py b/asdf/_tests/test_entry_points.py index 17c59b372..baaed37ba 100644 --- a/asdf/_tests/test_entry_points.py +++ b/asdf/_tests/test_entry_points.py @@ -8,7 +8,7 @@ from asdf import entry_points from asdf._version import version as asdf_package_version -from asdf.exceptions import AsdfDeprecationWarning, AsdfWarning +from asdf.exceptions import AsdfWarning from asdf.extension import ExtensionProxy from asdf.resource import ResourceMappingProxy @@ -158,17 +158,11 @@ def test_get_extensions(mock_entry_points): mock_entry_points.clear() mock_entry_points.append(("asdf_extensions", "legacy", "asdf._tests.test_entry_points:LegacyExtension")) - with pytest.warns(AsdfDeprecationWarning, match=".* uses the deprecated entry point asdf_extensions"): - extensions = entry_points.get_extensions() - assert len(extensions) == 1 - for e in extensions: - assert isinstance(e, ExtensionProxy) - assert e.package_name == "asdf" - assert e.package_version == asdf_package_version - assert e.legacy is True + extensions = entry_points.get_extensions() + assert len(extensions) == 0 # asdf_extensions is no longer supported mock_entry_points.clear() - mock_entry_points.append(("asdf_extensions", "failing", "asdf._tests.test_entry_points:FauxLegacyExtension")) + mock_entry_points.append(("asdf.extensions", "failing", "asdf._tests.test_entry_points:FauxLegacyExtension")) with pytest.warns(AsdfWarning, match=r"TypeError"): extensions = entry_points.get_extensions() assert len(extensions) == 0 diff --git a/asdf/_tests/test_versioning.py b/asdf/_tests/test_versioning.py index 975544613..4cf882ca6 100644 --- a/asdf/_tests/test_versioning.py +++ b/asdf/_tests/test_versioning.py @@ -1,5 +1,10 @@ from itertools import combinations +import pytest + +import asdf +from asdf.extension._legacy import default_extensions +from asdf.schema import load_schema from asdf.versioning import ( AsdfSpec, AsdfVersion, diff --git a/asdf/entry_points.py b/asdf/entry_points.py index dd1ba018f..3052de763 100644 --- a/asdf/entry_points.py +++ b/asdf/entry_points.py @@ -7,7 +7,7 @@ # see issue https://github.com/asdf-format/asdf/issues/1254 from importlib_metadata import entry_points -from .exceptions import AsdfDeprecationWarning, AsdfWarning +from .exceptions import AsdfWarning from .extension import ExtensionProxy from .resource import ResourceMappingProxy @@ -54,44 +54,10 @@ def _handle_error(e): # Catch errors loading entry points and warn instead of raising try: with warnings.catch_warnings(): - if entry_point.group == LEGACY_EXTENSIONS_GROUP: - if entry_point.name in ("astropy", "astropy-asdf"): - # Filter out the legacy `CustomType` deprecation warnings from the - # deprecated astropy.io.misc.asdf - # Testing will turn these into errors - # Most of the astropy.io.misc.asdf deprecation warnings fall under this category - warnings.filterwarnings( - "ignore", - category=AsdfDeprecationWarning, - message=r".*from astropy.io.misc.asdf.* subclasses the deprecated CustomType .*", - ) - warnings.filterwarnings( - "ignore", - category=AsdfDeprecationWarning, - message="asdf.types is deprecated", - ) - warnings.filterwarnings( - "ignore", - category=AsdfDeprecationWarning, - message="AsdfExtension is deprecated", - ) - warnings.filterwarnings( - "ignore", - category=AsdfDeprecationWarning, - message="BuiltinExtension is deprecated", - ) - warnings.filterwarnings( - "ignore", - category=AsdfDeprecationWarning, - message="asdf.tests.helpers is deprecated", - ) - elif entry_point.name != "builtin": - warnings.warn( - f"{package_name} uses the deprecated entry point {LEGACY_EXTENSIONS_GROUP}. " - f"Please use the new extension api and entry point {EXTENSIONS_GROUP}: " - "https://asdf.readthedocs.io/en/stable/asdf/extending/extensions.html", - AsdfDeprecationWarning, - ) + if entry_point.group == LEGACY_EXTENSIONS_GROUP and entry_point.name != "builtin": + # for now, the builtin extension is still registered via asdf_extensions + # so we only load this legacy extension and ignore all non-builtin extensions + continue elements = entry_point.load()() except Exception as e: From 76b28e54de07b8644f44e63f1b7944a757a67794 Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 6 Mar 2023 12:47:25 -0500 Subject: [PATCH 02/13] remove top level legacy extension deprecated modules --- asdf/_tests/test_deprecated.py | 47 ---------------------------------- asdf/asdftypes.py | 14 ---------- asdf/resolver.py | 31 ---------------------- asdf/type_index.py | 28 -------------------- asdf/types.py | 28 -------------------- 5 files changed, 148 deletions(-) delete mode 100644 asdf/asdftypes.py delete mode 100644 asdf/resolver.py delete mode 100644 asdf/type_index.py delete mode 100644 asdf/types.py diff --git a/asdf/_tests/test_deprecated.py b/asdf/_tests/test_deprecated.py index 78ae2243b..8626ed319 100644 --- a/asdf/_tests/test_deprecated.py +++ b/asdf/_tests/test_deprecated.py @@ -21,49 +21,12 @@ class NewCustomType(CustomType): pass -def test_resolver_module_deprecation(): - with pytest.warns(AsdfDeprecationWarning, match="^asdf.resolver is deprecated.*$"): - # importlib.reload doesn't appear to work here likely because of the - # sys.module and __file__ changes in asdf.resolver - if "asdf.resolver" in sys.modules: - del sys.modules["asdf.resolver"] - import asdf.resolver - # resolver does not define an __all__ so we will define one here - # for testing purposes - resolver_all = [ - "Resolver", - "ResolverChain", - "DEFAULT_URL_MAPPING", - "DEFAULT_TAG_TO_URL_MAPPING", - "default_url_mapping", - "default_tag_to_url_mapping", - "default_resolver", - ] - for attr in dir(asdf.resolver): - if attr not in resolver_all: - continue - with pytest.warns(AsdfDeprecationWarning, match="^asdf.resolver is deprecated.*$"): - getattr(asdf.resolver, attr) - - def test_assert_extension_correctness_deprecation(): extension = CustomExtension() with pytest.warns(AsdfDeprecationWarning, match="assert_extension_correctness is deprecated.*"): assert_extension_correctness(extension) -def test_type_index_module_deprecation(): - with pytest.warns(AsdfDeprecationWarning, match="^asdf.type_index is deprecated.*$"): - # importlib.reload doesn't appear to work here likely because of the - # sys.module and __file__ changes in asdf.type_index - if "asdf.type_index" in sys.modules: - del sys.modules["asdf.type_index"] - import asdf.type_index - for attr in asdf.type_index.__all__: - with pytest.warns(AsdfDeprecationWarning, match="^asdf.type_index is deprecated.*$"): - getattr(asdf.type_index, attr) - - @pytest.mark.parametrize("attr", ["url_mapping", "tag_mapping", "resolver", "extension_list", "type_index"]) def test_asdffile_legacy_extension_api_attr_deprecations(attr): with asdf.AsdfFile() as af, pytest.warns(AsdfDeprecationWarning, match=f"AsdfFile.{attr} is deprecated"): @@ -80,16 +43,6 @@ def test_asdfile_run_modifying_hook_deprecation(): af.run_modifying_hook("foo") -def test_types_module_deprecation(): - with pytest.warns(AsdfDeprecationWarning, match="^asdf.types is deprecated.*$"): - if "asdf.types" in sys.modules: - del sys.modules["asdf.types"] - import asdf.types - for attr in asdf.types.__all__: - with pytest.warns(AsdfDeprecationWarning, match="^asdf.types is deprecated.*$"): - getattr(asdf.types, attr) - - def test_default_extensions_deprecation(): with pytest.warns(AsdfDeprecationWarning, match="default_extensions is deprecated"): asdf.extension.default_extensions diff --git a/asdf/asdftypes.py b/asdf/asdftypes.py deleted file mode 100644 index b91927e5f..000000000 --- a/asdf/asdftypes.py +++ /dev/null @@ -1,14 +0,0 @@ -import warnings - -from ._types import AsdfType, CustomType, ExtensionTypeMeta, format_tag -from .exceptions import AsdfDeprecationWarning - -# This is not exhaustive, but represents the public API -from .versioning import join_tag_version, split_tag_version - -__all__ = ["join_tag_version", "split_tag_version", "AsdfType", "CustomType", "format_tag", "ExtensionTypeMeta"] - -warnings.warn( - "The module asdf.asdftypes has been deprecated and will be removed in 3.0. Use asdf.types instead.", - AsdfDeprecationWarning, -) diff --git a/asdf/resolver.py b/asdf/resolver.py deleted file mode 100644 index 505f2469a..000000000 --- a/asdf/resolver.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -This module is deprecated. Please see :ref:`extending_resources` -""" -import warnings - -from . import _resolver -from .exceptions import AsdfDeprecationWarning - - -def _warn(): - warnings.warn( - "asdf.resolver is deprecated " - "Please see Resources " - "https://asdf.readthedocs.io/en/stable/asdf/extending/resources.html", - AsdfDeprecationWarning, - ) - - -_warn() - - -def __getattr__(name): - attr = getattr(_resolver, name) - _warn() - if hasattr(attr, "__module__"): - attr.__module__ = __name__ - return attr - - -def __dir__(): - return dir(_resolver) diff --git a/asdf/type_index.py b/asdf/type_index.py deleted file mode 100644 index f9942d263..000000000 --- a/asdf/type_index.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -This module is deprecated. Please see :ref:`extending_converters` -""" -import warnings - -from . import _type_index -from .exceptions import AsdfDeprecationWarning - - -def _warn(): - warnings.warn( - "asdf.type_index is deprecated " - "Please see the new extension API " - "https://asdf.readthedocs.io/en/stable/asdf/extending/converters.html", - AsdfDeprecationWarning, - ) - - -_warn() -__all__ = _type_index.__all__ - - -def __getattr__(name): - attr = getattr(_type_index, name) - _warn() - if hasattr(attr, "__module__"): - attr.__module__ = __name__ - return attr diff --git a/asdf/types.py b/asdf/types.py deleted file mode 100644 index 2bf18aaf8..000000000 --- a/asdf/types.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -This module is deprecated. Please see :ref:`extending_converters` -""" -import warnings - -from . import _types -from .exceptions import AsdfDeprecationWarning - - -def _warn(): - warnings.warn( - "asdf.types is deprecated " - "Please see the new extension API " - "https://asdf.readthedocs.io/en/stable/asdf/extending/converters.html", - AsdfDeprecationWarning, - ) - - -_warn() -__all__ = _types.__all__ - - -def __getattr__(name): - attr = getattr(_types, name) - _warn() - if hasattr(attr, "__module__") and name != "format_tag": - attr.__module__ = __name__ - return attr From 6db16679e3a85bc6773f1041bbe696cdc28cc16b Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 6 Mar 2023 13:00:36 -0500 Subject: [PATCH 03/13] removed deprecated asdf.extension legacy api objects and functions --- asdf/__init__.py | 12 ------ asdf/_tests/test_api.py | 9 ++--- asdf/_tests/test_deprecated.py | 26 ------------- asdf/_tests/test_extension.py | 12 ++---- asdf/_tests/test_schema.py | 18 ++++----- asdf/extension/__init__.py | 70 ---------------------------------- asdf/schema.py | 18 +-------- 7 files changed, 17 insertions(+), 148 deletions(-) diff --git a/asdf/__init__.py b/asdf/__init__.py index 5a5fc4157..d5cdd4abd 100644 --- a/asdf/__init__.py +++ b/asdf/__init__.py @@ -6,7 +6,6 @@ __all__ = [ "AsdfFile", "CustomType", - "AsdfExtension", "Stream", "open", "IntegerType", @@ -30,14 +29,3 @@ from .stream import Stream from .tags.core import IntegerType from .tags.core.external_reference import ExternalArrayReference - - -def __getattr__(name): - if name == "AsdfExtension": - # defer import to only issue deprecation warning when - # asdf.AsdfExtension is used - from asdf import extension - - return extension.AsdfExtension - msg = f"module {__name__!r} has no attribute {name!r}" - raise AttributeError(msg) diff --git a/asdf/_tests/test_api.py b/asdf/_tests/test_api.py index 24ceec8f6..d0059aa36 100644 --- a/asdf/_tests/test_api.py +++ b/asdf/_tests/test_api.py @@ -12,8 +12,9 @@ from numpy.testing import assert_array_equal import asdf +import asdf.extension._legacy as _legacy_extension from asdf import _resolver as resolver -from asdf import config_context, extension, get_config, schema, treeutil, versioning +from asdf import config_context, get_config, treeutil, versioning from asdf.exceptions import AsdfDeprecationWarning, AsdfWarning from asdf.extension import ExtensionProxy @@ -262,7 +263,7 @@ def test_tag_to_schema_resolver_deprecation(): ff.tag_to_schema_resolver("foo") with pytest.warns(AsdfDeprecationWarning): - extension_list = extension.default_extensions.extension_list + extension_list = _legacy_extension.default_extensions.extension_list extension_list.tag_to_schema_resolver("foo") @@ -472,15 +473,13 @@ def test_resolver_deprecations(): resolver.default_resolver, resolver.default_tag_to_url_mapping, resolver.default_url_mapping, - schema.default_ext_resolver, ]: with pytest.warns(AsdfDeprecationWarning): resolver_method("foo") def test_get_default_resolver(): - with pytest.warns(AsdfDeprecationWarning, match="get_default_resolver is deprecated"): - resolver = extension.get_default_resolver() + resolver = _legacy_extension.get_default_resolver() result = resolver("tag:stsci.edu:asdf/core/ndarray-1.0.0") diff --git a/asdf/_tests/test_deprecated.py b/asdf/_tests/test_deprecated.py index 8626ed319..12d5ac061 100644 --- a/asdf/_tests/test_deprecated.py +++ b/asdf/_tests/test_deprecated.py @@ -43,38 +43,12 @@ def test_asdfile_run_modifying_hook_deprecation(): af.run_modifying_hook("foo") -def test_default_extensions_deprecation(): - with pytest.warns(AsdfDeprecationWarning, match="default_extensions is deprecated"): - asdf.extension.default_extensions - - -def test_default_resolver(): - with pytest.warns(AsdfDeprecationWarning, match="get_default_resolver is deprecated"): - asdf.extension.get_default_resolver() - - -def test_get_cached_asdf_extension_list_deprecation(): - with pytest.warns(AsdfDeprecationWarning, match="get_cached_asdf_extension_list is deprecated"): - asdf.extension.get_cached_asdf_extension_list([]) - - def test_asdf_type_format_tag(): with pytest.warns(AsdfDeprecationWarning, match="asdf.types.format_tag is deprecated"): asdf._types.format_tag asdf.testing.helpers.format_tag -@pytest.mark.parametrize("name", ["AsdfExtension", "AsdfExtensionList", "BuiltinExtension"]) -def test_extension_class_deprecation(name): - with pytest.warns(AsdfDeprecationWarning, match=f"{name} is deprecated"): - getattr(asdf.extension, name) - - -def test_top_level_asdf_extension_deprecation(): - with pytest.warns(AsdfDeprecationWarning, match="AsdfExtension is deprecated"): - asdf.AsdfExtension - - def test_asdf_tests_helpers_deprecation(): with pytest.warns(AsdfDeprecationWarning, match="asdf.tests.helpers is deprecated"): if "asdf.tests.helpers" in sys.modules: diff --git a/asdf/_tests/test_extension.py b/asdf/_tests/test_extension.py index c37a0579b..00e0c04c7 100644 --- a/asdf/_tests/test_extension.py +++ b/asdf/_tests/test_extension.py @@ -15,10 +15,9 @@ ManifestExtension, TagDefinition, Validator, - get_cached_asdf_extension_list, get_cached_extension_manager, ) -from asdf.extension._legacy import AsdfExtension, BuiltinExtension +from asdf.extension._legacy import AsdfExtension, BuiltinExtension, get_cached_asdf_extension_list def test_builtin_extension(): @@ -606,12 +605,9 @@ def test_converter_proxy(): def test_get_cached_asdf_extension_list(): extension = LegacyExtension() - with pytest.warns(AsdfDeprecationWarning, match="get_cached_asdf_extension_list is deprecated"): - extension_list = get_cached_asdf_extension_list([extension]) - with pytest.warns(AsdfDeprecationWarning, match="get_cached_asdf_extension_list is deprecated"): - assert get_cached_asdf_extension_list([extension]) is extension_list - with pytest.warns(AsdfDeprecationWarning, match="get_cached_asdf_extension_list is deprecated"): - assert get_cached_asdf_extension_list([LegacyExtension()]) is not extension_list + extension_list = get_cached_asdf_extension_list([extension]) + assert get_cached_asdf_extension_list([extension]) is extension_list + assert get_cached_asdf_extension_list([LegacyExtension()]) is not extension_list def test_manifest_extension(): diff --git a/asdf/_tests/test_schema.py b/asdf/_tests/test_schema.py index 35daf585b..a9337c817 100644 --- a/asdf/_tests/test_schema.py +++ b/asdf/_tests/test_schema.py @@ -10,10 +10,11 @@ import asdf.testing.helpers from asdf import _resolver as resolver from asdf import _types as types -from asdf import config_context, constants, extension, get_config, schema, tagged, util, yamlutil +from asdf import config_context, constants, get_config, schema, tagged, util, yamlutil from asdf._tests import _helpers as helpers from asdf._tests.objects import CustomExtension from asdf.exceptions import AsdfConversionWarning, AsdfDeprecationWarning, AsdfWarning +from asdf.extension import _legacy as _legacy_extension with pytest.warns(AsdfDeprecationWarning, match=".*subclasses the deprecated CustomType.*"): @@ -137,8 +138,7 @@ def test_load_schema_with_tag_address(tmp_path): def test_load_schema_with_file_url(tmp_path): - with pytest.warns(AsdfDeprecationWarning, match="get_default_resolver is deprecated"): - schema_def = """ + schema_def = """ %YAML 1.1 %TAG !asdf! tag:stsci.edu:asdf/ --- @@ -153,9 +153,9 @@ def test_load_schema_with_file_url(tmp_path): required: [foobar] ... - """.format( - extension.get_default_resolver()("tag:stsci.edu:asdf/core/ndarray-1.0.0"), - ) + """.format( + _legacy_extension.get_default_resolver()("tag:stsci.edu:asdf/core/ndarray-1.0.0"), + ) schema_path = tmp_path / "nugatory.yaml" schema_path.write_bytes(schema_def.encode()) @@ -645,11 +645,9 @@ def test_self_reference_resolution(): def test_schema_resolved_via_entry_points(): """Test that entry points mappings to core schema works""" - with pytest.warns(AsdfDeprecationWarning, match="get_default_resolver is deprecated"): - r = extension.get_default_resolver() + r = _legacy_extension.get_default_resolver() tag = asdf.testing.helpers.format_tag("stsci.edu", "asdf", "1.0.0", "fits/fits") - with pytest.warns(AsdfDeprecationWarning, match="default_extensions is deprecated"): - url = extension.default_extensions.extension_list.tag_mapping(tag) + url = _legacy_extension.default_extensions.extension_list.tag_mapping(tag) s = schema.load_schema(url, resolver=r, resolve_references=True) assert tag in repr(s) diff --git a/asdf/extension/__init__.py b/asdf/extension/__init__.py index 790c3300a..7e4f96e2a 100644 --- a/asdf/extension/__init__.py +++ b/asdf/extension/__init__.py @@ -2,11 +2,8 @@ Support for plugins that extend asdf to serialize additional custom types. """ -import warnings -from asdf.exceptions import AsdfDeprecationWarning -from . import _legacy from ._compressor import Compressor from ._converter import Converter, ConverterProxy from ._extension import Extension, ExtensionProxy @@ -27,71 +24,4 @@ "ConverterProxy", "Compressor", "Validator", - # Legacy API - "AsdfExtension", - "AsdfExtensionList", - "BuiltinExtension", - "default_extensions", - "get_default_resolver", - "get_cached_asdf_extension_list", ] - - -def get_cached_asdf_extension_list(extensions): - """ - Get a previously created AsdfExtensionList for the specified - extensions, or create and cache one if necessary. Building - the type index is expensive, so it helps performance to reuse - the index when possible. - Parameters - ---------- - extensions : list of asdf.extension.AsdfExtension - Returns - ------- - asdf.extension.AsdfExtensionList - """ - from ._legacy import get_cached_asdf_extension_list - - warnings.warn( - "get_cached_asdf_extension_list is deprecated. " - "Please see the new extension API " - "https://asdf.readthedocs.io/en/stable/asdf/extending/converters.html", - AsdfDeprecationWarning, - ) - return get_cached_asdf_extension_list(extensions) - - -def get_default_resolver(): - """ - Get the resolver that includes mappings from all installed extensions. - """ - from ._legacy import get_default_resolver - - warnings.warn( - "get_default_resolver is deprecated. " - "Please see the new extension API " - "https://asdf.readthedocs.io/en/stable/asdf/extending/converters.html", - AsdfDeprecationWarning, - ) - return get_default_resolver() - - -_deprecated_legacy = { - "default_extensions", - "AsdfExtension", - "AsdfExtensionList", - "BuiltinExtension", -} - - -def __getattr__(name): - if name in _deprecated_legacy: - warnings.warn( - f"{name} is deprecated. " - "Please see the new extension API " - "https://asdf.readthedocs.io/en/stable/asdf/extending/converters.html", - AsdfDeprecationWarning, - ) - return getattr(_legacy, name) - msg = f"module {__name__!r} has no attribute {name!r}" - raise AttributeError(msg) diff --git a/asdf/schema.py b/asdf/schema.py index e6df3e536..f1d960164 100644 --- a/asdf/schema.py +++ b/asdf/schema.py @@ -13,7 +13,7 @@ from jsonschema import validators as mvalidators from jsonschema.exceptions import RefResolutionError, ValidationError -from . import constants, extension, generic_io, reference, tagged, treeutil, util, versioning, yamlutil +from . import constants, generic_io, reference, tagged, treeutil, util, versioning, yamlutil from .config import get_config from .exceptions import AsdfDeprecationWarning, AsdfWarning from .extension import _legacy @@ -21,24 +21,8 @@ YAML_SCHEMA_METASCHEMA_ID = "http://stsci.edu/schemas/yaml-schema/draft-01" - __all__ = ["validate", "fill_defaults", "remove_defaults", "check_schema"] - -def default_ext_resolver(uri): - """ - Resolver that uses tag/url mappings from all installed extensions - """ - # Deprecating this because it doesn't play nicely with the caching on - # load_schema(...). - warnings.warn( - "The 'default_ext_resolver(...)' function is deprecated. Use " - "'asdf.extension.get_default_resolver()(...)' instead.", - AsdfDeprecationWarning, - ) - return extension.get_default_resolver()(uri) - - PYTHON_TYPE_TO_YAML_TAG = { None: "null", str: "str", From 710a61e1f1240ccfb0149ddedf5418d1df7fe3d0 Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 6 Mar 2023 13:07:29 -0500 Subject: [PATCH 04/13] remove legacy extension api deprecated AsdfFile methods/attrs --- asdf/_tests/commands/tests/test_tags.py | 6 +- asdf/_tests/tags/core/tests/test_ndarray.py | 4 +- asdf/_tests/test_api.py | 10 -- asdf/_tests/test_deprecated.py | 16 --- asdf/_tests/test_schema.py | 5 +- asdf/asdf.py | 108 -------------------- 6 files changed, 5 insertions(+), 144 deletions(-) diff --git a/asdf/_tests/commands/tests/test_tags.py b/asdf/_tests/commands/tests/test_tags.py index 16d299790..4afa252a4 100644 --- a/asdf/_tests/commands/tests/test_tags.py +++ b/asdf/_tests/commands/tests/test_tags.py @@ -4,7 +4,6 @@ from asdf import AsdfFile from asdf.commands import list_tags -from asdf.exceptions import AsdfDeprecationWarning @pytest.mark.parametrize("display_classes", [True, False]) @@ -20,8 +19,7 @@ def test_all_tags_present(): tags = {line.strip() for line in iostream.readlines()} af = AsdfFile() - with pytest.warns(AsdfDeprecationWarning, match="AsdfFile.type_index is deprecated"): - for tag in af.type_index._type_by_tag: - assert tag in tags + for tag in af._type_index._type_by_tag: + assert tag in tags for tag in af.extension_manager._converters_by_tag: assert tag in tags diff --git a/asdf/_tests/tags/core/tests/test_ndarray.py b/asdf/_tests/tags/core/tests/test_ndarray.py index 707d6637e..da9a0cbdb 100644 --- a/asdf/_tests/tags/core/tests/test_ndarray.py +++ b/asdf/_tests/tags/core/tests/test_ndarray.py @@ -14,7 +14,6 @@ from asdf import util from asdf._tests import _helpers as helpers from asdf._tests.objects import CustomTestType -from asdf.exceptions import AsdfDeprecationWarning from asdf.tags.core import ndarray from . import data as test_data @@ -131,8 +130,7 @@ def test_dont_load_data(): buff.seek(0) with asdf.open(buff) as ff: - with pytest.warns(AsdfDeprecationWarning, match="AsdfFile.run_hook is deprecated"): - ff.run_hook("reserve_blocks") + ff._run_hook("reserve_blocks") # repr and str shouldn't load data str(ff.tree["science_data"]) diff --git a/asdf/_tests/test_api.py b/asdf/_tests/test_api.py index d0059aa36..1457f0bb7 100644 --- a/asdf/_tests/test_api.py +++ b/asdf/_tests/test_api.py @@ -257,16 +257,6 @@ def test_copy(tmp_path): assert_array_equal(ff2.tree["my_array"], ff2.tree["my_array"]) -def test_tag_to_schema_resolver_deprecation(): - ff = asdf.AsdfFile() - with pytest.warns(AsdfDeprecationWarning): - ff.tag_to_schema_resolver("foo") - - with pytest.warns(AsdfDeprecationWarning): - extension_list = _legacy_extension.default_extensions.extension_list - extension_list.tag_to_schema_resolver("foo") - - def test_access_tree_outside_handler(tmp_path): tempname = str(tmp_path / "test.asdf") diff --git a/asdf/_tests/test_deprecated.py b/asdf/_tests/test_deprecated.py index 12d5ac061..4da798929 100644 --- a/asdf/_tests/test_deprecated.py +++ b/asdf/_tests/test_deprecated.py @@ -27,22 +27,6 @@ def test_assert_extension_correctness_deprecation(): assert_extension_correctness(extension) -@pytest.mark.parametrize("attr", ["url_mapping", "tag_mapping", "resolver", "extension_list", "type_index"]) -def test_asdffile_legacy_extension_api_attr_deprecations(attr): - with asdf.AsdfFile() as af, pytest.warns(AsdfDeprecationWarning, match=f"AsdfFile.{attr} is deprecated"): - getattr(af, attr) - - -def test_asdfile_run_hook_deprecation(): - with asdf.AsdfFile() as af, pytest.warns(AsdfDeprecationWarning, match="AsdfFile.run_hook is deprecated"): - af.run_hook("foo") - - -def test_asdfile_run_modifying_hook_deprecation(): - with asdf.AsdfFile() as af, pytest.warns(AsdfDeprecationWarning, match="AsdfFile.run_modifying_hook is deprecated"): - af.run_modifying_hook("foo") - - def test_asdf_type_format_tag(): with pytest.warns(AsdfDeprecationWarning, match="asdf.types.format_tag is deprecated"): asdf._types.format_tag diff --git a/asdf/_tests/test_schema.py b/asdf/_tests/test_schema.py index a9337c817..d64a71f7b 100644 --- a/asdf/_tests/test_schema.py +++ b/asdf/_tests/test_schema.py @@ -264,9 +264,8 @@ def test_asdf_file_resolver_hashing(): a1 = asdf.AsdfFile() a2 = asdf.AsdfFile() - with pytest.warns(AsdfDeprecationWarning, match="AsdfFile.resolver is deprecated"): - assert hash(a1.resolver) == hash(a2.resolver) - assert a1.resolver == a2.resolver + assert hash(a1._resolver) == hash(a2._resolver) + assert a1._resolver == a2._resolver def test_load_schema_from_resource_mapping(): diff --git a/asdf/asdf.py b/asdf/asdf.py index 6e7ae6391..e5a99db03 100644 --- a/asdf/asdf.py +++ b/asdf/asdf.py @@ -254,23 +254,6 @@ def extension_manager(self): self._extension_manager = get_cached_extension_manager(self._user_extensions + self._plugin_extensions) return self._extension_manager - @property - def extension_list(self): - """ - Get the AsdfExtensionList for this AsdfFile. - - Returns - ------- - asdf.extension.AsdfExtensionList - """ - warnings.warn( - "AsdfFile.extension_list is deprecated. " - "Please see the new extension API " - "https://asdf.readthedocs.io/en/stable/asdf/extending/converters.html", - AsdfDeprecationWarning, - ) - return self._extension_list - @property def _extension_list(self): if self._extension_list_ is None: @@ -498,70 +481,22 @@ def uri(self): return self._fd._uri return None - @property - def tag_to_schema_resolver(self): - warnings.warn( - "The 'tag_to_schema_resolver' property is deprecated. Use 'tag_mapping' instead.", - AsdfDeprecationWarning, - ) - return self._tag_to_schema_resolver - @property def _tag_to_schema_resolver(self): return self._extension_list.tag_mapping - @property - def tag_mapping(self): - warnings.warn( - "AsdfFile.tag_mapping is deprecated. " - "Please see Manifests " - "https://asdf.readthedocs.io/en/stable/asdf/extending/manifests.html", - AsdfDeprecationWarning, - ) - return self._tag_mapping - @property def _tag_mapping(self): return self._extension_list.tag_mapping - @property - def url_mapping(self): - warnings.warn( - "AsdfFile.url_mapping is deprecated. " - "Please see Resources " - "https://asdf.readthedocs.io/en/stable/asdf/extending/resources.html", - AsdfDeprecationWarning, - ) - return self._url_mapping - @property def _url_mapping(self): return self._extension_list.url_mapping - @property - def resolver(self): - warnings.warn( - "AsdfFile.resolver is deprecated. " - "Please see Resources " - "https://asdf.readthedocs.io/en/stable/asdf/extending/resources.html", - AsdfDeprecationWarning, - ) - return self._resolver - @property def _resolver(self): return self._extension_list.resolver - @property - def type_index(self): - warnings.warn( - "AsdfFile.type_index is deprecated. " - "Please see the new extension API " - "https://asdf.readthedocs.io/en/stable/asdf/extending/converters.html", - AsdfDeprecationWarning, - ) - return self._type_index - @property def _type_index(self): return self._extension_list.type_index @@ -1450,25 +1385,6 @@ def resolve_references(self, **kwargs): # tree will be validated. self.tree = reference.resolve_references(self._tree, self) - def run_hook(self, hookname): - """ - Run a "hook" for each custom type found in the tree. - - Parameters - ---------- - hookname : str - The name of the hook. If a `asdf.types.AsdfType` is found with a method - with this name, it will be called for every instance of the - corresponding custom type in the tree. - """ - warnings.warn( - "AsdfFile.run_hook is deprecated. " - "Please see the new extension API " - "https://asdf.readthedocs.io/en/stable/asdf/extending/converters.html", - AsdfDeprecationWarning, - ) - self._run_hook(hookname) - def _run_hook(self, hookname): type_index = self._type_index @@ -1480,30 +1396,6 @@ def _run_hook(self, hookname): if hook is not None: hook(node, self) - def run_modifying_hook(self, hookname, validate=True): - """ - Run a "hook" for each custom type found in the tree. The hook - is free to return a different object in order to modify the - tree. - - Parameters - ---------- - hookname : str - The name of the hook. If a `asdf.types.AsdfType` is found with a method - with this name, it will be called for every instance of the - corresponding custom type in the tree. - - validate : bool - When `True` (default) validate the resulting tree. - """ - warnings.warn( - "AsdfFile.run_modifying_hook is deprecated. " - "Please see the new extension API " - "https://asdf.readthedocs.io/en/stable/asdf/extending/converters.html", - AsdfDeprecationWarning, - ) - return self._run_modifying_hook(hookname, validate=validate) - def _run_modifying_hook(self, hookname, validate=True): type_index = self._type_index From 61727f9e171c254f77714d4851c4c668540c9bc1 Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 6 Mar 2023 13:13:41 -0500 Subject: [PATCH 05/13] update changes --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index 006727c8a..527f5786b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -9,6 +9,7 @@ The ASDF Standard is at v1.6.0 ``all_array_compression_kwargs`` to ``asdf.config.AsdfConfig`` [#1468] - Move built-in tags to converters (except ndarray and integer). [#1474] - Add block storage support to Converter [#1508] +- Remove deprecated legacy extension API [#1464] 2.15.0 (2023-03-28) ------------------- From 057d3b9700c850e347454e7e459135de19bf5226 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 7 Mar 2023 11:35:53 -0500 Subject: [PATCH 06/13] remove asdf.CustomType, cleanup docs --- CHANGES.rst | 2 +- asdf/__init__.py | 2 - asdf/_tests/objects.py | 3 +- asdf/_tests/test_types.py | 4 +- asdf/asdf.py | 20 +- asdf/extension/_extension.py | 4 +- docs/asdf/deprecations.rst | 28 +- docs/asdf/developer_api.rst | 7 +- docs/asdf/developer_versioning.rst | 29 + docs/asdf/extending/legacy.rst | 926 ----------------------------- docs/asdf/extending/schemas.rst | 55 ++ docs/asdf/user_api.rst | 2 +- docs/index.rst | 1 - 13 files changed, 118 insertions(+), 965 deletions(-) delete mode 100644 docs/asdf/extending/legacy.rst diff --git a/CHANGES.rst b/CHANGES.rst index 527f5786b..49666a369 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -28,7 +28,7 @@ The ASDF Standard is at v1.6.0 ones that we can successfully build and test against. [#1360] - Provide more informative filename when failing to open a file [#1357] - Add new plugin type for custom schema validators. [#1328] -- Add AsdfDeprecationWarning to `~asdf.types.CustomType` [#1359] +- Add AsdfDeprecationWarning to ``asdf.types.CustomType`` [#1359] - Throw more useful error when provided with a path containing an extra leading slash [#1356] - Add AsdfDeprecationWarning to AsdfInFits. Support for reading and diff --git a/asdf/__init__.py b/asdf/__init__.py index d5cdd4abd..59492e5c3 100644 --- a/asdf/__init__.py +++ b/asdf/__init__.py @@ -5,7 +5,6 @@ __all__ = [ "AsdfFile", - "CustomType", "Stream", "open", "IntegerType", @@ -21,7 +20,6 @@ from jsonschema import ValidationError from ._convenience import info -from ._types import CustomType from ._version import version as __version__ from .asdf import AsdfFile from .asdf import open_asdf as open diff --git a/asdf/_tests/objects.py b/asdf/_tests/objects.py index ba0e63245..95e08f07b 100644 --- a/asdf/_tests/objects.py +++ b/asdf/_tests/objects.py @@ -1,6 +1,7 @@ import pytest -from asdf import CustomType, util +from asdf import util +from asdf._types import CustomType from asdf.exceptions import AsdfDeprecationWarning from ._helpers import get_test_data_path diff --git a/asdf/_tests/test_types.py b/asdf/_tests/test_types.py index 1571ec155..add2d4fde 100644 --- a/asdf/_tests/test_types.py +++ b/asdf/_tests/test_types.py @@ -36,7 +36,7 @@ def inverse(self, value): with pytest.warns(AsdfDeprecationWarning, match=".*subclasses the deprecated CustomType.*"): - class FractionWithInverseType(asdf.CustomType): + class FractionWithInverseType(types.CustomType): name = "fraction_with_inverse" organization = "nowhere.org" version = (1, 0, 0) @@ -604,7 +604,7 @@ def __init__(self, bar): match=".*subclasses the deprecated CustomType.*", ): - class FooType(asdf.CustomType): + class FooType(types.CustomType): name = "foo" version = (1, 0, 0) supported_versions = [(1, 1, 0), (1, 2, 0)] diff --git a/asdf/asdf.py b/asdf/asdf.py index e5a99db03..7cec72d14 100644 --- a/asdf/asdf.py +++ b/asdf/asdf.py @@ -70,9 +70,7 @@ def __init__( extensions : object, optional Additional extensions to use when reading and writing the file. - May be any of the following: `asdf.extension.AsdfExtension`, - `asdf.extension.Extension`, `asdf.extension.AsdfExtensionList` - or a `list` of extensions. + May be an `asdf.extension.Extension` or a `list` of extensions. version : str, optional The ASDF Standard version. If not provided, defaults to the @@ -235,7 +233,7 @@ def extensions(self, value): Parameters ---------- - value : list of asdf.extension.AsdfExtension or asdf.extension.Extension + value : list of asdf.extension.Extension """ self._user_extensions = self._process_user_extensions(value) self._extension_manager = None @@ -352,9 +350,7 @@ def _process_user_extensions(self, extensions): Parameters ---------- extensions : object - May be any of the following: `asdf.extension.AsdfExtension`, - `asdf.extension.Extension`, `asdf.extension.AsdfExtensionList` - or a `list` of extensions. + May be an `asdf.extension.Extension` or a `list` of extensions. Returns ------- @@ -368,7 +364,7 @@ def _process_user_extensions(self, extensions): extensions = extensions.extensions if not isinstance(extensions, list): - msg = "The extensions parameter must be an extension, list of extensions, or instance of AsdfExtensionList" + msg = "The extensions parameter must be an extension or list of extensions" raise TypeError(msg) extensions = [ExtensionProxy.maybe_wrap(e) for e in extensions] @@ -1751,9 +1747,7 @@ def open_asdf( extensions : object, optional Additional extensions to use when reading and writing the file. - May be any of the following: `asdf.extension.AsdfExtension`, - `asdf.extension.Extension`, `asdf.extension.AsdfExtensionList` - or a `list` of extensions. + May be an `asdf.extension.Extension` or a `list` of extensions. ignore_version_mismatch : bool, optional When `True`, do not raise warnings for mismatched schema versions. @@ -1889,7 +1883,7 @@ def _mark_extension_used(self, extension): Parameters ---------- - extension : asdf.extension.AsdfExtension or asdf.extension.Extension + extension : asdf.extension.Extension """ self.__extensions_used.add(ExtensionProxy.maybe_wrap(extension)) @@ -1900,7 +1894,7 @@ def _extensions_used(self): Returns ------- - set of asdf.extension.AsdfExtension or asdf.extension.Extension + set of asdf.extension.Extension """ return self.__extensions_used diff --git a/asdf/extension/_extension.py b/asdf/extension/_extension.py index 7cccd0be1..c5693b407 100644 --- a/asdf/extension/_extension.py +++ b/asdf/extension/_extension.py @@ -325,7 +325,7 @@ def delegate(self): Returns ------- - asdf.extension.Extension or asdf.extension.AsdfExtension + asdf.extension.Extension """ return self._delegate @@ -367,7 +367,7 @@ def class_name(self): @property def legacy(self): """ - Get the extension's legacy flag. Subclasses of `asdf.extension.AsdfExtension` + Get the extension's legacy flag. Subclasses of ``asdf.extension.AsdfExtension`` are marked `True`. Returns diff --git a/docs/asdf/deprecations.rst b/docs/asdf/deprecations.rst index f7eeecaca..ba49497e2 100644 --- a/docs/asdf/deprecations.rst +++ b/docs/asdf/deprecations.rst @@ -21,31 +21,31 @@ Legacy Extension API Deprecation ================================ A large number of `asdf.exceptions.AsdfDeprecationWarning` messages appear related to -use of the :ref:`legacy extension api `. Some examples include: +use of the ``legacy extension api``. Some examples include: -* `asdf.types` -* `asdf.types.CustomType` +* ``asdf.types`` +* ``asdf.types.CustomType`` * ``asdf.type_index`` * ``asdf.resolver`` * the ``asdf_extensions`` entry point * portions of asdf.extension including: - * `asdf.extension.AsdfExtension` - * `asdf.extension.AsdfExtensionList` - * `asdf.extension.BuiltinExtension` + * ``asdf.extension.AsdfExtension`` + * ``asdf.extension.AsdfExtensionList`` + * ``asdf.extension.BuiltinExtension`` * ``asdf.extension.default_extensions`` * ``asdf.extension.get_cached_asdf_extensions`` - * `asdf.extension.get_default_resolver` + * ``asdf.extension.get_default_resolver`` * attributes to asdf.AsdfFile including: - * `asdf.AsdfFile.run_hook` - * `asdf.AsdfFile.run_modifying_hook` - * `asdf.AsdfFile.url_mapping` - * `asdf.AsdfFile.tag_mapping` - * `asdf.AsdfFile.type_index` - * `asdf.AsdfFile.resolver` - * `asdf.AsdfFile.extension_list` + * ``asdf.AsdfFile.run_hook`` + * ``asdf.AsdfFile.run_modifying_hook`` + * ``asdf.AsdfFile.url_mapping`` + * ``asdf.AsdfFile.tag_mapping`` + * ``asdf.AsdfFile.type_index`` + * ``asdf.AsdfFile.resolver`` + * ``asdf.AsdfFile.extension_list`` This deprecated api is replaced by new-style :ref:`converters `, :ref:`extensions ` and :ref:`validators `. diff --git a/docs/asdf/developer_api.rst b/docs/asdf/developer_api.rst index 6f913211e..b21afa45d 100644 --- a/docs/asdf/developer_api.rst +++ b/docs/asdf/developer_api.rst @@ -7,13 +7,15 @@ Developer API The classes and functions documented here will be of use to developers who wish to create their own custom ASDF types and extensions. -.. automodapi:: asdf.types - .. automodapi:: asdf.tagged .. automodapi:: asdf.exceptions :skip: ValidationError +.. + .. automodule:: asdf.extension + :members: Extension, ExtensionProxy, ManifestExtension, ExtensionManager, get_cached_extension_manager, TagDefinition, Converter, ConverterProxy, Compressor, Validator + .. automodapi:: asdf.extension .. automodapi:: asdf.resource @@ -31,6 +33,7 @@ to create their own custom ASDF types and extensions. .. automodapi:: asdf.tags.core :skip: ExternalArrayReference :skip: IntegerType + :no-inheritance-diagram: .. automodapi:: asdf.testing.helpers diff --git a/docs/asdf/developer_versioning.rst b/docs/asdf/developer_versioning.rst index 2625ce5b7..a3bbb581a 100644 --- a/docs/asdf/developer_versioning.rst +++ b/docs/asdf/developer_versioning.rst @@ -14,6 +14,35 @@ The ASDF Standard document provides a helpful :ref:`overview `_. Tags and schemas for types +that have not been serialized before should begin at ``1.0.0``. Versions for a +particular tag type need not move in lock-step with other tag types in the same +extension. + +The patch version should be bumped for bug fixes and other minor, +backwards-compatible changes. New features can be indicated with increments to +the minor version, as long as they remain backwards compatible with older +versions of the schema. Any changes that break backwards compatibility must be +indicated by a major version update. + +Since ASDF is intended to be an archival file format, authors of tags and +schemas should work to ensure that ASDF files created with older extensions can +continue to be processed. This means that every time a schema version is bumped +(with the possible exception of patch updates), a **new** schema file should be +created. + +For example, if we currently have a schema for ``xyz-1.0.0``, and we wish to +make changes and bump the version to ``xyz-1.1.0``, we should leave the +original schema intact. A **new** schema file should be created for +``xyz-1.1.0``, which can exist in parallel with the old file. The version of +the corresponding tag type should be bumped to ``1.1.0``. + +For more details on the behavior of schema and tag versioning from a user +perspective, see :ref:`version_and_compat`, and also +:ref:`custom_type_versions`. + + Overview -------- diff --git a/docs/asdf/extending/legacy.rst b/docs/asdf/extending/legacy.rst deleted file mode 100644 index a0c8e5b07..000000000 --- a/docs/asdf/extending/legacy.rst +++ /dev/null @@ -1,926 +0,0 @@ -.. currentmodule:: asdf.extensions - -.. _extending_legacy: - -Deprecated extension API -======================== - -.. note:: - - This page documents the original `asdf` extension API, which has been - deprecated in favor of :ref:`extending_extensions`. Since support - for the deprecated API will be removed in `asdf` 3.0, we recommend that - all new extensions be implemented with the new API. For more information - about this and other deprecations please see the :ref:`deprecations` page. - -Extensions provide a way for ASDF to represent complex types that are not -defined by the ASDF standard. Examples of types that require custom extensions -include types from third-party libraries, user-defined types, and complex types -that are part of the Python standard library but are not handled in the ASDF -standard. From ASDF's perspective, these are all considered 'custom' types. - -Supporting new types in ASDF is easy. Three components are required: - -1. A YAML Schema file for each new type. - -2. A tag class (inheriting from `asdf.CustomType`) corresponding to each new - custom type. The class must override `~asdf.CustomType.to_tree` and - `~asdf.CustomType.from_tree` from `asdf.CustomType` in order to define how - ASDF serializes and deserializes the custom type. - -3. A Python class to define an "extension" to ASDF, which is a set of related - types. This class must implement the `asdf.extension.AsdfExtension` abstract base - class. In general, a third-party library that defines multiple custom types - can group them all in the same extension. - -.. note:: - - The mechanisms of tag classes and extension classes are specific to this - particular implementation of ASDF. As of this writing, this is the only - complete implementation of the ASDF Standard. However, other language - implementations may use other mechanisms for processing custom types. - - All implementations of ASDF, regardless of language, will make use of the - same schemas for abstract data type definitions. This allows all ASDF files - to be language-agnostic, and also enables interoperability. - -An Example ----------- - -As an example, we will write an extension for ASDF that allows us to represent -Python's standard `fractions.Fraction` class for representing rational numbers. -We will call our new ASDF type ``fraction``. - -First, the YAML Schema, defining the type as a pair of integers: - -.. code-block:: yaml - - %YAML 1.1 - --- - $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" - id: "http://nowhere.org/schemas/custom/fraction-1.0.0" - title: An example custom type for handling fractions - - tag: "tag:nowhere.org:custom/fraction-1.0.0" - type: array - items: - type: integer - minItems: 2 - maxItems: 2 - ... - -Then, the Python implementation of the tag class and extension class. See the -`asdf.CustomType` and `asdf.extension.AsdfExtension` documentation for more information: - -.. runcode:: hidden - - import os - import asdf - # This is a hack in order to get the example below to work properly - __file__ = os.path.join(asdf.__path__[0], 'tests', 'data', 'fraction-1.0.0.yaml') - -.. runcode:: - - import os - - import asdf - from asdf import util - - import fractions - - class FractionType(asdf.CustomType): - name = 'fraction' - organization = 'nowhere.org' - version = (1, 0, 0) - standard = 'custom' - types = [fractions.Fraction] - - @classmethod - def to_tree(cls, node, ctx): - return [node.numerator, node.denominator] - - @classmethod - def from_tree(cls, tree, ctx): - return fractions.Fraction(tree[0], tree[1]) - - class FractionExtension(asdf.AsdfExtension): - @property - def types(self): - return [FractionType] - - @property - def tag_mapping(self): - return [('tag:nowhere.org:custom', - 'http://nowhere.org/schemas/custom{tag_suffix}')] - - @property - def url_mapping(self): - return [('http://nowhere.org/schemas/custom/', - util.filepath_to_url(os.path.dirname(__file__)) - + '/{url_suffix}.yaml')] - -Note that the method `~asdf.CustomType.to_tree` of the tag class -``FractionType`` defines how the library converts `fractions.Fraction` into a -tree that can be stored by ASDF. Conversely, the method -`~asdf.CustomType.from_tree` defines how the library reads a serialized -representation of the object and converts it back into an instance of -`fractions.Fraction`. - -Note that the values of the `~asdf.CustomType.name`, -`~asdf.CustomType.organization`, `~asdf.CustomType.standard`, and -`~asdf.CustomType.version` fields are all reflected in the ``id`` and ``tag`` -definitions in the schema. - -Note also that the base of the ``tag`` value (up to the ``name`` and ``version`` -components) is reflected in `~asdf.extension.AsdfExtension.tag_mapping` property of the -``FractionExtension`` type, which is used to map tags to URLs. The -`~asdf.extension.AsdfExtension.url_mapping` is used to map URLs (of the same form as the -``id`` field in the schema) to the actual location of a schema file. - -Once these classes and the schema have been defined, we can save an ASDF file -using them: - -.. runcode:: - - tree = {'fraction': fractions.Fraction(10, 3)} - - with asdf.AsdfFile(tree, extensions=FractionExtension()) as ff: - ff.write_to("test.asdf") - -.. asdf:: test.asdf ignore_unrecognized_tag - -Defining custom types ---------------------- - -In the example above, we showed how to create an extension that is capable of -serializing `fractions.Fraction`. The custom tag type that we created was -defined as a subclass of `asdf.CustomType`. - -Custom type attributes -********************** - -We overrode the following attributes of `~asdf.CustomType` in order to define -``FractionType`` (each bullet is also a link to the API documentation): - -* `~asdf.CustomType.name` -* `~asdf.CustomType.organization` -* `~asdf.CustomType.version` -* `~asdf.CustomType.standard` -* `~asdf.CustomType.types` - -Each of these attributes is important, and each is described in more detail in -the linked API documentation. - -The choice of `~asdf.CustomType.name` should be descriptive of the custom type -that is being serialized. The choice of `~asdf.CustomType.organization`, and -`~asdf.CustomType.standard` is fairly arbitrary, but also important. Custom -types that are provided by the same package should be grouped into the same -`~asdf.CustomType.standard` and `~asdf.CustomType.organization`. - -These three values, along with the `~asdf.CustomType.version`, are used to -define the YAML tag that will mark the serialized type in ASDF files. In our -example, the tag becomes ``tag:nowhere.org:custom/fraction-1.0.0``. The tag -is important when defining the `asdf.extension.AsdfExtension` subclass. - -Critically, these values must all be reflected in the associated schema. - -Custom type methods -******************* - -In addition to the attributes mentioned above, we also overrode the following -methods of `~asdf.CustomType` (each bullet is also a link to the API -documentation): - -* `~asdf.CustomType.to_tree` -* `~asdf.CustomType.from_tree` - -The `~asdf.CustomType.to_tree` method defines how an instance of a custom data -type is converted into data structures that represent a YAML tree that can be -serialized to a file. - -The `~asdf.CustomType.from_tree` method defines how a YAML tree can be -converted back into an instance of the original custom data type. - -In the example above, we used a `list` to contain the important attributes of -`fractions.Fraction`. However, this choice is fairly arbitrary, as long as it -is consistent between the way that `~asdf.CustomType.to_tree` and -`~asdf.CustomType.from_tree` are defined. For example, we could have also -chosen to use a `dict`: - -.. runcode:: - - import asdf - import fractions - - class FractionType(asdf.CustomType): - name = 'fraction' - organization = 'nowhere.org' - version = (1, 0, 0) - standard = 'custom' - types = [fractions.Fraction] - - @classmethod - def to_tree(cls, node, ctx): - return dict(numerator=node.numerator, - denominator=node.denominator) - - @classmethod - def from_tree(cls, tree, ctx): - return fractions.Fraction(tree['numerator'], - tree['denominator']) - -.. runcode:: hidden - - # Redefine the fraction extension for the sake of the example - FractionExtension.types = [FractionType] - - tree = {'fraction': fractions.Fraction(10, 3)} - - with asdf.AsdfFile(tree, extensions=FractionExtension()) as ff: - ff.write_to("test.asdf") - -In this case, the associated schema would look like the following:: - - %YAML 1.1 - --- - $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" - id: "http://nowhere.org/schemas/custom/fraction-1.0.0" - title: An example custom type for handling fractions - - tag: "tag:nowhere.org:custom/fraction-1.0.0" - type: object - properties: - numerator: - type: integer - denominator: - type: integer - ... - -We can compare the output using this representation to the example above: - -.. asdf:: test.asdf ignore_unrecognized_tag - - -Serializing more complex types -****************************** - -Sometimes the custom types that we wish to represent in ASDF themselves have -attributes which are also custom types. As a somewhat contrived example, -consider a 2D cartesian coordinate that uses ``fraction.Fraction`` to represent -each of the components. We will call this type ``Fractional2DCoordinate``. - -First we need to define a schema to represent this new type:: - - %YAML 1.1 - --- - $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" - id: "http://nowhere.org/schemas/custom/fractional_2d_coord-1.0.0" - title: An example custom type for handling components - - tag: "tag:nowhere.org:custom/fractional_2d_coord-1.0.0" - type: object - properties: - x: - $ref: fraction-1.0.0 - y: - $ref: fraction-1.0.0 - ... - -Note that in the schema, the ``x`` and ``y`` attributes are expressed as -references to our ``fraction-1.0.0`` schema. Since both of these schemas are -defined under the same standard and organization, we can simply use the name -and version of the ``fraction-1.0.0`` schema to refer to it. However, if the -reference type was defined in a different organization and standard, it would -be necessary to use the entire YAML tag in the reference (e.g. -``tag:nowhere.org:custom/fraction-1.0.0``). Relative tag references are also -allowed where appropriate. - -.. runcode:: hidden - - class Fractional2DCoordinate: - x = None - y = None - -We also need to define the custom tag type that corresponds to our new type: - -.. runcode:: - - import asdf - - class Fractional2DCoordinateType(asdf.CustomType): - name = 'fractional_2d_coord' - organization = 'nowhere.org' - version = (1, 0, 0) - standard = 'custom' - types = [Fractional2DCoordinate] - - @classmethod - def to_tree(cls, node, ctx): - tree = dict() - tree['x'] = node.x - tree['y'] = node.y - return tree - - @classmethod - def from_tree(cls, tree, ctx): - coord = Fractional2DCoordinate() - coord.x = tree['x'] - coord.y = tree['y'] - return coord - -In previous versions of this library, it was necessary for our -``Fractional2DCoordinateType`` class to call `~asdf.yamlutil` functions -explicitly to convert the ``x`` and ``y`` components to and from -their tree representations. Now, the library will automatically -convert nested custom types before calling `~asdf.CustomType.from_tree`, -and after receiving the result from `~asdf.CustomType.to_tree`. - -Since ``Fractional2DCoordinateType`` shares the same -`~asdf.CustomType.organization` and `~asdf.CustomType.standard` as -``FractionType``, it can be added to the same extension class: - -.. runcode:: - - class FractionExtension(asdf.AsdfExtension): - @property - def types(self): - return [FractionType, Fractional2DCoordinateType] - - @property - def tag_mapping(self): - return [('tag:nowhere.org:custom', - 'http://nowhere.org/schemas/custom{tag_suffix}')] - - @property - def url_mapping(self): - return [('http://nowhere.org/schemas/custom/', - util.filepath_to_url(os.path.dirname(__file__)) - + '/{url_suffix}.yaml')] - -Now we can use this extension to create an ASDF file: - -.. runcode:: - - coord = Fractional2DCoordinate() - coord.x = fractions.Fraction(22, 7) - coord.y = fractions.Fraction(355, 113) - - tree = {'coordinate': coord} - - with asdf.AsdfFile(tree, extensions=FractionExtension()) as ff: - ff.write_to("coord.asdf") - -.. asdf:: coord.asdf ignore_unrecognized_tag - -Note that in the resulting ASDF file, the ``x`` and ``y`` components of -our new ``fraction_2d_coord`` type are tagged as ``fraction-1.0.0``. - -Serializing reference cycles -**************************** - -Special considerations must be made when deserializing a custom type that -contains a reference to itself among its descendants. Consider a -`fractions.Fraction` subclass that maintains a reference to its multiplicative -inverse: - -.. runcode:: - - class FractionWithInverse(fractions.Fraction): - def __init__(self, *args, **kwargs): - self._inverse = None - - @property - def inverse(self): - return self._inverse - - @inverse.setter - def inverse(self, value): - self._inverse = value - -The inverse of the inverse of a fraction is the fraction itself, -so you might wish to construct your objects in the following way: - -.. runcode:: - - f1 = FractionWithInverse(3, 5) - f2 = FractionWithInverse(5, 3) - f1.inverse = f2 - f2.inverse = f1 - -Which creates an "infinite loop" between the two fractions. An ordinary -`~asdf.CustomType` wouldn't be able to deserialize this, since each object -requires that the other be deserialized first! Let's see what happens -when we define our `~asdf.CustomType.from_tree` method in a naive way: - -.. runcode:: - - class FractionWithInverseType(asdf.CustomType): - name = 'fraction_with_inverse' - organization = 'nowhere.org' - version = (1, 0, 0) - standard = 'custom' - types = [FractionWithInverse] - - @classmethod - def to_tree(cls, node, ctx): - return { - "numerator": node.numerator, - "denominator": node.denominator, - "inverse": node.inverse - } - - @classmethod - def from_tree(cls, tree, ctx): - result = FractionWithInverse( - tree["numerator"], - tree["denominator"] - ) - result.inverse = tree["inverse"] - return result - -After adding our type to the extension class, the tree will serialize correctly: - -.. runcode:: hidden - - FractionExtension.types = [FractionType, Fractional2DCoordinateType, FractionWithInverseType] - -.. runcode:: - - tree = {'fraction': f1} - - with asdf.AsdfFile(tree, extensions=FractionExtension()) as ff: - ff.write_to("with_inverse.asdf") - -But upon deserialization, we notice a problem: - -.. runcode:: - - with asdf.open("with_inverse.asdf", extensions=FractionExtension()) as ff: - reconstituted_f1 = ff["fraction"] - - assert reconstituted_f1.inverse.inverse is asdf.treeutil.PendingValue - -The presence of `~asdf.treeutil._PendingValue` is `asdf`'s way of telling you -that the value corresponding to the key ``inverse`` was not fully deserialized -at the time that you retrieved it. We can handle this situation by making our -`~asdf.CustomType.from_tree` a generator function: - -.. runcode:: - - class FractionWithInverseType(asdf.CustomType): - name = 'fraction_with_inverse' - organization = 'nowhere.org' - version = (1, 0, 0) - standard = 'custom' - types = [FractionWithInverse] - - @classmethod - def to_tree(cls, node, ctx): - return { - "numerator": node.numerator, - "denominator": node.denominator, - "inverse": node.inverse - } - - @classmethod - def from_tree(cls, tree, ctx): - result = FractionWithInverse( - tree["numerator"], - tree["denominator"] - ) - yield result - result.inverse = tree["inverse"] - -The generator version of `~asdf.CustomType.from_tree` yields the partially constructed -``FractionWithInverse`` object before setting its inverse property. This allows -asdf to proceed to constructing the inverse ``FractionWithInverse`` object, -and resume the original `~asdf.CustomType.from_tree` execution only when the inverse -is actually available. - -With this new version of `~asdf.CustomType.from_tree`, we can successfully deserialize -our ASDF file: - -.. runcode:: hidden - - FractionExtension.types = [FractionType, Fractional2DCoordinateType, FractionWithInverseType] - -.. runcode:: - - with asdf.open("with_inverse.asdf", extensions=FractionExtension()) as ff: - reconstituted_f1 = ff["fraction"] - - assert reconstituted_f1.inverse.inverse is reconstituted_f1 - - -Assigning schema and tag versions -********************************* - -Authors of new tags and schemas should strive to use the conventions described -by `semantic versioning `_. Tags and schemas for types -that have not been serialized before should begin at ``1.0.0``. Versions for a -particular tag type need not move in lock-step with other tag types in the same -extension. - -The patch version should be bumped for bug fixes and other minor, -backwards-compatible changes. New features can be indicated with increments to -the minor version, as long as they remain backwards compatible with older -versions of the schema. Any changes that break backwards compatibility must be -indicated by a major version update. - -Since ASDF is intended to be an archival file format, authors of tags and -schemas should work to ensure that ASDF files created with older extensions can -continue to be processed. This means that every time a schema version is bumped -(with the possible exception of patch updates), a **new** schema file should be -created. - -For example, if we currently have a schema for ``xyz-1.0.0``, and we wish to -make changes and bump the version to ``xyz-1.1.0``, we should leave the -original schema intact. A **new** schema file should be created for -``xyz-1.1.0``, which can exist in parallel with the old file. The version of -the corresponding tag type should be bumped to ``1.1.0``. - -For more details on the behavior of schema and tag versioning from a user -perspective, see :ref:`version_and_compat`, and also -:ref:`custom_type_versions`. - -Explicit version support -************************ - -To some extent schemas and tag classes will be closely tied to the custom data -types that they represent. This means that in some cases API changes or other -changes to the representation of the underlying types will force us to modify -our schemas and tag classes. ASDF's schema versioning allows us to handle -changes in schemas over time. - -Let's consider an imaginary custom type called ``Person`` that we want to -serialize in ASDF. The first version of ``Person`` was constructed using a -first and last name: - -.. code-block:: python - - person = Person("James", "Webb") - print(person.first, person.last) - -Our version 1.0.0 YAML schema for ``Person`` might look like the following: - -.. code-block:: yaml - - %YAML 1.1 - --- - $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" - id: "http://nowhere.org/schemas/custom/person-1.0.0" - title: An example custom type for representing a Person - - tag: "tag:nowhere.org:custom/person-1.0.0" - type: array - items: - type: string - minItems: 2 - maxItems: 2 - ... - -And our tag implementation would look something like this: - -.. code-block:: python - - import asdf - from people import Person - - - class PersonType(asdf.CustomType): - name = "person" - organization = "nowhere.org" - version = (1, 0, 0) - standard = "custom" - types = [Person] - - @classmethod - def to_tree(cls, node, ctx): - return [node.first, node.last] - - @classmethod - def from_tree(cls, tree, ctx): - return Person(tree[0], tree[1]) - -However, a newer version of ``Person`` now requires a middle name in the -constructor as well: - -.. code-block:: python - - person = Person("James", "Edwin", "Webb") - print(person.first, person.middle, person.last) - -So we update our YAML schema to version 1.1.0 in order to support newer -versions of Person: - -.. code-block:: yaml - - %YAML 1.1 - --- - $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" - id: "http://nowhere.org/schemas/custom/person-1.1.0" - title: An example custom type for representing a Person - - tag: "tag:nowhere.org:custom/person-1.1.0" - type: array - items: - type: string - minItems: 3 - maxItems: 3 - ... - -We need to update our tag class implementation as well. However, we need to be -careful. We still want to be able to read version 1.0.0 of our schema and be -able to convert it to the newer version of ``Person`` objects. To accomplish -this, we will make use of the `~asdf.CustomType.supported_versions` attribute -for our tag class. This will allow us to declare explicit support for the -schema versions our tag class implements. - -Under the hood, `asdf` creates multiple copies of our ``PersonType`` tag class, -each with a different `~asdf.CustomType.version` attribute corresponding to one -of the supported versions. This means that in our new tag class implementation, -we can condition our `~asdf.CustomType.from_tree` implementation on the value -of ``version`` to determine which schema version should be used when reading: - -.. code-block:: python - - import asdf - from people import Person - - - class PersonType(asdf.CustomType): - name = "person" - organization = "nowhere.org" - version = (1, 1, 0) - supported_versions = [(1, 0, 0), (1, 1, 0)] - standard = "custom" - types = [Person] - - @classmethod - def to_tree(cls, node, ctx): - return [node.first, node.middle, node.last] - - @classmethod - def from_tree(cls, tree, ctx): - # Handle the older version of the person schema - if cls.version == (1, 0, 0): - # Construct a Person object with an empty middle name field - return Person(tree[0], "", tree[1]) - else: - # The newer version of the schema stores the middle name too - return person(tree[0], tree[1], tree[2]) - -Note that the implementation of ``to_tree`` is not conditioned on -``cls.version`` since we do not need to convert new ``Person`` objects back to -the older version of the schema. - -Handling subclasses -******************* - -By default, if a custom type is serialized by an `asdf` tag class, then all -subclasses of that type can also be serialized. However, no attributes that are -specific to the subclass will be stored in the file. When reading the file, an -instance of the base custom type will be returned instead of the subclass that -was written. - -To properly handle subclasses of custom types already recognized by `asdf`, it is -necessary to implement a separate tag class that is specific to the subclass to -be serialized. - -Previous versions of this library implemented an experimental feature that -allowed ADSF to serialize subclass attributes using the same tag class, but -this feature was dropped as it produced files that were not portable. - -Creating custom schemas ------------------------ - -All custom types to be serialized by `asdf` require custom schemas. The best -resource for creating ASDF schemas can be found in the :ref:`ASDF Standard -` documentation. - -In most cases, ASDF schemas will be included as part of a packaged software -distribution. In these cases, it is important for the -`~asdf.extension.AsdfExtension.url_mapping` of the corresponding `~asdf.extension.AsdfExtension` -extension class to map the schema URL to an actual location on disk. However, -it is possible for schemas to be hosted online as well, in which case the URL -mapping can map (perhaps trivially) to an actual network location. See -:ref:`defining_extensions` for more information. - -It is also important for packages that provide custom schemas to test them, -both to make sure that they are valid, and to ensure that any examples they -provide are also valid. See :ref:`testing_custom_schemas` for more information. - -Adding custom validators ------------------------- - -A new type may also add new validation keywords to the schema -language. This can be used to impose type-specific restrictions on the -values in an ASDF file. This feature is used internally so a schema -can specify the required datatype of an array. - -To support custom validation keywords, set the `~asdf.CustomType.validators` -member of a `~asdf.CustomType` subclass to a dictionary where the keys are the -validation keyword name and the values are validation functions. The -validation functions are of the same form as the validation functions in the -underlying `jsonschema` library, and are passed the following arguments: - - - ``validator``: A `jsonschema.Validator` instance. - - - ``value``: The value of the schema keyword. - - - ``instance``: The instance to validate. This will be made up of - basic datatypes as represented in the YAML file (list, dict, - number, strings), and not include any object types. - - - ``schema``: The entire schema that applies to instance. Useful to - get other related schema keywords. - -The validation function should either return ``None`` if the instance -is valid or ``yield`` one or more `jsonschema.ValidationError` objects if -the instance is invalid. - -To continue the example from above, for the ``FractionType`` say we -want to add a validation keyword "``simplified``" that, when ``true``, -asserts that the corresponding fraction is in simplified form: - -.. code-block:: python - - from asdf import ValidationError - - - def validate_simplified(validator, simplified, instance, schema): - if simplified: - reduced = fraction.Fraction(instance[0], instance[1]) - if reduced.numerator != instance[0] or reduced.denominator != instance[1]: - yield ValidationError("Fraction is not in simplified form.") - - - FractionType.validators = {"simplified": validate_simplified} - -.. _defining_extensions: - -Defining custom extension classes ---------------------------------- - -Extension classes are the mechanism that `asdf` uses to register custom tag types -so that they can be used when processing ASDF files. Packages that define their -own custom tag types must also define extensions in order for those types to be -used. - -All extension classes must implement the `asdf.extension.AsdfExtension` abstract base -class. A custom extension will override each of the following properties of -`asdf.extension.AsdfExtension` (the text in each bullet is also a link to the corresponding -documentation): - -* `~asdf.extension.AsdfExtension.types` -* `~asdf.extension.AsdfExtension.tag_mapping` -* `~asdf.extension.AsdfExtension.url_mapping` - -.. _packaging_extensions: - -Overriding built-in extensions -****************************** - -It is possible for externally defined extensions to override tag types that are -provided by `asdf`'s built-in extension. For example, maybe an external package -wants to provide a different implementation of `~asdf.tags.core.NDArrayType`. -In this case, the external package does not need to provide custom schemas -since the schema for the type to be overridden is already provided as part of -the ASDF standard. - -Instead, the extension class may inherit from `asdf`'s -`asdf.extension.BuiltinExtension` and simply override the -`~asdf.extension.AsdfExtension.types` property to indicate the type that is being -overridden. Doing this preserves the `~asdf.extension.AsdfExtension.tag_mapping` and -`~asdf.extension.AsdfExtension.url_mapping` that is used by the ``BuiltinExtension``, which -allows the schemas that are packaged by `asdf` to be located. - -`asdf` will give precedence to the type that is provided by the external -extension, effectively overriding the corresponding type in the built-in -extension. Note that it is currently undefined if multiple external extensions -are provided that override the same built-in type. - -Packaging custom extensions ---------------------------- - -Packaging schemas -***************** - -If a package provides custom schemas, the schema files must be installed as -part of that package distribution. In general, schema files must be installed -into a subdirectory of the package distribution. The `asdf` extension class must -supply a `~asdf.extension.AsdfExtension.url_mapping` that maps to the installed location -of the schemas. See :ref:`defining_extensions` for more details. - -Registering entry points -************************ - -Packages that provide their own ASDF extensions can (and should!) install them -so that they are automatically detectable by the `asdf` Python package. This is -accomplished using Python's `setuptools `_ -entry points. Entry points are registered in a package's ``setup.py`` file. - -Consider a package that provides an extension class ``MyPackageExtension`` in the -submodule ``mypackage.asdf.extensions``. We need to register this class as an -extension entry point that `asdf` will recognize. First, we create a dictionary: - -.. code:: python - - entry_points = {} - entry_points["asdf_extensions"] = [ - "mypackage = mypackage.asdf.extensions:MyPackageExtension" - ] - -The key used in the ``entry_points`` dictionary must be ``'asdf_extensions'``. -The value must be an array of one or more strings, each with the following -format: - - ``extension_name = fully.specified.submodule:ExtensionClass`` - -The extension name can be any arbitrary string, but it should be descriptive of -the package and the extension. In most cases the package itself name will -suffice. - -Note that depending on individual package requirements, there may be other -entries in the ``entry_points`` dictionary. - -The entry points must be passed to the call to ``setuptools.setup``: - -.. code:: python - - from setuptools import setup - - entry_points = {} - entry_points["asdf_extensions"] = [ - "mypackage = mypackage.asdf.extensions:MyPackageExtension" - ] - - setup( - # We omit other package-specific arguments that are not - # relevant to this example - entry_points=entry_points, - ) - -When running ``python setup.py install`` or ``python setup.py develop`` on this -package, the entry points will be registered automatically. This allows the -`asdf` package to recognize the extensions without any user intervention. Users -of your package that wish to read ASDF files using types that you have -registered will not need to use any extension explicitly. Instead, `asdf` will -automatically recognize the types you have registered and will process them -appropriately. See :ref:`other_packages` for more information on using -extensions. - -.. _testing_custom_schemas: - -Testing custom schemas ----------------------- - -Packages that provide their own schemas can test them using `asdf`'s -:ref:`pytest ` plugin for schema testing. -Schemas are tested for overall validity, and any examples given within the -schemas are also tested. - -The schema tester plugin is automatically registered when the `asdf` package is -installed. In order to enable testing, it is necessary to add the directory -containing your schema files to the pytest section of your project's build configuration -(``pyproject.toml`` or ``setup.cfg``). If you do not already have such a file, creating -one with the following should be sufficient: - -.. tab:: pyproject.toml - - .. code-block:: toml - - [tool.pytest.ini_options] - asdf_schema_root = 'path/to/schemas another/path/to/schemas' - -.. tab:: setup.cfg - - .. code-block:: ini - - [tool:pytest] - asdf_schema_root = path/to/schemas another/path/to/schemas - -The schema directory paths should be paths that are relative to the top of the -package directory **when it is installed**. If this is different from the path -in the source directory, then both paths can be used to facilitate in-place -testing (see `asdf`'s own ``pyproject.toml`` for an example of this). - -.. note:: - - Older versions of `asdf` (prior to 2.4.0) required the plugin to be registered - in your project's ``conftest.py`` file. As of 2.4.0, the plugin is now - registered automatically and so this line should be removed from your - ``conftest.py`` file, unless you need to retain compatibility with older - versions of `asdf`. - -The ``asdf_schema_skip_names`` configuration variable can be used to skip -schema files that live within one of the ``asdf_schema_root`` directories but -should not be tested. The names should be given as simple base file names -(without directory paths or extensions). Again, see `asdf`'s own ``pyproject.toml`` file -for an example. - -The schema tests do **not** run by default. In order to enable the tests by -default for your package, add ``asdf_schema_tests_enabled = 'true'`` to the -``[tool.pytest.ini_options]`` section of your ``pyproject.toml`` file (or ``[tool:pytest]`` in ``setup.cfg``). -If you do not wish to enable the schema tests by default, you can add the ``--asdf-tests`` option to -the ``pytest`` command line to enable tests on a per-run basis. diff --git a/docs/asdf/extending/schemas.rst b/docs/asdf/extending/schemas.rst index 2cb5898ac..2e9dbd98b 100644 --- a/docs/asdf/extending/schemas.rst +++ b/docs/asdf/extending/schemas.rst @@ -241,6 +241,61 @@ function will validate a Python object against a schema: The validate function will return successfully if the object is valid, or raise an error if not. +.. _testing_custom_schemas: + +Testing custom schemas +---------------------- + +Packages that provide their own schemas can test them using `asdf`'s +:ref:`pytest ` plugin for schema testing. +Schemas are tested for overall validity, and any examples given within the +schemas are also tested. + +The schema tester plugin is automatically registered when the `asdf` package is +installed. In order to enable testing, it is necessary to add the directory +containing your schema files to the pytest section of your project's build configuration +(``pyproject.toml`` or ``setup.cfg``). If you do not already have such a file, creating +one with the following should be sufficient: + +.. tab:: pyproject.toml + + .. code-block:: toml + + [tool.pytest.ini_options] + asdf_schema_root = 'path/to/schemas another/path/to/schemas' + +.. tab:: setup.cfg + + .. code-block:: ini + + [tool:pytest] + asdf_schema_root = path/to/schemas another/path/to/schemas + +The schema directory paths should be paths that are relative to the top of the +package directory **when it is installed**. If this is different from the path +in the source directory, then both paths can be used to facilitate in-place +testing (see `asdf`'s own ``pyproject.toml`` for an example of this). + +.. note:: + + Older versions of `asdf` (prior to 2.4.0) required the plugin to be registered + in your project's ``conftest.py`` file. As of 2.4.0, the plugin is now + registered automatically and so this line should be removed from your + ``conftest.py`` file, unless you need to retain compatibility with older + versions of `asdf`. + +The ``asdf_schema_skip_names`` configuration variable can be used to skip +schema files that live within one of the ``asdf_schema_root`` directories but +should not be tested. The names should be given as simple base file names +(without directory paths or extensions). Again, see `asdf`'s own ``pyproject.toml`` file +for an example. + +The schema tests do **not** run by default. In order to enable the tests by +default for your package, add ``asdf_schema_tests_enabled = 'true'`` to the +``[tool.pytest.ini_options]`` section of your ``pyproject.toml`` file (or ``[tool:pytest]`` in ``setup.cfg``). +If you do not wish to enable the schema tests by default, you can add the ``--asdf-tests`` option to +the ``pytest`` command line to enable tests on a per-run basis. + See also: ========= diff --git a/docs/asdf/user_api.rst b/docs/asdf/user_api.rst index 252e0a2c4..2b2647eb3 100644 --- a/docs/asdf/user_api.rst +++ b/docs/asdf/user_api.rst @@ -7,8 +7,8 @@ User API .. automodapi:: asdf :include-all-objects: :inherited-members: + :no-inheritance-diagram: :skip: ValidationError - :skip: AsdfExtension .. automodapi:: asdf.search diff --git a/docs/index.rst b/docs/index.rst index 134282de6..6ebcb0d92 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -59,7 +59,6 @@ Extending ASDF asdf/extending/manifests asdf/extending/compressors asdf/extending/validators - asdf/extending/legacy API Documentation ================= From 6c059c3e91e1d80f6311c874a965d564a9d0b9af Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 7 Mar 2023 11:44:22 -0500 Subject: [PATCH 07/13] make AsdfType private --- asdf/_types.py | 6 +++--- asdf/reference.py | 2 +- asdf/tags/core/integer.py | 2 +- asdf/tags/core/ndarray.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/asdf/_types.py b/asdf/_types.py index 9a78b2d0f..a4725ed02 100644 --- a/asdf/_types.py +++ b/asdf/_types.py @@ -9,7 +9,7 @@ from .exceptions import AsdfDeprecationWarning from .versioning import AsdfSpec, AsdfVersion -__all__ = ["format_tag", "CustomType", "AsdfType", "ExtensionType"] # noqa: F822 +__all__ = ["format_tag", "CustomType", "_AsdfType", "ExtensionType"] # noqa: F822 # regex used to parse module name from optional version string @@ -161,7 +161,7 @@ def __new__(cls, name, bases, attrs): new_cls = super().__new__(cls, name, bases, attrs) # Classes using this metaclass get added to the list of built-in # extensions - if name != "AsdfType": + if name != "_AsdfType": _all_asdftypes.add(new_cls) return new_cls @@ -388,7 +388,7 @@ def incompatible_version(cls, version): return False -class AsdfType(ExtensionType, metaclass=AsdfTypeMeta): +class _AsdfType(ExtensionType, metaclass=AsdfTypeMeta): """ Base class for all built-in ASDF types. Types that inherit this class will be automatically added to the list of built-ins. This should *not* be used diff --git a/asdf/reference.py b/asdf/reference.py index a24a243fb..b7bbda4f4 100644 --- a/asdf/reference.py +++ b/asdf/reference.py @@ -42,7 +42,7 @@ def resolve_fragment(tree, pointer): return tree -class Reference(_types.AsdfType): +class Reference(_types._AsdfType): yaml_tag = "tag:yaml.org,2002:map" def __init__(self, uri, base_uri=None, asdffile=None, target=None): diff --git a/asdf/tags/core/integer.py b/asdf/tags/core/integer.py index 022ff3534..898ac3ad6 100644 --- a/asdf/tags/core/integer.py +++ b/asdf/tags/core/integer.py @@ -5,7 +5,7 @@ from asdf import _types -class IntegerType(_types.AsdfType): +class IntegerType(_types._AsdfType): """ Enables the storage of arbitrarily large integer values diff --git a/asdf/tags/core/ndarray.py b/asdf/tags/core/ndarray.py index 15a263df5..b52767840 100644 --- a/asdf/tags/core/ndarray.py +++ b/asdf/tags/core/ndarray.py @@ -226,7 +226,7 @@ def ascii_to_unicode(x): return ascii_to_unicode(tolist(array)) -class NDArrayType(_types.AsdfType): +class NDArrayType(_types._AsdfType): name = "core/ndarray" version = "1.0.0" supported_versions = {"1.0.0", "1.1.0"} @@ -390,7 +390,7 @@ def __getattribute__(self, name): msg = f"'{self.__class__.name}' object has no attribute '{name}'" raise AttributeError(msg) - return _types.AsdfType.__getattribute__(self, name) + return _types._AsdfType.__getattribute__(self, name) @classmethod def from_tree(cls, node, ctx): From 803fd95e48f375191817acc41b436314b2963f91 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 7 Mar 2023 11:49:28 -0500 Subject: [PATCH 08/13] make AsdfExtension private --- asdf/_tests/_helpers.py | 2 +- asdf/_tests/test_extension.py | 4 ++-- asdf/asdf.py | 2 +- asdf/config.py | 4 ++-- asdf/extension/_extension.py | 10 +++++----- asdf/extension/_legacy.py | 8 ++++---- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/asdf/_tests/_helpers.py b/asdf/_tests/_helpers.py index 033cd4433..cdf25a997 100644 --- a/asdf/_tests/_helpers.py +++ b/asdf/_tests/_helpers.py @@ -412,7 +412,7 @@ def assert_extension_correctness(extension): Parameters ---------- - extension : asdf.AsdfExtension + extension : asdf._AsdfExtension The extension to validate """ __tracebackhide__ = True diff --git a/asdf/_tests/test_extension.py b/asdf/_tests/test_extension.py index 00e0c04c7..8cb4082ea 100644 --- a/asdf/_tests/test_extension.py +++ b/asdf/_tests/test_extension.py @@ -17,7 +17,7 @@ Validator, get_cached_extension_manager, ) -from asdf.extension._legacy import AsdfExtension, BuiltinExtension, get_cached_asdf_extension_list +from asdf.extension._legacy import BuiltinExtension, _AsdfExtension, get_cached_asdf_extension_list def test_builtin_extension(): @@ -172,7 +172,7 @@ def test_extension_proxy(): proxy = ExtensionProxy(extension) assert isinstance(proxy, Extension) - assert isinstance(proxy, AsdfExtension) + assert isinstance(proxy, _AsdfExtension) assert proxy.extension_uri == "asdf://somewhere.org/extensions/minimum-1.0" assert proxy.legacy_class_names == set() diff --git a/asdf/asdf.py b/asdf/asdf.py index 7cec72d14..719cd9904 100644 --- a/asdf/asdf.py +++ b/asdf/asdf.py @@ -358,7 +358,7 @@ def _process_user_extensions(self, extensions): """ if extensions is None: extensions = [] - elif isinstance(extensions, (_legacy.AsdfExtension, Extension, ExtensionProxy)): + elif isinstance(extensions, (_legacy._AsdfExtension, Extension, ExtensionProxy)): extensions = [extensions] elif isinstance(extensions, _legacy.AsdfExtensionList): extensions = extensions.extensions diff --git a/asdf/config.py b/asdf/config.py index 5b9e8d994..cb78d0642 100644 --- a/asdf/config.py +++ b/asdf/config.py @@ -163,7 +163,7 @@ def add_extension(self, extension): Parameters ---------- - extension : asdf.extension.AsdfExtension or asdf.extension.Extension + extension : asdf.extension.Extension """ with self._lock: extension = ExtensionProxy.maybe_wrap(extension) @@ -175,7 +175,7 @@ def remove_extension(self, extension=None, *, package=None): Parameters ---------- - extension : asdf.extension.AsdfExtension or asdf.extension.Extension or str, optional + extension : asdf.extension.Extension or str, optional An extension instance or URI pattern to remove. package : str, optional Remove only extensions provided by this package. If the ``extension`` diff --git a/asdf/extension/_extension.py b/asdf/extension/_extension.py index c5693b407..11a288ed0 100644 --- a/asdf/extension/_extension.py +++ b/asdf/extension/_extension.py @@ -6,7 +6,7 @@ from ._compressor import Compressor from ._converter import ConverterProxy -from ._legacy import AsdfExtension +from ._legacy import _AsdfExtension from ._tag import TagDefinition from ._validator import Validator @@ -131,7 +131,7 @@ def validators(self): return [] -class ExtensionProxy(Extension, AsdfExtension): +class ExtensionProxy(Extension, _AsdfExtension): """ Proxy that wraps an extension, provides default implementations of optional methods, and carries additional information on the @@ -146,7 +146,7 @@ def maybe_wrap(cls, delegate): return ExtensionProxy(delegate) def __init__(self, delegate, package_name=None, package_version=None): - if not isinstance(delegate, (Extension, AsdfExtension)): + if not isinstance(delegate, (Extension, _AsdfExtension)): msg = "Extension must implement the Extension or AsdfExtension interface" raise TypeError(msg) @@ -156,7 +156,7 @@ def __init__(self, delegate, package_name=None, package_version=None): self._class_name = get_class_name(delegate) - self._legacy = isinstance(delegate, AsdfExtension) + self._legacy = isinstance(delegate, _AsdfExtension) # Sort these out up-front so that errors are raised when the extension is loaded # and not in the middle of the user's session. The extension will fail to load @@ -367,7 +367,7 @@ def class_name(self): @property def legacy(self): """ - Get the extension's legacy flag. Subclasses of ``asdf.extension.AsdfExtension`` + Get the extension's legacy flag. Subclasses of ``asdf.extension._AsdfExtension`` are marked `True`. Returns diff --git a/asdf/extension/_legacy.py b/asdf/extension/_legacy.py index cc1db0df9..b97b20e24 100644 --- a/asdf/extension/_legacy.py +++ b/asdf/extension/_legacy.py @@ -7,10 +7,10 @@ from asdf._type_index import AsdfTypeIndex from asdf.exceptions import AsdfDeprecationWarning -__all__ = ["AsdfExtension"] +__all__ = ["_AsdfExtension"] -class AsdfExtension(metaclass=abc.ABCMeta): +class _AsdfExtension(metaclass=abc.ABCMeta): """ Abstract base class defining a (legacy) extension to ASDF. New code should use `asdf.extension.Extension` instead. @@ -18,7 +18,7 @@ class AsdfExtension(metaclass=abc.ABCMeta): @classmethod def __subclasshook__(cls, class_): - if cls is AsdfExtension: + if cls is _AsdfExtension: return hasattr(class_, "types") and hasattr(class_, "tag_mapping") return NotImplemented @@ -173,7 +173,7 @@ def get_cached_asdf_extension_list(extensions): Parameters ---------- - extensions : list of asdf.extension.AsdfExtension + extensions : list of asdf.extension._AsdfExtension Returns ------- From 4b4de3baf13e9f6e2946bd362954b60b6421b39b Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 7 Mar 2023 12:02:00 -0500 Subject: [PATCH 09/13] remove commented lines in docs --- docs/asdf/developer_api.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/asdf/developer_api.rst b/docs/asdf/developer_api.rst index b21afa45d..270a91915 100644 --- a/docs/asdf/developer_api.rst +++ b/docs/asdf/developer_api.rst @@ -12,10 +12,6 @@ to create their own custom ASDF types and extensions. .. automodapi:: asdf.exceptions :skip: ValidationError -.. - .. automodule:: asdf.extension - :members: Extension, ExtensionProxy, ManifestExtension, ExtensionManager, get_cached_extension_manager, TagDefinition, Converter, ConverterProxy, Compressor, Validator - .. automodapi:: asdf.extension .. automodapi:: asdf.resource From 37919ec4e0d37d2ad2147aa81ab0f4a822320fd4 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 7 Mar 2023 13:24:18 -0500 Subject: [PATCH 10/13] remove more AsdfExtension references --- asdf/_tests/test_asdf.py | 12 +++--------- asdf/_tests/test_entry_points.py | 2 +- asdf/_tests/test_extension.py | 4 ++-- asdf/extension/_extension.py | 2 +- asdf/extension/_manager.py | 2 +- 5 files changed, 8 insertions(+), 14 deletions(-) diff --git a/asdf/_tests/test_asdf.py b/asdf/_tests/test_asdf.py index e45b621ca..5ab31d0ee 100644 --- a/asdf/_tests/test_asdf.py +++ b/asdf/_tests/test_asdf.py @@ -125,10 +125,7 @@ def test_asdf_file_extensions(): af.extensions = arg assert af.extensions == [ExtensionProxy(extension)] - msg = ( - r"[The extensions parameter must be an extension.*, " - r"Extension must implement the Extension or AsdfExtension interface]" - ) + msg = r"[The extensions parameter must be an extension.*, Extension must implement the Extension interface]" for arg in (object(), [object()]): with pytest.raises(TypeError, match=msg): AsdfFile(extensions=arg) @@ -185,10 +182,7 @@ def test_open_asdf_extensions(tmp_path): with open_asdf(path, extensions=arg) as af: assert af.extensions == [ExtensionProxy(extension)] - msg = ( - r"[The extensions parameter must be an extension.*, " - r"Extension must implement the Extension or AsdfExtension interface]" - ) + msg = r"[The extensions parameter must be an extension.*, Extension must implement the Extension interface]" for arg in (object(), [object()]): with pytest.raises(TypeError, match=msg), open_asdf(path, extensions=arg) as af: pass @@ -211,7 +205,7 @@ def test_serialization_context(): assert context.url == context._url == "file://test.asdf" - with pytest.raises(TypeError, match=r"Extension must implement the Extension or AsdfExtension interface"): + with pytest.raises(TypeError, match=r"Extension must implement the Extension interface"): context._mark_extension_used(object()) with pytest.raises(ValueError, match=r"ASDF Standard version .* is not supported by asdf==.*"): diff --git a/asdf/_tests/test_entry_points.py b/asdf/_tests/test_entry_points.py index baaed37ba..ec45ca05d 100644 --- a/asdf/_tests/test_entry_points.py +++ b/asdf/_tests/test_entry_points.py @@ -151,7 +151,7 @@ def test_get_extensions(mock_entry_points): ) with pytest.warns( AsdfWarning, - match=r"TypeError: Extension must implement the Extension or AsdfExtension interface", + match=r"TypeError: Extension must implement the Extension interface", ): extensions = entry_points.get_extensions() assert len(extensions) == 2 diff --git a/asdf/_tests/test_extension.py b/asdf/_tests/test_extension.py index 8cb4082ea..6a6293957 100644 --- a/asdf/_tests/test_extension.py +++ b/asdf/_tests/test_extension.py @@ -162,7 +162,7 @@ def test_extension_proxy_maybe_wrap(): assert proxy.delegate is extension assert ExtensionProxy.maybe_wrap(proxy) is proxy - with pytest.raises(TypeError, match=r"Extension must implement the Extension or AsdfExtension interface"): + with pytest.raises(TypeError, match=r"Extension must implement the Extension interface"): ExtensionProxy.maybe_wrap(object()) @@ -241,7 +241,7 @@ def test_extension_proxy(): assert proxy.class_name == "asdf._tests.test_extension.FullExtension" # Should fail when the input is not one of the two extension interfaces: - with pytest.raises(TypeError, match=r"Extension must implement the Extension or AsdfExtension interface"): + with pytest.raises(TypeError, match=r"Extension must implement the Extension interface"): ExtensionProxy(object) # Should fail with a bad converter: diff --git a/asdf/extension/_extension.py b/asdf/extension/_extension.py index 11a288ed0..d19ce81ac 100644 --- a/asdf/extension/_extension.py +++ b/asdf/extension/_extension.py @@ -147,7 +147,7 @@ def maybe_wrap(cls, delegate): def __init__(self, delegate, package_name=None, package_version=None): if not isinstance(delegate, (Extension, _AsdfExtension)): - msg = "Extension must implement the Extension or AsdfExtension interface" + msg = "Extension must implement the Extension interface" raise TypeError(msg) self._delegate = delegate diff --git a/asdf/extension/_manager.py b/asdf/extension/_manager.py index 3509a201d..155f84914 100644 --- a/asdf/extension/_manager.py +++ b/asdf/extension/_manager.py @@ -203,7 +203,7 @@ def get_cached_extension_manager(extensions): Parameters ---------- - extensions : list of asdf.extension.AsdfExtension or asdf.extension.Extension + extensions : list of asdf.extension.Extension Returns ------- From 296599c43fd1a0726dca36c00deb34dcea80e0a2 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 7 Mar 2023 13:32:53 -0500 Subject: [PATCH 11/13] remove resolver ref --- asdf/schema.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/asdf/schema.py b/asdf/schema.py index f1d960164..ff056c1f8 100644 --- a/asdf/schema.py +++ b/asdf/schema.py @@ -534,8 +534,9 @@ def get_validator( A dictionary mapping properties to validators to use (instead of the built-in ones and ones provided by extension types). - url_mapping : resolver.Resolver, optional - A resolver to convert remote URLs into local ones. + url_mapping : callable, optional + A callable that takes one string argument and returns a string + to convert remote URLs into local ones. _visit_repeat_nodes : bool, optional Force the validator to visit nodes that it has already From a5da3bf1b3aa8fc39273078055ef2d034ce2379c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 24 Apr 2023 13:11:56 +0000 Subject: [PATCH 12/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- asdf/_tests/test_versioning.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/asdf/_tests/test_versioning.py b/asdf/_tests/test_versioning.py index 4cf882ca6..975544613 100644 --- a/asdf/_tests/test_versioning.py +++ b/asdf/_tests/test_versioning.py @@ -1,10 +1,5 @@ from itertools import combinations -import pytest - -import asdf -from asdf.extension._legacy import default_extensions -from asdf.schema import load_schema from asdf.versioning import ( AsdfSpec, AsdfVersion, From 19dc360d2632e436edf012ca28f21db2ee7ee0c5 Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 8 May 2023 09:17:21 -0400 Subject: [PATCH 13/13] remove _AsdfType from __all__ --- asdf/_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asdf/_types.py b/asdf/_types.py index a4725ed02..45e2c567a 100644 --- a/asdf/_types.py +++ b/asdf/_types.py @@ -9,7 +9,7 @@ from .exceptions import AsdfDeprecationWarning from .versioning import AsdfSpec, AsdfVersion -__all__ = ["format_tag", "CustomType", "_AsdfType", "ExtensionType"] # noqa: F822 +__all__ = ["format_tag", "CustomType", "ExtensionType"] # noqa: F822 # regex used to parse module name from optional version string