From 0999bdb6f86fa86bdc7de835104faf030268b572 Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Thu, 26 Aug 2021 21:13:54 +0200 Subject: [PATCH 1/3] Replaced TypeSpecifier by simpler DataType class --- test/test_era5.py | 2 +- test/test_store.py | 2 +- xcube_cds/store.py | 54 +++++++++++++++++++++++--------------------- xcube_cds/version.py | 2 +- 4 files changed, 31 insertions(+), 29 deletions(-) diff --git a/test/test_era5.py b/test/test_era5.py index 6e44677..6ded9c9 100644 --- a/test/test_era5.py +++ b/test/test_era5.py @@ -218,7 +218,7 @@ def test_get_type_specifiers_for_data(self): store = CDSDataStore() self.assertEqual( (TYPE_SPECIFIER_CUBE, ), - store.get_type_specifiers_for_data('reanalysis-era5-land') + store.get_data_types_for_data('reanalysis-era5-land') ) def test_has_data_true(self): diff --git a/test/test_store.py b/test/test_store.py index 88e77f1..fe52be1 100644 --- a/test/test_store.py +++ b/test/test_store.py @@ -134,7 +134,7 @@ def test_get_data_store_params_schema(self): }, CDSDataStore.get_data_store_params_schema().to_dict()) def test_get_type_specifiers(self): - type_specifiers = CDSDataStore.get_type_specifiers() + type_specifiers = CDSDataStore.get_data_types() self.assertEqual(1, len(type_specifiers)) self.assertIsInstance(type_specifiers[0], str) self.assertTupleEqual(('dataset[cube]',), type_specifiers) diff --git a/xcube_cds/store.py b/xcube_cds/store.py index fb4fc72..f96a37a 100644 --- a/xcube_cds/store.py +++ b/xcube_cds/store.py @@ -46,14 +46,15 @@ import xarray as xr import xcube.core.normalize +from xcube.core.store import DATASET_TYPE from xcube.core.store import DataDescriptor from xcube.core.store import DataOpener from xcube.core.store import DataStore from xcube.core.store import DataStoreError +from xcube.core.store import DataType +from xcube.core.store import DataTypeLike from xcube.core.store import DatasetDescriptor from xcube.core.store import DefaultSearchMixin -from xcube.core.store import TYPE_SPECIFIER_CUBE -from xcube.core.store import TypeSpecifier from xcube.util.jsonschema import JsonArraySchema from xcube.util.jsonschema import JsonBooleanSchema from xcube.util.jsonschema import JsonDateSchema @@ -753,18 +754,19 @@ def get_data_store_params_schema(cls) -> JsonObjectSchema: ) @classmethod - def get_type_specifiers(cls) -> Tuple[str, ...]: - return str(TYPE_SPECIFIER_CUBE), + def get_data_types(cls) -> Tuple[str, ...]: + return DATASET_TYPE.alias, - def get_type_specifiers_for_data(self, data_id: str) -> Tuple[str, ...]: + def get_data_types_for_data(self, data_id: str) -> Tuple[str, ...]: self._validate_data_id(data_id) - return str(TYPE_SPECIFIER_CUBE), + return DATASET_TYPE.alias, - def get_data_ids(self, type_specifier: Optional[str] = None, + def get_data_ids(self, + data_type: DataTypeLike = None, include_attrs: Container[str] = None) -> \ Union[Iterator[str], Iterator[Tuple[str, Dict[str, Any]]]]: - if self._is_type_specifier_satisfied(type_specifier): + if self._is_data_type_satisfied(data_type): # Only if the type specifier isn't compatible return_tuples = include_attrs is not None # TODO: respect names other than "title" in include_attrs @@ -773,7 +775,7 @@ def get_data_ids(self, type_specifier: Optional[str] = None, for data_id, handler in self._handler_registry.items(): if return_tuples: if include_titles: - yield data_id,\ + yield data_id, \ {'title': handler.get_human_readable_data_id(data_id)} else: @@ -781,30 +783,30 @@ def get_data_ids(self, type_specifier: Optional[str] = None, else: yield data_id - def has_data(self, data_id: str, type_specifier: Optional[str] = None) \ + def has_data(self, data_id: str, data_type: Optional[str] = None) \ -> bool: - return self._is_type_specifier_satisfied(type_specifier) and \ + return self._is_data_type_satisfied(data_type) and \ data_id in self._handler_registry def describe_data(self, data_id: str, - type_specifier: Optional[str] = None) \ + data_type: Optional[str] = None) \ -> DatasetDescriptor: self._validate_data_id(data_id) - self._validate_type_specifier(type_specifier) + self._validate_data_type(data_type) return self._handler_registry[data_id].describe_data(data_id) # noinspection PyTypeChecker - def search_data(self, type_specifier: Optional[str] = None, + def search_data(self, data_type: Optional[str] = None, **search_params) \ -> Iterator[DataDescriptor]: - self._validate_type_specifier(type_specifier) - return super().search_data(type_specifier=type_specifier, + self._validate_data_type(data_type) + return super().search_data(data_type=data_type, **search_params) def get_data_opener_ids(self, data_id: Optional[str] = None, - type_specifier: Optional[str] = None) \ + data_type: Optional[str] = None) \ -> Tuple[str, ...]: - self._validate_type_specifier(type_specifier) + self._validate_data_type(data_type) self._validate_data_id(data_id, allow_none=True) return CDS_DATA_OPENER_ID, @@ -827,23 +829,23 @@ def open_data(self, data_id: str, opener_id: Optional[str] = None, # Implementation helpers @staticmethod - def _validate_type_specifier(type_specifier: Union[str, TypeSpecifier]): - if not CDSDataStore._is_type_specifier_satisfied(type_specifier): + def _validate_data_type(data_type: DataTypeLike): + if not CDSDataStore._is_data_type_satisfied(data_type): raise DataStoreError( - f'Supplied type specifier "{type_specifier}" is not compatible ' - f'with "{TYPE_SPECIFIER_CUBE}."' + f'Supplied type specifier {data_type!r} is not compatible ' + f'with "{DATASET_TYPE!r}."' ) @staticmethod - def _is_type_specifier_satisfied( - type_specifier: Union[str, TypeSpecifier]) -> bool: + def _is_data_type_satisfied( + data_type: DataTypeLike) -> bool: # At present, all datasets are available as cubes, so we simply check # against TYPE_SPECIFIER_CUBE. If more (non-cube) datasets are added, # the logic will have to be delegated to CDSDatasetHandler # implementations. - if type_specifier is None: + if data_type is None: return True - return TYPE_SPECIFIER_CUBE.satisfies(type_specifier) + return DATASET_TYPE.is_sub_type_of(data_type) @staticmethod def _assert_valid_opener_id(opener_id): diff --git a/xcube_cds/version.py b/xcube_cds/version.py index ff20cbf..d1d8a29 100644 --- a/xcube_cds/version.py +++ b/xcube_cds/version.py @@ -20,4 +20,4 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -version = '0.8.1' +version = '0.9.0.dev0' From 0e7ea45ef394859a792be06c80e4c8bcc311238e Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Fri, 27 Aug 2021 11:03:27 +0200 Subject: [PATCH 2/3] Made test run green --- test/test_era5.py | 14 +++++++------- test/test_store.py | 44 +++++++++++++++++++++----------------------- xcube_cds/store.py | 7 +++---- 3 files changed, 31 insertions(+), 34 deletions(-) diff --git a/test/test_era5.py b/test/test_era5.py index 6ded9c9..3857923 100644 --- a/test/test_era5.py +++ b/test/test_era5.py @@ -27,13 +27,13 @@ import unittest -import xcube -import xcube.core from jsonschema import ValidationError -from xcube.core.store import TYPE_SPECIFIER_CUBE -from xcube.core.store import VariableDescriptor +import xcube +import xcube.core from test.mocks import CDSClientMock +from xcube.core.store import DATASET_TYPE +from xcube.core.store import VariableDescriptor from xcube_cds.store import CDSDataOpener from xcube_cds.store import CDSDataStore @@ -179,8 +179,8 @@ def test_open_data_null_variables_list(self): store = CDSDataStore(client_class=CDSClientMock, endpoint_url=_CDS_API_URL, cds_api_key=_CDS_API_KEY) - data_id = 'reanalysis-era5-single-levels-monthly-means:'\ - 'monthly_averaged_reanalysis' + data_id = 'reanalysis-era5-single-levels-monthly-means:' \ + 'monthly_averaged_reanalysis' schema = store.get_open_data_params_schema(data_id) n_vars = len(schema.properties['variable_names'].items.enum) dataset = store.open_data( @@ -217,7 +217,7 @@ def test_era5_describe_data(self): def test_get_type_specifiers_for_data(self): store = CDSDataStore() self.assertEqual( - (TYPE_SPECIFIER_CUBE, ), + (DATASET_TYPE.alias,), store.get_data_types_for_data('reanalysis-era5-land') ) diff --git a/test/test_store.py b/test/test_store.py index fe52be1..4e11583 100644 --- a/test/test_store.py +++ b/test/test_store.py @@ -49,12 +49,10 @@ import xcube import xcube.core +from test.mocks import CDSClientMock +from xcube.core.store import DATASET_TYPE from xcube.core.store import DataDescriptor from xcube.core.store import DataStoreError -from xcube.core.store import TYPE_SPECIFIER_CUBE -from xcube.core.store import TYPE_SPECIFIER_DATASET - -from test.mocks import CDSClientMock from xcube_cds.constants import CDS_DATA_OPENER_ID from xcube_cds.datasets.reanalysis_era5 import ERA5DatasetHandler from xcube_cds.store import CDSDataOpener @@ -114,8 +112,8 @@ def test_get_open_params_schema_without_data_id(self): def test_search_data_invalid_id(self): store = CDSDataStore(endpoint_url=_CDS_API_URL, cds_api_key=_CDS_API_KEY) - with self.assertRaises(DataStoreError): - store.search_data('This is an invalid ID.') + with self.assertRaises(ValueError): + store.search_data(data_type='This is an invalid ID.') def test_search_data_valid_id(self): store = CDSDataStore(endpoint_url=_CDS_API_URL, @@ -133,27 +131,31 @@ def test_get_data_store_params_schema(self): 'additionalProperties': False }, CDSDataStore.get_data_store_params_schema().to_dict()) - def test_get_type_specifiers(self): - type_specifiers = CDSDataStore.get_data_types() - self.assertEqual(1, len(type_specifiers)) - self.assertIsInstance(type_specifiers[0], str) - self.assertTupleEqual(('dataset[cube]',), type_specifiers) + def test_get_data_types(self): + data_types = CDSDataStore.get_data_types() + self.assertEqual(1, len(data_types)) + self.assertIsInstance(data_types[0], str) + self.assertTupleEqual((DATASET_TYPE.alias,), data_types) def test_has_data_false(self): self.assertFalse(CDSDataStore().has_data('nonexistent data ID')) def test_get_data_opener_ids_invalid_type_id(self): - with self.assertRaises(DataStoreError): - CDSDataStore().get_data_opener_ids(CDS_DATA_OPENER_ID, - 'this is an invalid ID') + with self.assertRaises(ValueError): + CDSDataStore().get_data_opener_ids( + data_id=CDS_DATA_OPENER_ID, + data_type='this is an invalid ID' + ) def test_get_data_opener_ids_invalid_opener_id(self): with self.assertRaises(ValueError): - CDSDataStore().get_data_opener_ids('this is an invalid ID', - TYPE_SPECIFIER_DATASET) + CDSDataStore().get_data_opener_ids( + data_id='this is an invalid ID', + data_type=DATASET_TYPE + ) def test_get_data_opener_ids_with_default_arguments(self): - self.assertTupleEqual((CDS_DATA_OPENER_ID, ), + self.assertTupleEqual((CDS_DATA_OPENER_ID,), CDSDataStore().get_data_opener_ids()) def test_get_store_open_params_schema_without_data_id(self): @@ -166,9 +168,8 @@ def test_get_data_ids(self): store = CDSDataStore(client_class=CDSClientMock, endpoint_url=_CDS_API_URL, cds_api_key=_CDS_API_KEY) - self.assertEqual([], list(store.get_data_ids('unsupported_type_spec'))) - self.assertEqual([], - list(store.get_data_ids('dataset[unsupported_flag]'))) + with self.assertRaises(ValueError): + list(store.get_data_ids(data_type='unsupported_data_type')) # The number of available datasets is expected to increase over time, # so to avoid overfitting the test we just check that more than a few @@ -177,8 +178,6 @@ def test_get_data_ids(self): minimum_expected_datasets = 5 self.assertGreater(len(list(store.get_data_ids('dataset'))), minimum_expected_datasets) - self.assertGreater(len(list(store.get_data_ids('dataset[cube]'))), - minimum_expected_datasets) def test_era5_transform_params_empty_variable_list(self): handler = ERA5DatasetHandler() @@ -190,7 +189,6 @@ def test_era5_transform_params_empty_variable_list(self): class ClientUrlTest(unittest.TestCase): - """Tests connected with passing CDS API URL and key to opener or store.""" def setUp(self): diff --git a/xcube_cds/store.py b/xcube_cds/store.py index f96a37a..0065795 100644 --- a/xcube_cds/store.py +++ b/xcube_cds/store.py @@ -765,7 +765,6 @@ def get_data_ids(self, data_type: DataTypeLike = None, include_attrs: Container[str] = None) -> \ Union[Iterator[str], Iterator[Tuple[str, Dict[str, Any]]]]: - if self._is_data_type_satisfied(data_type): # Only if the type specifier isn't compatible return_tuples = include_attrs is not None @@ -832,8 +831,8 @@ def open_data(self, data_id: str, opener_id: Optional[str] = None, def _validate_data_type(data_type: DataTypeLike): if not CDSDataStore._is_data_type_satisfied(data_type): raise DataStoreError( - f'Supplied type specifier {data_type!r} is not compatible ' - f'with "{DATASET_TYPE!r}."' + f'Supplied data type {data_type!r} is not compatible' + f' with "{DATASET_TYPE!r}."' ) @staticmethod @@ -845,7 +844,7 @@ def _is_data_type_satisfied( # implementations. if data_type is None: return True - return DATASET_TYPE.is_sub_type_of(data_type) + return DATASET_TYPE.is_super_type_of(data_type) @staticmethod def _assert_valid_opener_id(opener_id): From 25bd3d4a1d4e40274c8070c7598883fa074cf9ca Mon Sep 17 00:00:00 2001 From: Pontus Lurcock Date: Wed, 1 Sep 2021 12:33:04 +0200 Subject: [PATCH 3/3] Some minor tidy-ups Improving test method names, etc. --- test/test_era5.py | 2 +- test/test_store.py | 15 +++++++-------- xcube_cds/store.py | 5 ++--- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/test/test_era5.py b/test/test_era5.py index 3857923..a9c27ce 100644 --- a/test/test_era5.py +++ b/test/test_era5.py @@ -214,7 +214,7 @@ def test_era5_describe_data(self): self.assertTupleEqual(('time', 'latitude', 'longitude'), vd.dims) - def test_get_type_specifiers_for_data(self): + def test_get_data_types_for_data(self): store = CDSDataStore() self.assertEqual( (DATASET_TYPE.alias,), diff --git a/test/test_store.py b/test/test_store.py index 4e11583..9990083 100644 --- a/test/test_store.py +++ b/test/test_store.py @@ -52,7 +52,6 @@ from test.mocks import CDSClientMock from xcube.core.store import DATASET_TYPE from xcube.core.store import DataDescriptor -from xcube.core.store import DataStoreError from xcube_cds.constants import CDS_DATA_OPENER_ID from xcube_cds.datasets.reanalysis_era5 import ERA5DatasetHandler from xcube_cds.store import CDSDataOpener @@ -109,13 +108,13 @@ def test_get_open_params_schema_without_data_id(self): actual['properties'].keys() ) - def test_search_data_invalid_id(self): + def test_search_data_invalid_data_type(self): store = CDSDataStore(endpoint_url=_CDS_API_URL, cds_api_key=_CDS_API_KEY) with self.assertRaises(ValueError): - store.search_data(data_type='This is an invalid ID.') + store.search_data(data_type='This is an invalid data type.') - def test_search_data_valid_id(self): + def test_search_data_valid_data_type(self): store = CDSDataStore(endpoint_url=_CDS_API_URL, cds_api_key=_CDS_API_KEY) result = list(store.search_data('dataset')) @@ -140,17 +139,17 @@ def test_get_data_types(self): def test_has_data_false(self): self.assertFalse(CDSDataStore().has_data('nonexistent data ID')) - def test_get_data_opener_ids_invalid_type_id(self): + def test_get_data_opener_ids_invalid_data_type(self): with self.assertRaises(ValueError): CDSDataStore().get_data_opener_ids( data_id=CDS_DATA_OPENER_ID, - data_type='this is an invalid ID' + data_type='this is an invalid data type' ) - def test_get_data_opener_ids_invalid_opener_id(self): + def test_get_data_opener_ids_invalid_data_id(self): with self.assertRaises(ValueError): CDSDataStore().get_data_opener_ids( - data_id='this is an invalid ID', + data_id='this is an invalid data ID', data_type=DATASET_TYPE ) diff --git a/xcube_cds/store.py b/xcube_cds/store.py index 0065795..53c9838 100644 --- a/xcube_cds/store.py +++ b/xcube_cds/store.py @@ -51,7 +51,6 @@ from xcube.core.store import DataOpener from xcube.core.store import DataStore from xcube.core.store import DataStoreError -from xcube.core.store import DataType from xcube.core.store import DataTypeLike from xcube.core.store import DatasetDescriptor from xcube.core.store import DefaultSearchMixin @@ -776,7 +775,7 @@ def get_data_ids(self, if include_titles: yield data_id, \ {'title': - handler.get_human_readable_data_id(data_id)} + handler.get_human_readable_data_id(data_id)} else: yield data_id, {} else: @@ -795,7 +794,7 @@ def describe_data(self, data_id: str, return self._handler_registry[data_id].describe_data(data_id) # noinspection PyTypeChecker - def search_data(self, data_type: Optional[str] = None, + def search_data(self, data_type: Optional[DataTypeLike] = None, **search_params) \ -> Iterator[DataDescriptor]: self._validate_data_type(data_type)