Revert "Merge pull request #527 from dcs4cop/toniof-516-unify-datasetio"

This reverts commit 61815c3, reversing
changes made to 0b9c41c.

Author: dzelge

examples/serve/demo/config-with-stores.yml

@@ -36,7 +36,7 @@ DataStores:
         # client_kwargs:
         #  endpoint_url: https://s3.eu-central-1.amazonaws.com
     Datasets:
-      - Path: "*.zarr"
+      - Identifier: "*.zarr"
         Style: "default"
         # ChunkCacheSize: 1G
 

test/core/store/test_storepool.py

@@ -326,34 +326,6 @@ def test_multi_stores_with_params(self):
         self.assertIsInstance(pool, DataStorePool)
         self.assertEqual(["local-1", "local-2", "ram-1", "ram-2"], pool.store_instance_ids)
         for instance_id in pool.store_instance_ids:
-            self.assertTrue(pool.has_store_instance(instance_id))
+            self.assertTrue(pool.has_store_config(instance_id))
             self.assertIsInstance(pool.get_store_config(instance_id), DataStoreConfig)
             self.assertIsInstance(pool.get_store(instance_id), DataStore)
-
-    def test_get_store_instance_id(self):
-        store_params_1 = {
-            "root": "./bibo"
-        }
-        ds_config_1 = DataStoreConfig(store_id='file',
-                                      store_params=store_params_1)
-        ds_configs = {'dir-1': ds_config_1}
-        pool = DataStorePool(ds_configs)
-
-        store_params_2 = {
-            "root": "./babo"
-        }
-        ds_config_2 = DataStoreConfig(store_id='file',
-                                      store_params=store_params_2)
-        ds_config_3 = DataStoreConfig(store_id='file',
-                                      store_params=store_params_1,
-                                      title='A third configuration')
-
-        self.assertEqual('dir-1', pool.get_store_instance_id(ds_config_1))
-        self.assertEqual('dir-1', pool.get_store_instance_id(ds_config_1,
-                                                             strict_check=True))
-
-        self.assertIsNone(pool.get_store_instance_id(ds_config_2))
-
-        self.assertEqual('dir-1', pool.get_store_instance_id(ds_config_3))
-        self.assertIsNone(pool.get_store_instance_id(ds_config_3,
-                                                     strict_check=True))

test/webapi/test_config.py

@@ -60,7 +60,7 @@ def test_from_dict(self):
                     },
                     "Datasets": [
                         {
-                            "Path": "*.zarr",
+                            "Identifier": "*.zarr",
                             "Style": "default"
                         }
                     ]

test/webapi/test_context.py

@@ -211,246 +211,3 @@ def test_interpolates_vars(self):
                          normalize_prefix('/${name}'))
         self.assertEqual(f'/xcube/v{version}',
                          normalize_prefix('/${name}/v${version}'))
-
-
-class MaybeAssignStoreInstanceIdsTest(unittest.TestCase):
-
-    def test_find_common_store(self):
-        ctx = new_test_service_context()
-        dataset_configs = [
-            {
-                'Identifier': 'z_0',
-                'FileSystem': 'local',
-                'Path': '/one/path/abc.zarr'
-            },
-            {
-                'Identifier': 'z_1',
-                'FileSystem': 'local',
-                'Path': '/one/path/def.zarr'
-            },
-            {
-                'Identifier': 'z_4',
-                'FileSystem': 'obs',
-                'Path': '/one/path/mno.zarr'
-            },
-            {
-                'Identifier': 'z_2',
-                'FileSystem': 'local',
-                'Path': '/another/path/ghi.zarr'
-            },
-            {
-                'Identifier': 'z_3',
-                'FileSystem': 'local',
-                'Path': '/one/more/path/jkl.zarr'
-            },
-            {
-                'Identifier': 'z_5',
-                'FileSystem': 'obs',
-                'Path': '/one/path/pqr.zarr'
-            },
-            {
-                'Identifier': 'z_6',
-                'FileSystem': 'local',
-                'Path': '/one/path/stu.zarr'
-            },
-            {
-                'Identifier': 'z_7',
-                'FileSystem': 'local',
-                'Path': '/one/more/path/vwx.zarr'
-            },
-        ]
-        ctx.config['Datasets'] = dataset_configs
-        adjusted_dataset_configs = ctx.get_dataset_configs()
-
-        expected_dataset_configs = [
-            {
-                'Identifier': 'z_0',
-                'FileSystem': 'local',
-                'Path': 'path/abc.zarr',
-                'StoreInstanceId': 'local_2'
-            },
-            {
-                'Identifier': 'z_1',
-                'FileSystem': 'local',
-                'Path': 'path/def.zarr',
-                'StoreInstanceId': 'local_2'
-            },
-            {
-                'Identifier': 'z_4',
-                'FileSystem': 'obs',
-                'Path': 'mno.zarr',
-                'StoreInstanceId': 'obs_1'
-            },
-            {
-                'Identifier': 'z_2',
-                'FileSystem': 'local',
-                'Path': 'ghi.zarr',
-                'StoreInstanceId': 'local_1'
-            },
-            {
-                'Identifier': 'z_3',
-                'FileSystem': 'local',
-                'Path': 'more/path/jkl.zarr',
-                'StoreInstanceId': 'local_2'
-            },
-            {
-                'Identifier': 'z_5',
-                'FileSystem': 'obs',
-                'Path': 'pqr.zarr',
-                'StoreInstanceId': 'obs_1'
-            },
-            {
-                'Identifier': 'z_6',
-                'FileSystem': 'local',
-                'Path': 'path/stu.zarr',
-                'StoreInstanceId': 'local_2'
-            },
-            {
-                'Identifier': 'z_7',
-                'FileSystem': 'local',
-                'Path': 'more/path/vwx.zarr',
-                'StoreInstanceId': 'local_2'
-            },
-        ]
-        self.assertEqual(expected_dataset_configs, adjusted_dataset_configs)
-
-    def test_with_instance_id(self):
-        ctx = new_test_service_context()
-        dataset_config = {'Identifier': 'zero',
-                          'Title': 'Test 0',
-                          'FileSystem': 'local',
-                          'StoreInstanceId': 'some_id'}
-        dataset_config_copy = dataset_config.copy()
-
-        ctx.config['Datasets'] = [dataset_config]
-        dataset_config = ctx.get_dataset_configs()[0]
-
-        self.assertEqual(dataset_config_copy, dataset_config)
-
-    def test_local(self):
-        ctx = new_test_service_context()
-        dataset_config = {'Identifier': 'one',
-                          'Title': 'Test 1',
-                          'FileSystem': 'local',
-                          'Path': 'cube-1-250-250.zarr'}
-
-        ctx.config['Datasets'] = [dataset_config]
-        dataset_config = ctx.get_dataset_configs()[0]
-
-        self.assertEqual(['Identifier', 'Title', 'FileSystem', 'Path',
-                         'StoreInstanceId'],
-                         list(dataset_config.keys()))
-        self.assertEqual('one',
-                         dataset_config['Identifier'])
-        self.assertEqual('Test 1', dataset_config['Title'])
-        self.assertEqual('local', dataset_config['FileSystem'])
-        self.assertEqual('cube-1-250-250.zarr', dataset_config["Path"])
-        self.assertEqual('local_1', dataset_config['StoreInstanceId'])
-
-    def test_s3(self):
-        ctx = new_test_service_context()
-        dataset_config = {'Identifier': 'two',
-                          'Title': 'Test 2',
-                          'FileSystem': 'obs',
-                          'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
-                          'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr',
-                          'Region': 'eu-central-1'}
-
-        ctx.config['Datasets'] = [dataset_config]
-        dataset_config = ctx.get_dataset_configs()[0]
-
-        self.assertEqual(['Identifier', 'Title', 'FileSystem', 'Endpoint',
-                          'Path', 'Region', 'StoreInstanceId'],
-                         list(dataset_config.keys()))
-        self.assertEqual('two', dataset_config['Identifier'])
-        self.assertEqual('Test 2', dataset_config['Title'])
-        self.assertEqual('obs', dataset_config['FileSystem'])
-        self.assertEqual('https://s3.eu-central-1.amazonaws.com',
-                         dataset_config['Endpoint'])
-        self.assertEqual('OLCI-SNS-RAW-CUBE-2.zarr', dataset_config['Path'])
-        self.assertEqual('eu-central-1', dataset_config['Region'])
-        self.assertEqual('obs_1', dataset_config['StoreInstanceId'])
-
-    def test_memory(self):
-        ctx = new_test_service_context()
-        dataset_config = {'Identifier': 'three',
-                          'Title': 'Test 3',
-                          'FileSystem': 'memory'}
-        dataset_config_copy = dataset_config.copy()
-
-        ctx.config['Datasets'] = [dataset_config]
-        dataset_config = ctx.get_dataset_configs()[0]
-
-        self.assertEqual(dataset_config_copy, dataset_config)
-
-    def test_missing_file_system(self):
-        ctx = new_test_service_context()
-        dataset_config = {'Identifier': 'five',
-                          'Title': 'Test 5',
-                          'Path': 'cube-1-250-250.zarr'}
-
-        ctx.config['Datasets'] = [dataset_config]
-        dataset_config = ctx.get_dataset_configs()[0]
-
-        self.assertEqual(['Identifier', 'Title', 'Path', 'StoreInstanceId'],
-                         list(dataset_config.keys()))
-        self.assertEqual('five', dataset_config['Identifier'])
-        self.assertEqual('Test 5', dataset_config['Title'])
-        self.assertEqual('cube-1-250-250.zarr', dataset_config['Path'])
-        self.assertEqual('local_1', dataset_config['StoreInstanceId'])
-
-    def test_invalid_file_system(self):
-        ctx = new_test_service_context()
-        dataset_config = {'Identifier': 'five',
-                          'Title': 'Test 5a',
-                          'FileSystem': 'invalid',
-                          'Path': 'cube-1-250-250.zarr'}
-
-        ctx.config['Datasets'] = [dataset_config]
-        dataset_config = ctx.get_dataset_configs()[0]
-
-        self.assertEqual(['Identifier', 'Title', 'FileSystem', 'Path'],
-                         list(dataset_config.keys()))
-        self.assertEqual('five', dataset_config['Identifier'])
-        self.assertEqual('Test 5a', dataset_config['Title'])
-        self.assertEqual('invalid', dataset_config['FileSystem'])
-        self.assertEqual('cube-1-250-250.zarr', dataset_config['Path'])
-
-    def test_local_store_already_existing(self):
-        ctx = new_test_service_context()
-        dataset_config_1 = {'Identifier': 'six',
-                            'Title': 'Test 6',
-                            'FileSystem': 'local',
-                            'Path': 'cube-1-250-250.zarr'}
-        dataset_config_2 = {'Identifier': 'six_a',
-                            'Title': 'Test 6 a',
-                            'FileSystem': 'local',
-                            'Path': 'cube-5-100-200.zarr'}
-
-        ctx.config['Datasets'] = [dataset_config_1, dataset_config_2]
-        dataset_configs = ctx.get_dataset_configs()
-
-        self.assertEqual(dataset_configs[0]['StoreInstanceId'],
-                         dataset_configs[1]['StoreInstanceId'])
-
-    def test_s3_store_already_existing(self):
-        ctx = new_test_service_context()
-        dataset_config_1 = {'Identifier': 'seven',
-                            'Title': 'Test 7',
-                            'FileSystem': 'obs',
-                            'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
-                            'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr',
-                            'Region': 'eu-central-1'}
-
-        dataset_config_2 = {'Identifier': 'seven_a',
-                            'Title': 'Test 7 a',
-                            'FileSystem': 'obs',
-                            'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
-                            'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-3.zarr',
-                            'Region': 'eu-central-1'}
-
-        ctx.config['Datasets'] = [dataset_config_1, dataset_config_2]
-        dataset_configs = ctx.get_dataset_configs()
-
-        self.assertEqual(dataset_configs[0]['StoreInstanceId'],
-                         dataset_configs[1]['StoreInstanceId'])

xcube/core/store/storepool.py

@@ -262,25 +262,7 @@ def store_instance_ids(self) -> List[str]:
     def store_configs(self) -> List[DataStoreConfig]:
         return [v.store_config for k, v in self._instances.items()]
 
-    def get_store_instance_id(self,
-                              store_config: DataStoreConfig,
-                              strict_check: bool = False) -> Optional[str]:
-        assert_instance(store_config, DataStoreConfig, 'store_config')
-        for id, instance in self._instances.items():
-            if strict_check:
-                if instance.store_config == store_config:
-                    return id
-            else:
-                if instance.store_config.store_id == store_config.store_id and \
-                    instance.store_config.store_params == \
-                        store_config.store_params:
-                    return id
-        return None
-
-    def has_store_config(self, store_config: DataStoreConfig) -> bool:
-        return self.get_store_instance_id(store_config) is not None
-
-    def has_store_instance(self, store_instance_id: str) -> bool:
+    def has_store_config(self, store_instance_id: str) -> bool:
         assert_instance(store_instance_id, str, 'store_instance_id')
         return store_instance_id in self._instances
 

xcube/webapi/config.py

@@ -91,7 +91,6 @@ def get_schema(cls) -> JsonObjectSchema:
             factory=DatasetConfig,
             required=[
                 'Identifier',
-                'Path'
             ],
             properties=dict(
                 Identifier=IdentifierSchema,
@@ -182,11 +181,10 @@ def get_schema(cls) -> JsonObjectSchema:
         return JsonObjectSchema(
             factory=DataStoreDatasetConfig,
             required=[
-                'Path'
+                'Identifier'
             ],
             properties=dict(
                 Identifier=IdentifierSchema,
-                Path=PathSchema,
                 StoreInstanceId=IdentifierSchema,  # will be set by server
                 StoreOpenParams=JsonObjectSchema(additional_properties=True),
                 **_get_common_dataset_properties()

xcube/webapi/context.py

@@ -281,9 +281,11 @@ def _get_key_and_local_path(self, ds_id: str, path: str):
         if path and '..' in path.split('/'):
             raise ServiceBadRequestError(f'AWS S3 data access: received illegal key {key!r}')
 
-        bucket_mapping = self.service_context.get_s3_bucket_mapping()
-        local_path = bucket_mapping.get(ds_id)
-        local_path = os.path.join(local_path, path)
+        local_path = dataset_config.get('Path')
+        if os.path.isabs(local_path):
+            local_path = os.path.join(local_path, path)
+        else:
+            local_path = os.path.join(self.service_context.base_dir, local_path, path)
 
         local_path = os.path.normpath(local_path)