diff --git a/RELEASE.md b/RELEASE.md index c8ffa1188a..260ebe289c 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -3,6 +3,7 @@ ## Major features and improvements ## Bug fixes and other changes +* Compare for protocol and delimiter in `PartitionedDataSet` to be able to pass the protocol to partitions which paths starts with the same characters as the protocol (e.g. `s3://s3-my-bucket`). ## Breaking changes to the API diff --git a/kedro/io/partitioned_dataset.py b/kedro/io/partitioned_dataset.py index 07b34ca65c..683abac283 100644 --- a/kedro/io/partitioned_dataset.py +++ b/kedro/io/partitioned_dataset.py @@ -263,10 +263,11 @@ def _list_partitions(self) -> list[str]: ] def _join_protocol(self, path: str) -> str: - if self._path.startswith(self._protocol) and not path.startswith( - self._protocol + protocol_prefix = f"{self._protocol}://" + if self._path.startswith(protocol_prefix) and not path.startswith( + protocol_prefix ): - return f"{self._protocol}://{path}" + return f"{protocol_prefix}{path}" return path def _partition_to_path(self, path: str): diff --git a/tests/io/test_partitioned_dataset.py b/tests/io/test_partitioned_dataset.py index 05993593ae..2d0af84a61 100644 --- a/tests/io/test_partitioned_dataset.py +++ b/tests/io/test_partitioned_dataset.py @@ -401,7 +401,7 @@ def test_dataset_creds(self, pds_config, expected_ds_creds, global_creds): assert pds._credentials == global_creds -BUCKET_NAME = "fake_bucket_name" +BUCKET_NAME = "s3_fake_bucket_name" S3_DATASET_DEFINITION = [ "pandas.CSVDataSet", "kedro.extras.datasets.pandas.CSVDataSet",