Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Storage]Unify service version and update changelog #20723

Merged
merged 2 commits into from
Sep 17, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions sdk/storage/azure-storage-blob/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
# Release History

## 12.9.0 (2021-09-15)
**Stable release of preview features**
- Added support for service version 2020-10-02 (STG78)
- Added support for object level immutability policy with versioning (Version Level WORM).
- Added support for listing deleted root blobs that have versions.
- Added OAuth support for sync copy blob source.

## 12.9.0b1 (2021-07-27)
**New Features**
- Added support for object level immutability policy with versioning (Version Level WORM).
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -176,8 +176,7 @@ def __init__(
self._query_str, credential = self._format_query_string(sas_token, credential, snapshot=self.snapshot)
super(BlobClient, self).__init__(parsed_url, service='blob', credential=credential, **kwargs)
self._client = AzureBlobStorage(self.url, pipeline=self._pipeline)
default_api_version = self._client._config.version # pylint: disable=protected-access
self._client._config.version = get_api_version(kwargs, default_api_version) # pylint: disable=protected-access
self._client._config.version = get_api_version(kwargs) # pylint: disable=protected-access

def _format_url(self, hostname):
container_name = self.container_name
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,7 @@ def __init__(
self._query_str, credential = self._format_query_string(sas_token, credential)
super(BlobServiceClient, self).__init__(parsed_url, service='blob', credential=credential, **kwargs)
self._client = AzureBlobStorage(self.url, pipeline=self._pipeline)
default_api_version = self._client._config.version # pylint: disable=protected-access
self._client._config.version = get_api_version(kwargs, default_api_version) # pylint: disable=protected-access
self._client._config.version = get_api_version(kwargs) # pylint: disable=protected-access

def _format_url(self, hostname):
"""Format the endpoint URL according to the current location
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -156,8 +156,7 @@ def __init__(
self._query_str, credential = self._format_query_string(sas_token, credential)
super(ContainerClient, self).__init__(parsed_url, service='blob', credential=credential, **kwargs)
self._client = AzureBlobStorage(self.url, pipeline=self._pipeline)
default_api_version = self._client._config.version # pylint: disable=protected-access
self._client._config.version = get_api_version(kwargs, default_api_version) # pylint: disable=protected-access
self._client._config.version = get_api_version(kwargs) # pylint: disable=protected-access

def _format_url(self, hostname):
container_name = self.container_name
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -126,13 +126,13 @@ def get_container_cpk_scope_info(kwargs):
return None


def get_api_version(kwargs, default):
def get_api_version(kwargs):
# type: (Dict[str, Any], str) -> str
api_version = kwargs.pop('api_version', None)
api_version = kwargs.get('api_version', None)
if api_version and api_version not in _SUPPORTED_API_VERSIONS:
versions = '\n'.join(_SUPPORTED_API_VERSIONS)
raise ValueError("Unsupported API version '{}'. Please select from:\n{}".format(api_version, versions))
return api_version or default
return api_version or _SUPPORTED_API_VERSIONS[-1]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

-1 is last element?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes



def serialize_blob_tags_header(tags=None):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
# license information.
# --------------------------------------------------------------------------

VERSION = "12.9.0b1"
VERSION = "12.9.0"
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,7 @@ def __init__(
credential=credential,
**kwargs)
self._client = AzureBlobStorage(url=self.url, pipeline=self._pipeline)
default_api_version = self._client._config.version # pylint: disable=protected-access
self._client._config.version = get_api_version(kwargs, default_api_version) # pylint: disable=protected-access
self._client._config.version = get_api_version(kwargs) # pylint: disable=protected-access

@distributed_trace_async
async def get_account_information(self, **kwargs): # type: ignore
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,7 @@ def __init__(
credential=credential,
**kwargs)
self._client = AzureBlobStorage(url=self.url, pipeline=self._pipeline)
default_api_version = self._client._config.version # pylint: disable=protected-access
self._client._config.version = get_api_version(kwargs, default_api_version) # pylint: disable=protected-access
self._client._config.version = get_api_version(kwargs) # pylint: disable=protected-access

@distributed_trace_async
async def get_user_delegation_key(self, key_start_time, # type: datetime
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,7 @@ def __init__(
credential=credential,
**kwargs)
self._client = AzureBlobStorage(url=self.url, pipeline=self._pipeline)
default_api_version = self._client._config.version # pylint: disable=protected-access
self._client._config.version = get_api_version(kwargs, default_api_version) # pylint: disable=protected-access
self._client._config.version = get_api_version(kwargs) # pylint: disable=protected-access

@distributed_trace_async
async def create_container(self, metadata=None, public_access=None, **kwargs):
Expand Down
2 changes: 1 addition & 1 deletion sdk/storage/azure-storage-blob/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@
author_email='ascl@microsoft.com',
url='https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/storage/azure-storage-blob',
classifiers=[
'Development Status :: 4 - Beta',
"Development Status :: 5 - Production/Stable",
'Programming Language :: Python',
'Programming Language :: Python :: 2',
'Programming Language :: Python :: 2.7',
Expand Down
5 changes: 5 additions & 0 deletions sdk/storage/azure-storage-file-datalake/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
# Release History

## 12.5.0 (2021-09-15)
**Stable release of preview features**
- Added support for service version 2020-10-02 (STG78)
- Added support for quick query parquet

## 12.5.0b1 (2021-07-27)
**New features**
- Added support for quick query parquet
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -527,6 +527,7 @@ def get_file_client(self, file # type: Union[FileProperties, str]
)
return DataLakeFileClient(
self.url, self.file_system_name, file_path=file_path, credential=self._raw_credential,
api_version=self.api_version,
_hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline,
require_encryption=self.require_encryption,
key_encryption_key=self.key_encryption_key,
Expand Down Expand Up @@ -557,6 +558,7 @@ def get_sub_directory_client(self, sub_directory # type: Union[DirectoryPropert
)
return DataLakeDirectoryClient(
self.url, self.file_system_name, directory_name=subdir_path, credential=self._raw_credential,
api_version=self.api_version,
_hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline,
require_encryption=self.require_encryption,
key_encryption_key=self.key_encryption_key,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@
from ._data_lake_directory_client import DataLakeDirectoryClient
from ._data_lake_file_client import DataLakeFileClient
from ._models import UserDelegationKey, FileSystemPropertiesPaged, LocationMode
from ._serialize import convert_dfs_url_to_blob_url
from ._serialize import convert_dfs_url_to_blob_url, get_api_version
from ._generated import AzureDataLakeStorageRESTAPI


class DataLakeServiceClient(StorageAccountHostsMixin):
Expand Down Expand Up @@ -94,6 +95,9 @@ def __init__(
# ADLS doesn't support secondary endpoint, make sure it's empty
self._hosts[LocationMode.SECONDARY] = ""

self._client = AzureDataLakeStorageRESTAPI(self.url, pipeline=self._pipeline)
self._client._config.version = get_api_version(kwargs) #pylint: disable=protected-access

def __enter__(self):
self._blob_service_client.__enter__()
return self
Expand Down Expand Up @@ -388,6 +392,7 @@ def get_file_system_client(self, file_system # type: Union[FileSystemProperties
policies=self._pipeline._impl_policies # pylint: disable = protected-access
)
return FileSystemClient(self.url, file_system_name, credential=self._raw_credential,
api_version=self.api_version,
_configuration=self._config,
_pipeline=_pipeline, _hosts=self._hosts,
require_encryption=self.require_encryption, key_encryption_key=self.key_encryption_key,
Expand Down Expand Up @@ -436,6 +441,7 @@ def get_directory_client(self, file_system, # type: Union[FileSystemProperties,
)
return DataLakeDirectoryClient(self.url, file_system_name, directory_name=directory_name,
credential=self._raw_credential,
api_version=self.api_version,
_configuration=self._config, _pipeline=_pipeline,
_hosts=self._hosts,
require_encryption=self.require_encryption,
Expand Down Expand Up @@ -486,6 +492,7 @@ def get_file_client(self, file_system, # type: Union[FileSystemProperties, str]
)
return DataLakeFileClient(
self.url, file_system_name, file_path=file_path, credential=self._raw_credential,
api_version=self.api_version,
_hosts=self._hosts, _configuration=self._config, _pipeline=_pipeline,
require_encryption=self.require_encryption,
key_encryption_key=self.key_encryption_key,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from azure.core.paging import ItemPaged
from azure.storage.blob import ContainerClient
from ._shared.base_client import TransportWrapper, StorageAccountHostsMixin, parse_query, parse_connection_str
from ._serialize import convert_dfs_url_to_blob_url
from ._serialize import convert_dfs_url_to_blob_url, get_api_version
from ._list_paths_helper import DeletedPathPropertiesPaged
from ._models import LocationMode, FileSystemProperties, PublicAccess, DeletedPathProperties, FileProperties, \
DirectoryProperties
Expand Down Expand Up @@ -105,9 +105,12 @@ def __init__(
# ADLS doesn't support secondary endpoint, make sure it's empty
self._hosts[LocationMode.SECONDARY] = ""
self._client = AzureDataLakeStorageRESTAPI(self.url, file_system=file_system_name, pipeline=self._pipeline)
api_version = get_api_version(kwargs)
self._client._config.version = api_version # pylint: disable=protected-access
self._datalake_client_for_blob_operation = AzureDataLakeStorageRESTAPI(self._container_client.url,
file_system=file_system_name,
pipeline=self._pipeline)
self._datalake_client_for_blob_operation._config.version = api_version # pylint: disable=protected-access

def _format_url(self, hostname):
file_system_name = self.file_system_name
Expand Down Expand Up @@ -839,6 +842,7 @@ def get_directory_client(self, directory # type: Union[DirectoryProperties, str
)
return DataLakeDirectoryClient(self.url, self.file_system_name, directory_name=directory_name,
credential=self._raw_credential,
api_version=self.api_version,
_configuration=self._config, _pipeline=_pipeline,
_hosts=self._hosts,
require_encryption=self.require_encryption,
Expand Down Expand Up @@ -879,6 +883,7 @@ def get_file_client(self, file_path # type: Union[FileProperties, str]
)
return DataLakeFileClient(
self.url, self.file_system_name, file_path=file_path, credential=self._raw_credential,
api_version=self.api_version,
_hosts=self._hosts, _configuration=self._config, _pipeline=_pipeline,
require_encryption=self.require_encryption,
key_encryption_key=self.key_encryption_key,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@
from ._models import LocationMode, DirectoryProperties, AccessControlChangeResult, AccessControlChanges, \
AccessControlChangeCounters, AccessControlChangeFailure
from ._serialize import convert_dfs_url_to_blob_url, get_mod_conditions, \
get_path_http_headers, add_metadata_headers, get_lease_id, get_source_mod_conditions, get_access_conditions
get_path_http_headers, add_metadata_headers, get_lease_id, get_source_mod_conditions, get_access_conditions, \
get_api_version
from ._shared.base_client import StorageAccountHostsMixin, parse_query
from ._shared.response_handlers import return_response_headers, return_headers_and_deserialized

Expand Down Expand Up @@ -81,13 +82,18 @@ def __init__(
_hosts=datalake_hosts, **kwargs)
# ADLS doesn't support secondary endpoint, make sure it's empty
self._hosts[LocationMode.SECONDARY] = ""
api_version = get_api_version(kwargs)

self._client = AzureDataLakeStorageRESTAPI(self.url, file_system=file_system_name, path=path_name,
pipeline=self._pipeline)
self._client._config.version = api_version # pylint: disable=protected-access

self._datalake_client_for_blob_operation = AzureDataLakeStorageRESTAPI(
self._blob_client.url,
file_system=file_system_name,
path=path_name,
pipeline=self._pipeline)
self._datalake_client_for_blob_operation._config.version = api_version # pylint: disable=protected-access

def __exit__(self, *args):
self._blob_client.close()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,28 @@
SourceModifiedAccessConditions, LeaseAccessConditions


_SUPPORTED_API_VERSIONS = [
'2019-02-02',
'2019-07-07',
'2019-10-10',
'2019-12-12',
'2020-02-10',
'2020-04-08',
'2020-06-12',
'2020-08-04',
'2020-10-02'
]


def get_api_version(kwargs):
# type: (Dict[str, Any], str) -> str
api_version = kwargs.get('api_version', None)
if api_version and api_version not in _SUPPORTED_API_VERSIONS:
versions = '\n'.join(_SUPPORTED_API_VERSIONS)
raise ValueError("Unsupported API version '{}'. Please select from:\n{}".format(api_version, versions))
return api_version or _SUPPORTED_API_VERSIONS[-1]


def convert_dfs_url_to_blob_url(dfs_account_url):
return dfs_account_url.replace('.dfs.', '.blob.', 1)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
# license information.
# --------------------------------------------------------------------------

VERSION = "12.5.0b1"
VERSION = "12.5.0"
Original file line number Diff line number Diff line change
Expand Up @@ -506,6 +506,7 @@ def get_file_client(self, file # type: Union[FileProperties, str]
)
return DataLakeFileClient(
self.url, self.file_system_name, file_path=file_path, credential=self._raw_credential,
api_version=self.api_version,
_hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline,
_location_mode=self._location_mode, require_encryption=self.require_encryption,
key_encryption_key=self.key_encryption_key,
Expand Down Expand Up @@ -545,6 +546,7 @@ def get_sub_directory_client(self, sub_directory # type: Union[DirectoryPropert
)
return DataLakeDirectoryClient(
self.url, self.file_system_name, directory_name=subdir_path, credential=self._raw_credential,
api_version=self.api_version,
_hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline,
_location_mode=self._location_mode, require_encryption=self.require_encryption,
key_encryption_key=self.key_encryption_key,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from azure.core.pipeline import AsyncPipeline

from azure.storage.blob.aio import BlobServiceClient
from .._serialize import get_api_version
from .._generated.aio import AzureDataLakeStorageRESTAPI
from .._deserialize import get_datalake_service_properties
from .._shared.base_client_async import AsyncTransportWrapper, AsyncStorageAccountHostsMixin
Expand Down Expand Up @@ -80,6 +81,7 @@ def __init__(
self._blob_service_client = BlobServiceClient(self._blob_account_url, credential, **kwargs)
self._blob_service_client._hosts[LocationMode.SECONDARY] = "" #pylint: disable=protected-access
self._client = AzureDataLakeStorageRESTAPI(self.url, pipeline=self._pipeline)
self._client._config.version = get_api_version(kwargs) #pylint: disable=protected-access
self._loop = kwargs.get('loop', None)

async def __aenter__(self):
Expand Down Expand Up @@ -337,6 +339,7 @@ def get_file_system_client(self, file_system # type: Union[FileSystemProperties
policies=self._pipeline._impl_policies # pylint: disable = protected-access
)
return FileSystemClient(self.url, file_system_name, credential=self._raw_credential,
api_version=self.api_version,
_configuration=self._config,
_pipeline=self._pipeline, _hosts=self._hosts,
require_encryption=self.require_encryption, key_encryption_key=self.key_encryption_key,
Expand Down Expand Up @@ -385,6 +388,7 @@ def get_directory_client(self, file_system, # type: Union[FileSystemProperties,
)
return DataLakeDirectoryClient(self.url, file_system_name, directory_name=directory_name,
credential=self._raw_credential,
api_version=self.api_version,
_configuration=self._config, _pipeline=self._pipeline,
_hosts=self._hosts,
require_encryption=self.require_encryption,
Expand Down Expand Up @@ -435,6 +439,7 @@ def get_file_client(self, file_system, # type: Union[FileSystemProperties, str]
)
return DataLakeFileClient(
self.url, file_system_name, file_path=file_path, credential=self._raw_credential,
api_version=self.api_version,
_hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline,
require_encryption=self.require_encryption,
key_encryption_key=self.key_encryption_key,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

from azure.core.tracing.decorator_async import distributed_trace_async
from azure.storage.blob.aio import ContainerClient
from .._serialize import get_api_version
from .._deserialize import process_storage_error, is_file_path
from .._generated.models import ListBlobsIncludeItem

Expand Down Expand Up @@ -98,6 +99,10 @@ def __init__(
self._datalake_client_for_blob_operation = AzureDataLakeStorageRESTAPI(self._container_client.url,
file_system=file_system_name,
pipeline=self._pipeline)
api_version = get_api_version(kwargs)
self._client._config.version = api_version # pylint: disable=protected-access
self._datalake_client_for_blob_operation._config.version = api_version # pylint: disable=protected-access

self._loop = kwargs.get('loop', None)

async def __aexit__(self, *args):
Expand Down Expand Up @@ -787,6 +792,7 @@ def get_directory_client(self, directory # type: Union[DirectoryProperties, str
)
return DataLakeDirectoryClient(self.url, self.file_system_name, directory_name=directory_name,
credential=self._raw_credential,
api_version=self.api_version,
_configuration=self._config, _pipeline=_pipeline,
_hosts=self._hosts,
require_encryption=self.require_encryption,
Expand Down Expand Up @@ -828,6 +834,7 @@ def get_file_client(self, file_path # type: Union[FileProperties, str]
)
return DataLakeFileClient(
self.url, self.file_system_name, file_path=file_path, credential=self._raw_credential,
api_version=self.api_version,
_hosts=self._hosts, _configuration=self._config, _pipeline=_pipeline,
require_encryption=self.require_encryption,
key_encryption_key=self.key_encryption_key,
Expand Down
Loading