[Storage]Unify service version and update changelog (Azure#20723)

* [Storage]Unify service version and update changelog

* fix test

sdk/storage/azure-storage-blob/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Release History
 
+## 12.9.0 (2021-09-15)
+**Stable release of preview features**
+- Added support for service version 2020-10-02 (STG78)
+- Added support for object level immutability policy with versioning (Version Level WORM).
+- Added support for listing deleted root blobs that have versions.
+- Added OAuth support for sync copy blob source.
+
 ## 12.9.0b1 (2021-07-27)
 **New Features**
 - Added support for object level immutability policy with versioning (Version Level WORM).

sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py

@@ -176,8 +176,7 @@ def __init__(
         self._query_str, credential = self._format_query_string(sas_token, credential, snapshot=self.snapshot)
         super(BlobClient, self).__init__(parsed_url, service='blob', credential=credential, **kwargs)
         self._client = AzureBlobStorage(self.url, pipeline=self._pipeline)
-        default_api_version = self._client._config.version  # pylint: disable=protected-access
-        self._client._config.version = get_api_version(kwargs, default_api_version)  # pylint: disable=protected-access
+        self._client._config.version = get_api_version(kwargs)  # pylint: disable=protected-access
 
     def _format_url(self, hostname):
         container_name = self.container_name

sdk/storage/azure-storage-blob/azure/storage/blob/_blob_service_client.py

@@ -134,8 +134,7 @@ def __init__(
         self._query_str, credential = self._format_query_string(sas_token, credential)
         super(BlobServiceClient, self).__init__(parsed_url, service='blob', credential=credential, **kwargs)
         self._client = AzureBlobStorage(self.url, pipeline=self._pipeline)
-        default_api_version = self._client._config.version  # pylint: disable=protected-access
-        self._client._config.version = get_api_version(kwargs, default_api_version)  # pylint: disable=protected-access
+        self._client._config.version = get_api_version(kwargs)  # pylint: disable=protected-access
 
     def _format_url(self, hostname):
         """Format the endpoint URL according to the current location

sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py

@@ -156,8 +156,7 @@ def __init__(
         self._query_str, credential = self._format_query_string(sas_token, credential)
         super(ContainerClient, self).__init__(parsed_url, service='blob', credential=credential, **kwargs)
         self._client = AzureBlobStorage(self.url, pipeline=self._pipeline)
-        default_api_version = self._client._config.version  # pylint: disable=protected-access
-        self._client._config.version = get_api_version(kwargs, default_api_version) # pylint: disable=protected-access
+        self._client._config.version = get_api_version(kwargs) # pylint: disable=protected-access
 
     def _format_url(self, hostname):
         container_name = self.container_name

sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py

@@ -126,13 +126,13 @@ def get_container_cpk_scope_info(kwargs):
     return None
 
 
-def get_api_version(kwargs, default):
+def get_api_version(kwargs):
     # type: (Dict[str, Any], str) -> str
-    api_version = kwargs.pop('api_version', None)
+    api_version = kwargs.get('api_version', None)
     if api_version and api_version not in _SUPPORTED_API_VERSIONS:
         versions = '\n'.join(_SUPPORTED_API_VERSIONS)
         raise ValueError("Unsupported API version '{}'. Please select from:\n{}".format(api_version, versions))
-    return api_version or default
+    return api_version or _SUPPORTED_API_VERSIONS[-1]
 
 
 def serialize_blob_tags_header(tags=None):

sdk/storage/azure-storage-blob/azure/storage/blob/_version.py

@@ -4,4 +4,4 @@
 # license information.
 # --------------------------------------------------------------------------
 
-VERSION = "12.9.0b1"
+VERSION = "12.9.0"

sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py

@@ -120,8 +120,7 @@ def __init__(
             credential=credential,
             **kwargs)
         self._client = AzureBlobStorage(url=self.url, pipeline=self._pipeline)
-        default_api_version = self._client._config.version  # pylint: disable=protected-access
-        self._client._config.version = get_api_version(kwargs, default_api_version)  # pylint: disable=protected-access
+        self._client._config.version = get_api_version(kwargs)  # pylint: disable=protected-access
 
     @distributed_trace_async
     async def get_account_information(self, **kwargs): # type: ignore

sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_service_client_async.py

@@ -118,8 +118,7 @@ def __init__(
             credential=credential,
             **kwargs)
         self._client = AzureBlobStorage(url=self.url, pipeline=self._pipeline)
-        default_api_version = self._client._config.version  # pylint: disable=protected-access
-        self._client._config.version = get_api_version(kwargs, default_api_version)  # pylint: disable=protected-access
+        self._client._config.version = get_api_version(kwargs)  # pylint: disable=protected-access
 
     @distributed_trace_async
     async def get_user_delegation_key(self, key_start_time,  # type: datetime

sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py

@@ -117,8 +117,7 @@ def __init__(
             credential=credential,
             **kwargs)
         self._client = AzureBlobStorage(url=self.url, pipeline=self._pipeline)
-        default_api_version = self._client._config.version  # pylint: disable=protected-access
-        self._client._config.version = get_api_version(kwargs, default_api_version)  # pylint: disable=protected-access
+        self._client._config.version = get_api_version(kwargs)  # pylint: disable=protected-access
 
     @distributed_trace_async
     async def create_container(self, metadata=None, public_access=None, **kwargs):

sdk/storage/azure-storage-blob/setup.py

@@ -72,7 +72,7 @@
     author_email='ascl@microsoft.com',
     url='https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/storage/azure-storage-blob',
     classifiers=[
-        'Development Status :: 4 - Beta',
+        "Development Status :: 5 - Production/Stable",
         'Programming Language :: Python',
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',

sdk/storage/azure-storage-file-datalake/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Release History
 
+## 12.5.0 (2021-09-15)
+**Stable release of preview features**
+- Added support for service version 2020-10-02 (STG78)
+- Added support for quick query parquet
+
 ## 12.5.0b1 (2021-07-27)
 **New features**
 - Added support for quick query parquet

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_directory_client.py

@@ -527,6 +527,7 @@ def get_file_client(self, file  # type: Union[FileProperties, str]
         )
         return DataLakeFileClient(
             self.url, self.file_system_name, file_path=file_path, credential=self._raw_credential,
+            api_version=self.api_version,
             _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline,
             require_encryption=self.require_encryption,
             key_encryption_key=self.key_encryption_key,
@@ -557,6 +558,7 @@ def get_sub_directory_client(self, sub_directory  # type: Union[DirectoryPropert
         )
         return DataLakeDirectoryClient(
             self.url, self.file_system_name, directory_name=subdir_path, credential=self._raw_credential,
+            api_version=self.api_version,
             _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline,
             require_encryption=self.require_encryption,
             key_encryption_key=self.key_encryption_key,

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_service_client.py

@@ -20,7 +20,8 @@
 from ._data_lake_directory_client import DataLakeDirectoryClient
 from ._data_lake_file_client import DataLakeFileClient
 from ._models import UserDelegationKey, FileSystemPropertiesPaged, LocationMode
-from ._serialize import convert_dfs_url_to_blob_url
+from ._serialize import convert_dfs_url_to_blob_url, get_api_version
+from ._generated import AzureDataLakeStorageRESTAPI
 
 
 class DataLakeServiceClient(StorageAccountHostsMixin):
@@ -94,6 +95,9 @@ def __init__(
         # ADLS doesn't support secondary endpoint, make sure it's empty
         self._hosts[LocationMode.SECONDARY] = ""
 
+        self._client = AzureDataLakeStorageRESTAPI(self.url, pipeline=self._pipeline)
+        self._client._config.version = get_api_version(kwargs)  #pylint: disable=protected-access
+
     def __enter__(self):
         self._blob_service_client.__enter__()
         return self
@@ -388,6 +392,7 @@ def get_file_system_client(self, file_system  # type: Union[FileSystemProperties
             policies=self._pipeline._impl_policies # pylint: disable = protected-access
         )
         return FileSystemClient(self.url, file_system_name, credential=self._raw_credential,
+                                api_version=self.api_version,
                                 _configuration=self._config,
                                 _pipeline=_pipeline, _hosts=self._hosts,
                                 require_encryption=self.require_encryption, key_encryption_key=self.key_encryption_key,
@@ -436,6 +441,7 @@ def get_directory_client(self, file_system,  # type: Union[FileSystemProperties,
         )
         return DataLakeDirectoryClient(self.url, file_system_name, directory_name=directory_name,
                                        credential=self._raw_credential,
+                                       api_version=self.api_version,
                                        _configuration=self._config, _pipeline=_pipeline,
                                        _hosts=self._hosts,
                                        require_encryption=self.require_encryption,
@@ -486,6 +492,7 @@ def get_file_client(self, file_system,  # type: Union[FileSystemProperties, str]
         )
         return DataLakeFileClient(
             self.url, file_system_name, file_path=file_path, credential=self._raw_credential,
+            api_version=self.api_version,
             _hosts=self._hosts, _configuration=self._config, _pipeline=_pipeline,
             require_encryption=self.require_encryption,
             key_encryption_key=self.key_encryption_key,

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_file_system_client.py

@@ -19,7 +19,7 @@
 from azure.core.paging import ItemPaged
 from azure.storage.blob import ContainerClient
 from ._shared.base_client import TransportWrapper, StorageAccountHostsMixin, parse_query, parse_connection_str
-from ._serialize import convert_dfs_url_to_blob_url
+from ._serialize import convert_dfs_url_to_blob_url, get_api_version
 from ._list_paths_helper import DeletedPathPropertiesPaged
 from ._models import LocationMode, FileSystemProperties, PublicAccess, DeletedPathProperties, FileProperties, \
     DirectoryProperties
@@ -105,9 +105,12 @@ def __init__(
         # ADLS doesn't support secondary endpoint, make sure it's empty
         self._hosts[LocationMode.SECONDARY] = ""
         self._client = AzureDataLakeStorageRESTAPI(self.url, file_system=file_system_name, pipeline=self._pipeline)
+        api_version = get_api_version(kwargs)
+        self._client._config.version = api_version  # pylint: disable=protected-access
         self._datalake_client_for_blob_operation = AzureDataLakeStorageRESTAPI(self._container_client.url,
                                                                                file_system=file_system_name,
                                                                                pipeline=self._pipeline)
+        self._datalake_client_for_blob_operation._config.version = api_version  # pylint: disable=protected-access
 
     def _format_url(self, hostname):
         file_system_name = self.file_system_name
@@ -839,6 +842,7 @@ def get_directory_client(self, directory  # type: Union[DirectoryProperties, str
         )
         return DataLakeDirectoryClient(self.url, self.file_system_name, directory_name=directory_name,
                                        credential=self._raw_credential,
+                                       api_version=self.api_version,
                                        _configuration=self._config, _pipeline=_pipeline,
                                        _hosts=self._hosts,
                                        require_encryption=self.require_encryption,
@@ -879,6 +883,7 @@ def get_file_client(self, file_path  # type: Union[FileProperties, str]
         )
         return DataLakeFileClient(
             self.url, self.file_system_name, file_path=file_path, credential=self._raw_credential,
+            api_version=self.api_version,
             _hosts=self._hosts, _configuration=self._config, _pipeline=_pipeline,
             require_encryption=self.require_encryption,
             key_encryption_key=self.key_encryption_key,

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_path_client.py

@@ -22,7 +22,8 @@
 from ._models import LocationMode, DirectoryProperties, AccessControlChangeResult, AccessControlChanges, \
     AccessControlChangeCounters, AccessControlChangeFailure
 from ._serialize import convert_dfs_url_to_blob_url, get_mod_conditions, \
-    get_path_http_headers, add_metadata_headers, get_lease_id, get_source_mod_conditions, get_access_conditions
+    get_path_http_headers, add_metadata_headers, get_lease_id, get_source_mod_conditions, get_access_conditions, \
+    get_api_version
 from ._shared.base_client import StorageAccountHostsMixin, parse_query
 from ._shared.response_handlers import return_response_headers, return_headers_and_deserialized
 
@@ -81,13 +82,18 @@ def __init__(
                                          _hosts=datalake_hosts, **kwargs)
         # ADLS doesn't support secondary endpoint, make sure it's empty
         self._hosts[LocationMode.SECONDARY] = ""
+        api_version = get_api_version(kwargs)
+
         self._client = AzureDataLakeStorageRESTAPI(self.url, file_system=file_system_name, path=path_name,
                                                    pipeline=self._pipeline)
+        self._client._config.version = api_version  # pylint: disable=protected-access
+
         self._datalake_client_for_blob_operation = AzureDataLakeStorageRESTAPI(
             self._blob_client.url,
             file_system=file_system_name,
             path=path_name,
             pipeline=self._pipeline)
+        self._datalake_client_for_blob_operation._config.version = api_version  # pylint: disable=protected-access
 
     def __exit__(self, *args):
         self._blob_client.close()

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_serialize.py

@@ -9,6 +9,28 @@
     SourceModifiedAccessConditions, LeaseAccessConditions
 
 
+_SUPPORTED_API_VERSIONS = [
+    '2019-02-02',
+    '2019-07-07',
+    '2019-10-10',
+    '2019-12-12',
+    '2020-02-10',
+    '2020-04-08',
+    '2020-06-12',
+    '2020-08-04',
+    '2020-10-02'
+]
+
+
+def get_api_version(kwargs):
+    # type: (Dict[str, Any], str) -> str
+    api_version = kwargs.get('api_version', None)
+    if api_version and api_version not in _SUPPORTED_API_VERSIONS:
+        versions = '\n'.join(_SUPPORTED_API_VERSIONS)
+        raise ValueError("Unsupported API version '{}'. Please select from:\n{}".format(api_version, versions))
+    return api_version or _SUPPORTED_API_VERSIONS[-1]
+
+
 def convert_dfs_url_to_blob_url(dfs_account_url):
     return dfs_account_url.replace('.dfs.', '.blob.', 1)
 

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_version.py

@@ -4,4 +4,4 @@
 # license information.
 # --------------------------------------------------------------------------
 
-VERSION = "12.5.0b1"
+VERSION = "12.5.0"

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_directory_client_async.py

@@ -506,6 +506,7 @@ def get_file_client(self, file  # type: Union[FileProperties, str]
         )
         return DataLakeFileClient(
             self.url, self.file_system_name, file_path=file_path, credential=self._raw_credential,
+            api_version=self.api_version,
             _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline,
             _location_mode=self._location_mode, require_encryption=self.require_encryption,
             key_encryption_key=self.key_encryption_key,
@@ -545,6 +546,7 @@ def get_sub_directory_client(self, sub_directory  # type: Union[DirectoryPropert
         )
         return DataLakeDirectoryClient(
             self.url, self.file_system_name, directory_name=subdir_path, credential=self._raw_credential,
+            api_version=self.api_version,
             _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline,
             _location_mode=self._location_mode, require_encryption=self.require_encryption,
             key_encryption_key=self.key_encryption_key,

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_service_client_async.py

@@ -10,6 +10,7 @@
 from azure.core.pipeline import AsyncPipeline
 
 from azure.storage.blob.aio import BlobServiceClient
+from .._serialize import get_api_version
 from .._generated.aio import AzureDataLakeStorageRESTAPI
 from .._deserialize import get_datalake_service_properties
 from .._shared.base_client_async import AsyncTransportWrapper, AsyncStorageAccountHostsMixin
@@ -80,6 +81,7 @@ def __init__(
         self._blob_service_client = BlobServiceClient(self._blob_account_url, credential, **kwargs)
         self._blob_service_client._hosts[LocationMode.SECONDARY] = ""  #pylint: disable=protected-access
         self._client = AzureDataLakeStorageRESTAPI(self.url, pipeline=self._pipeline)
+        self._client._config.version = get_api_version(kwargs)  #pylint: disable=protected-access
         self._loop = kwargs.get('loop', None)
 
     async def __aenter__(self):
@@ -337,6 +339,7 @@ def get_file_system_client(self, file_system  # type: Union[FileSystemProperties
             policies=self._pipeline._impl_policies # pylint: disable = protected-access
         )
         return FileSystemClient(self.url, file_system_name, credential=self._raw_credential,
+                                api_version=self.api_version,
                                 _configuration=self._config,
                                 _pipeline=self._pipeline, _hosts=self._hosts,
                                 require_encryption=self.require_encryption, key_encryption_key=self.key_encryption_key,
@@ -385,6 +388,7 @@ def get_directory_client(self, file_system,  # type: Union[FileSystemProperties,
         )
         return DataLakeDirectoryClient(self.url, file_system_name, directory_name=directory_name,
                                        credential=self._raw_credential,
+                                       api_version=self.api_version,
                                        _configuration=self._config, _pipeline=self._pipeline,
                                        _hosts=self._hosts,
                                        require_encryption=self.require_encryption,
@@ -435,6 +439,7 @@ def get_file_client(self, file_system,  # type: Union[FileSystemProperties, str]
         )
         return DataLakeFileClient(
             self.url, file_system_name, file_path=file_path, credential=self._raw_credential,
+            api_version=self.api_version,
             _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline,
             require_encryption=self.require_encryption,
             key_encryption_key=self.key_encryption_key,

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_file_system_client_async.py

@@ -18,6 +18,7 @@
 
 from azure.core.tracing.decorator_async import distributed_trace_async
 from azure.storage.blob.aio import ContainerClient
+from .._serialize import get_api_version
 from .._deserialize import process_storage_error, is_file_path
 from .._generated.models import ListBlobsIncludeItem
 
@@ -98,6 +99,10 @@ def __init__(
         self._datalake_client_for_blob_operation = AzureDataLakeStorageRESTAPI(self._container_client.url,
                                                                                file_system=file_system_name,
                                                                                pipeline=self._pipeline)
+        api_version = get_api_version(kwargs)
+        self._client._config.version = api_version  # pylint: disable=protected-access
+        self._datalake_client_for_blob_operation._config.version = api_version  # pylint: disable=protected-access
+
         self._loop = kwargs.get('loop', None)
 
     async def __aexit__(self, *args):
@@ -787,6 +792,7 @@ def get_directory_client(self, directory  # type: Union[DirectoryProperties, str
         )
         return DataLakeDirectoryClient(self.url, self.file_system_name, directory_name=directory_name,
                                        credential=self._raw_credential,
+                                       api_version=self.api_version,
                                        _configuration=self._config, _pipeline=_pipeline,
                                        _hosts=self._hosts,
                                        require_encryption=self.require_encryption,
@@ -828,6 +834,7 @@ def get_file_client(self, file_path  # type: Union[FileProperties, str]
         )
         return DataLakeFileClient(
             self.url, self.file_system_name, file_path=file_path, credential=self._raw_credential,
+            api_version=self.api_version,
             _hosts=self._hosts, _configuration=self._config, _pipeline=_pipeline,
             require_encryption=self.require_encryption,
             key_encryption_key=self.key_encryption_key,