From 15ac564f39c99d86f439f1f6641c3e99427aa15e Mon Sep 17 00:00:00 2001 From: xiafu Date: Tue, 30 Jun 2020 18:29:49 -0700 Subject: [PATCH] [Blob][QQ]Default output_format to input_format --- .../azure/storage/blob/_blob_client.py | 2 + ...only_input_json_serialization_setting.yaml | 219 ++++++++++++++++++ .../tests/test_quick_query.py | 37 +++ 3 files changed, 258 insertions(+) create mode 100644 sdk/storage/azure-storage-blob/tests/recordings/test_quick_query.test_quick_query_with_only_input_json_serialization_setting.yaml diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 2517a8559b72..5add1a0e97bb 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -657,6 +657,8 @@ def _quick_query_options(self, query_expression, delimiter = output_format.lineterminator except AttributeError: delimiter = output_format.delimiter + else: + output_format = input_format query_request = QueryRequest( expression=query_expression, input_serialization=serialize_query_format(input_format), diff --git a/sdk/storage/azure-storage-blob/tests/recordings/test_quick_query.test_quick_query_with_only_input_json_serialization_setting.yaml b/sdk/storage/azure-storage-blob/tests/recordings/test_quick_query.test_quick_query_with_only_input_json_serialization_setting.yaml new file mode 100644 index 000000000000..017e53a7f903 --- /dev/null +++ b/sdk/storage/azure-storage-blob/tests/recordings/test_quick_query.test_quick_query_with_only_input_json_serialization_setting.yaml @@ -0,0 +1,219 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - azsdk-python-storage-blob/12.3.3 Python/3.7.3 (Windows-10-10.0.18362-SP0) + x-ms-date: + - Wed, 01 Jul 2020 01:25:22 GMT + x-ms-version: + - '2019-12-12' + method: PUT + uri: https://storagename.blob.core.windows.net/utqqcontainercb261fef?restype=container + response: + body: + string: '' + headers: + content-length: + - '0' + date: + - Wed, 01 Jul 2020 01:25:22 GMT + etag: + - '"0x8D81D5D9FFCC701"' + last-modified: + - Wed, 01 Jul 2020 01:25:22 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-version: + - '2019-12-12' + status: + code: 201 + message: Created +- request: + body: '{"name": "owner", "id": 1}{"name2": "owner2"}{"name": "owner", "id": 1}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '71' + Content-Type: + - application/octet-stream + User-Agent: + - azsdk-python-storage-blob/12.3.3 Python/3.7.3 (Windows-10-10.0.18362-SP0) + x-ms-blob-type: + - BlockBlob + x-ms-date: + - Wed, 01 Jul 2020 01:25:22 GMT + x-ms-tags: + - '' + x-ms-version: + - '2019-12-12' + method: PUT + uri: https://storagename.blob.core.windows.net/utqqcontainercb261fef/csvfilecb261fef + response: + body: + string: '' + headers: + content-length: + - '0' + content-md5: + - 5tYmuyKFS9mJzdfHlAFncQ== + date: + - Wed, 01 Jul 2020 01:25:22 GMT + etag: + - '"0x8D81D5DA010ECF2"' + last-modified: + - Wed, 01 Jul 2020 01:25:22 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-content-crc64: + - EFBi2aad5IE= + x-ms-request-server-encrypted: + - 'true' + x-ms-version: + - '2019-12-12' + status: + code: 201 + message: Created +- request: + body: ' + + SQLSELECT name from BlobStoragejson + + json + + ' + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '465' + Content-Type: + - application/xml; charset=utf-8 + User-Agent: + - azsdk-python-storage-blob/12.3.3 Python/3.7.3 (Windows-10-10.0.18362-SP0) + x-ms-date: + - Wed, 01 Jul 2020 01:25:22 GMT + x-ms-version: + - '2019-12-12' + method: POST + uri: https://storagename.blob.core.windows.net/utqqcontainercb261fef/csvfilecb261fef?comp=query + response: + body: + string: !!binary | + T2JqAQIWYXZyby5zY2hlbWG+HlsKICB7CiAgICAidHlwZSI6ICJyZWNvcmQiLAogICAgIm5hbWUi + OiAiY29tLm1pY3Jvc29mdC5henVyZS5zdG9yYWdlLnF1ZXJ5QmxvYkNvbnRlbnRzLnJlc3VsdERh + dGEiLAogICAgImRvYyI6ICJIb2xkcyByZXN1bHQgZGF0YSBpbiB0aGUgZm9ybWF0IHNwZWNpZmll + ZCBmb3IgdGhpcyBxdWVyeSAoQ1NWLCBKU09OLCBldGMuKS4iLAogICAgImZpZWxkcyI6IFsKICAg + ICAgewogICAgICAgICJuYW1lIjogImRhdGEiLAogICAgICAgICJ0eXBlIjogImJ5dGVzIgogICAg + ICB9CiAgICBdCiAgfSwKICB7CiAgICAidHlwZSI6ICJyZWNvcmQiLAogICAgIm5hbWUiOiAiY29t + Lm1pY3Jvc29mdC5henVyZS5zdG9yYWdlLnF1ZXJ5QmxvYkNvbnRlbnRzLmVycm9yIiwKICAgICJk + b2MiOiAiQW4gZXJyb3IgdGhhdCBvY2N1cnJlZCB3aGlsZSBwcm9jZXNzaW5nIHRoZSBxdWVyeS4i + LAogICAgImZpZWxkcyI6IFsKICAgICAgewogICAgICAgICJuYW1lIjogImZhdGFsIiwKICAgICAg + ICAidHlwZSI6ICJib29sZWFuIiwKICAgICAgICAiZG9jIjogIklmIHRydWUsIHRoaXMgZXJyb3Ig + cHJldmVudHMgZnVydGhlciBxdWVyeSBwcm9jZXNzaW5nLiAgTW9yZSByZXN1bHQgZGF0YSBtYXkg + YmUgcmV0dXJuZWQsIGJ1dCB0aGVyZSBpcyBubyBndWFyYW50ZWUgdGhhdCBhbGwgb2YgdGhlIG9y + aWdpbmFsIGRhdGEgd2lsbCBiZSBwcm9jZXNzZWQuICBJZiBmYWxzZSwgdGhpcyBlcnJvciBkb2Vz + IG5vdCBwcmV2ZW50IGZ1cnRoZXIgcXVlcnkgcHJvY2Vzc2luZy4iCiAgICAgIH0sCiAgICAgIHsK + ICAgICAgICAibmFtZSI6ICJuYW1lIiwKICAgICAgICAidHlwZSI6ICJzdHJpbmciLAogICAgICAg + ICJkb2MiOiAiVGhlIG5hbWUgb2YgdGhlIGVycm9yIgogICAgICB9LAogICAgICB7CiAgICAgICAg + Im5hbWUiOiAiZGVzY3JpcHRpb24iLAogICAgICAgICJ0eXBlIjogInN0cmluZyIsCiAgICAgICAg + ImRvYyI6ICJBIGRlc2NyaXB0aW9uIG9mIHRoZSBlcnJvciIKICAgICAgfSwKICAgICAgewogICAg + ICAgICJuYW1lIjogInBvc2l0aW9uIiwKICAgICAgICAidHlwZSI6ICJsb25nIiwKICAgICAgICAi + ZG9jIjogIlRoZSBibG9iIG9mZnNldCBhdCB3aGljaCB0aGUgZXJyb3Igb2NjdXJyZWQiCiAgICAg + IH0KICAgIF0KICB9LAogIHsKICAgICJ0eXBlIjogInJlY29yZCIsCiAgICAibmFtZSI6ICJjb20u + bWljcm9zb2Z0LmF6dXJlLnN0b3JhZ2UucXVlcnlCbG9iQ29udGVudHMucHJvZ3Jlc3MiLAogICAg + ImRvYyI6ICJJbmZvcm1hdGlvbiBhYm91dCB0aGUgcHJvZ3Jlc3Mgb2YgdGhlIHF1ZXJ5IiwKICAg + ICJmaWVsZHMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJieXRlc1NjYW5uZWQiLAogICAg + ICAgICJ0eXBlIjogImxvbmciLAogICAgICAgICJkb2MiOiAiVGhlIG51bWJlciBvZiBieXRlcyB0 + aGF0IGhhdmUgYmVlbiBzY2FubmVkIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAi + dG90YWxCeXRlcyIsCiAgICAgICAgInR5cGUiOiAibG9uZyIsCiAgICAgICAgImRvYyI6ICJUaGUg + dG90YWwgbnVtYmVyIG9mIGJ5dGVzIHRvIGJlIHNjYW5uZWQgaW4gdGhpcyBxdWVyeSIKICAgICAg + fQogICAgXQogIH0sCiAgewogICAgInR5cGUiOiAicmVjb3JkIiwKICAgICJuYW1lIjogImNvbS5t + aWNyb3NvZnQuYXp1cmUuc3RvcmFnZS5xdWVyeUJsb2JDb250ZW50cy5lbmQiLAogICAgImRvYyI6 + ICJTZW50IGFzIHRoZSBmaW5hbCBtZXNzYWdlIG9mIHRoZSByZXNwb25zZSwgaW5kaWNhdGluZyB0 + aGF0IGFsbCByZXN1bHRzIGhhdmUgYmVlbiBzZW50LiIsCiAgICAiZmllbGRzIjogWwogICAgICB7 + CiAgICAgICAgIm5hbWUiOiAidG90YWxCeXRlcyIsCiAgICAgICAgInR5cGUiOiAibG9uZyIsCiAg + ICAgICAgImRvYyI6ICJUaGUgdG90YWwgbnVtYmVyIG9mIGJ5dGVzIHRvIGJlIHNjYW5uZWQgaW4g + dGhpcyBxdWVyeSIKICAgICAgfQogICAgXQogIH0KXQoAMXp9EeYLOEaqWbd+uR5gOAJOAEp7Im5h + bWUiOiJvd25lciJ9Cnt9CnsibmFtZSI6Im93bmVyIn0KMXp9EeYLOEaqWbd+uR5gOAIKBI4BjgEx + en0R5gs4RqpZt365HmA4AgYGjgExen0R5gs4RqpZt365HmA4 + headers: + accept-ranges: + - bytes + content-type: + - avro/binary + date: + - Wed, 01 Jul 2020 01:25:38 GMT + etag: + - '"0x8D81D5DA010ECF2"' + last-modified: + - Wed, 01 Jul 2020 01:25:22 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + transfer-encoding: + - chunked + x-ms-blob-type: + - BlockBlob + x-ms-creation-time: + - Wed, 01 Jul 2020 01:25:22 GMT + x-ms-lease-state: + - available + x-ms-lease-status: + - unlocked + x-ms-version: + - '2019-12-12' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - azsdk-python-storage-blob/12.3.3 Python/3.7.3 (Windows-10-10.0.18362-SP0) + x-ms-date: + - Wed, 01 Jul 2020 01:25:39 GMT + x-ms-version: + - '2019-12-12' + method: DELETE + uri: https://storagename.blob.core.windows.net/utqqcontainercb261fef?restype=container + response: + body: + string: '' + headers: + content-length: + - '0' + date: + - Wed, 01 Jul 2020 01:25:39 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-version: + - '2019-12-12' + status: + code: 202 + message: Accepted +version: 1 diff --git a/sdk/storage/azure-storage-blob/tests/test_quick_query.py b/sdk/storage/azure-storage-blob/tests/test_quick_query.py index 615a8d1a835c..4a6d510cf320 100644 --- a/sdk/storage/azure-storage-blob/tests/test_quick_query.py +++ b/sdk/storage/azure-storage-blob/tests/test_quick_query.py @@ -838,4 +838,41 @@ def on_error(error): self.assertEqual(listdata, [b'{"name":"owner"}',b'{}',b'{"name":"owner"}', b'']) self._teardown(bsc) + @GlobalStorageAccountPreparer() + def test_quick_query_with_only_input_json_serialization_setting(self, resource_group, location, storage_account, + storage_account_key): + # Arrange + bsc = BlobServiceClient( + self.account_url(storage_account, "blob"), + credential=storage_account_key) + self._setup(bsc) + + data1 = b'{\"name\": \"owner\", \"id\": 1}' + data2 = b'{\"name2\": \"owner2\"}' + data = data1 + data2 + data1 + + # upload the json file + blob_name = self._get_blob_reference() + blob_client = bsc.get_blob_client(self.container_name, blob_name) + blob_client.upload_blob(data, overwrite=True) + + errors = [] + def on_error(error): + errors.append(error) + + input_format = DelimitedJSON(delimiter='\n') + output_format = None + + resp = blob_client.query_blob( + "SELECT name from BlobStorage", + on_error=on_error, + blob_format=input_format, + output_format=output_format) + query_result = resp.readall() + + self.assertEqual(len(errors), 0) + self.assertEqual(resp._size, len(data)) + self.assertEqual(query_result, b'{"name":"owner"}\n{}\n{"name":"owner"}\n') + self._teardown(bsc) + # ------------------------------------------------------------------------------