From 9d3198b87b665adc719c70080d5a90599deb1973 Mon Sep 17 00:00:00 2001 From: Layla Bristol Date: Fri, 19 Apr 2019 17:10:54 -0700 Subject: [PATCH 01/11] Added functions to client for loading and saving schemas to a file. --- bigquery/google/cloud/bigquery/client.py | 60 ++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index 86e131438f32..171224e84320 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -23,6 +23,8 @@ import functools import gzip +import io +import json import os import tempfile import uuid @@ -50,6 +52,7 @@ from google.cloud.bigquery.model import ModelReference from google.cloud.bigquery.query import _QueryResults from google.cloud.bigquery.retry import DEFAULT_RETRY +from google.cloud.bigquery.schema import SchemaField from google.cloud.bigquery.table import _table_arg_to_table from google.cloud.bigquery.table import _table_arg_to_table_ref from google.cloud.bigquery.table import Table @@ -71,6 +74,9 @@ _READ_LESS_THAN_SIZE = ( "Size {:d} was specified but the file-like object only had " "{:d} bytes remaining." ) +_NEED_JSON_FILE_ARGUMENT = ( + "The JSON file argument should be a file object or a file path" +) _NEED_TABLE_ARGUMENT = ( "The table argument should be a table ID string, Table, or TableReference" ) @@ -1929,6 +1935,60 @@ def list_rows( ) return row_iterator + def schema_from_json(self, file_or_path): + """Takes a file object or file path that contains json that describes + a table schema. + + Returns: + List of schema field objects. + """ + file_obj = None + json_data = None + schema_field_list = list() + + if isinstance(file_or_path, io.IOBase): + file_obj = file_or_path + else: + try: + file_obj = open(file_or_path) + except OSError: + raise TypeError(_NEED_JSON_FILE_ARGUMENT) + + try: + json_data = json.load(file_obj) + except JSONDecodeError: + raise TypeError(_NEED_JSON_FILE_ARGUMENT) + + for field in json_data: + schema_field = SchemaField.from_api_repr(field) + schema_field_list.append(schema_field) + + return schema_field_list + + def schema_to_json(self, schema_list, destination): + """Takes a list of schema field objects. + + Serializes the list of schema field objects as json to a file. + + Destination is a file path or a file object. + """ + file_obj = None + json_schema_list = list() + + if isinstance(destination, io.IOBase): + file_obj = destination + else: + try: + file_obj = open(destination, mode="w") + except OSError: + raise TypeError(_NEED_JSON_FILE_ARGUMENT) + + for field in schema_list: + schema_field = field.to_api_repr() + json_schema_list.append(schema_field) + + file_obj.write(json.dumps(json_schema_list, indent=2, sort_keys=True)) + # pylint: disable=unused-argument def _item_to_project(iterator, resource): From 894bb26bad4c5e561e72e283323781929974be96 Mon Sep 17 00:00:00 2001 From: Layla Bristol Date: Fri, 19 Apr 2019 17:12:17 -0700 Subject: [PATCH 02/11] Tests for schema to/from json. --- bigquery/tests/unit/test_client.py | 81 ++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index 46734079a03d..da80579f43cb 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -5161,3 +5161,84 @@ def test__do_multipart_upload_wrong_size(self): with pytest.raises(ValueError): client._do_multipart_upload(file_obj, {}, file_obj_len + 1, None) + + def test__schema_from_json(self): + from google.cloud.bigquery.schema import SchemaField + + file_content = """[ + { + "description": "quarter", + "mode": "REQUIRED", + "name": "qtr", + "type": "STRING" + }, + { + "description": "sales representative", + "mode": "NULLABLE", + "name": "rep", + "type": "STRING" + }, + { + "description": "total sales", + "mode": "NULLABLE", + "name": "sales", + "type": "FLOAT" + } + ]""" + expected = list() + json_data = json.loads(file_content) + + for field in json_data: + schema_field = SchemaField.from_api_repr(field) + expected.append(schema_field) + + client = self._make_client() + mock_file_path = "/mocked/file.json" + + open_patch = mock.patch( + "builtins.open", new=mock.mock_open(read_data=file_content) + ) + with open_patch as _mock_file: + actual = client.schema_from_json(mock_file_path) + _mock_file.assert_called_once_with(mock_file_path) + + assert expected == actual + + def test__schema_to_json(self): + from google.cloud.bigquery.schema import SchemaField + + file_content = """[ + { + "description": "quarter", + "mode": "REQUIRED", + "name": "qtr", + "type": "STRING" + }, + { + "description": "sales representative", + "mode": "NULLABLE", + "name": "rep", + "type": "STRING" + }, + { + "description": "total sales", + "mode": "NULLABLE", + "name": "sales", + "type": "FLOAT" + } + ]""" + schema_list = list() + json_data = json.loads(file_content) + + for field in json_data: + schema_field = SchemaField.from_api_repr(field) + schema_list.append(schema_field) + + client = self._make_client() + mock_file_path = "/mocked/file.json" + + open_patch = mock.patch("builtins.open", mock.mock_open()) + with open_patch as _mock_file: + actual = client.schema_to_json(schema_list, mock_file_path) + _mock_file.assert_called_once_with(mock_file_path, mode="w") + _mock_file().write.assert_called_once_with(file_content) From 136cfcbd9cbc917cb26ec2e88d0032885b9673d1 Mon Sep 17 00:00:00 2001 From: Layla Bristol Date: Tue, 23 Apr 2019 09:55:09 -0700 Subject: [PATCH 03/11] Updated functions to close file and made test changes per feedback. --- bigquery/google/cloud/bigquery/client.py | 57 ++++++------ bigquery/tests/unit/test_client.py | 106 ++++++++++++++++++++--- 2 files changed, 123 insertions(+), 40 deletions(-) diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index 171224e84320..ab1ad7b56c0d 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -1935,6 +1935,22 @@ def list_rows( ) return row_iterator + def _schema_from_json_file_object(self, file): + """Helper function for schema_from_json that takes a + file object that describes a table schema. + + Returns: + List of schema field objects. + """ + schema_field_list = list() + json_data = json.load(file) + + for field in json_data: + schema_field = SchemaField.from_api_repr(field) + schema_field_list.append(schema_field) + + return schema_field_list + def schema_from_json(self, file_or_path): """Takes a file object or file path that contains json that describes a table schema. @@ -1942,52 +1958,37 @@ def schema_from_json(self, file_or_path): Returns: List of schema field objects. """ - file_obj = None - json_data = None - schema_field_list = list() - if isinstance(file_or_path, io.IOBase): - file_obj = file_or_path + return self._schema_from_json_file_object(file_or_path) else: try: - file_obj = open(file_or_path) + with open(file_or_path) as file: + return self._schema_from_json_file_object(file) except OSError: - raise TypeError(_NEED_JSON_FILE_ARGUMENT) - - try: - json_data = json.load(file_obj) - except JSONDecodeError: - raise TypeError(_NEED_JSON_FILE_ARGUMENT) - - for field in json_data: - schema_field = SchemaField.from_api_repr(field) - schema_field_list.append(schema_field) - - return schema_field_list + raise ValueError(_NEED_JSON_FILE_ARGUMENT) def schema_to_json(self, schema_list, destination): """Takes a list of schema field objects. Serializes the list of schema field objects as json to a file. - Destination is a file path or a file object. + Destination is a file path or a file object. """ file_obj = None json_schema_list = list() - if isinstance(destination, io.IOBase): - file_obj = destination - else: - try: - file_obj = open(destination, mode="w") - except OSError: - raise TypeError(_NEED_JSON_FILE_ARGUMENT) - for field in schema_list: schema_field = field.to_api_repr() json_schema_list.append(schema_field) - file_obj.write(json.dumps(json_schema_list, indent=2, sort_keys=True)) + if isinstance(destination, io.IOBase): + destination.write(json.dumps(json_schema_list, indent=2, sort_keys=True)) + else: + try: + with open(destination, mode="w") as file: + file.write(json.dumps(json_schema_list, indent=2, sort_keys=True)) + except OSError: + raise ValueError(_NEED_JSON_FILE_ARGUMENT) # pylint: disable=unused-argument diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index da80579f43cb..a782ae6c381a 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -5162,7 +5162,7 @@ def test__do_multipart_upload_wrong_size(self): with pytest.raises(ValueError): client._do_multipart_upload(file_obj, {}, file_obj_len + 1, None) - def test__schema_from_json(self): + def test_schema_from_json_with_file_path(self): from google.cloud.bigquery.schema import SchemaField file_content = """[ @@ -5185,12 +5185,13 @@ def test__schema_from_json(self): "type": "FLOAT" } ]""" - expected = list() - json_data = json.loads(file_content) - for field in json_data: - schema_field = SchemaField.from_api_repr(field) - expected.append(schema_field) + expected = list() + expected.append(SchemaField("qtr", "STRING", "REQUIRED", "quarter")) + expected.append( + SchemaField("rep", "STRING", "NULLABLE", "sales representative") + ) + expected.append(SchemaField("sales", "FLOAT", "NULLABLE", "total sales")) client = self._make_client() mock_file_path = "/mocked/file.json" @@ -5198,13 +5199,17 @@ def test__schema_from_json(self): open_patch = mock.patch( "builtins.open", new=mock.mock_open(read_data=file_content) ) + with open_patch as _mock_file: actual = client.schema_from_json(mock_file_path) _mock_file.assert_called_once_with(mock_file_path) + # This assert is to make sure __exit__ is called in the context + # manager that opens the file in the function + _mock_file().__exit__.assert_called_once_with(None, None, None) assert expected == actual - def test__schema_to_json(self): + def test_schema_from_json_with_file_object(self): from google.cloud.bigquery.schema import SchemaField file_content = """[ @@ -5227,12 +5232,50 @@ def test__schema_to_json(self): "type": "FLOAT" } ]""" - schema_list = list() - json_data = json.loads(file_content) - for field in json_data: - schema_field = SchemaField.from_api_repr(field) - schema_list.append(schema_field) + expected = list() + expected.append(SchemaField("qtr", "STRING", "REQUIRED", "quarter")) + expected.append( + SchemaField("rep", "STRING", "NULLABLE", "sales representative") + ) + expected.append(SchemaField("sales", "FLOAT", "NULLABLE", "total sales")) + + client = self._make_client() + + fake_file = io.StringIO(file_content) + actual = client.schema_from_json(fake_file) + + assert expected == actual + + def test_schema_to_json_with_file_path(self): + from google.cloud.bigquery.schema import SchemaField + + file_content = """[ + { + "description": "quarter", + "mode": "REQUIRED", + "name": "qtr", + "type": "STRING" + }, + { + "description": "sales representative", + "mode": "NULLABLE", + "name": "rep", + "type": "STRING" + }, + { + "description": "total sales", + "mode": "NULLABLE", + "name": "sales", + "type": "FLOAT" + } +]""" + schema_list = list() + schema_list.append(SchemaField("qtr", "STRING", "REQUIRED", "quarter")) + schema_list.append( + SchemaField("rep", "STRING", "NULLABLE", "sales representative") + ) + schema_list.append(SchemaField("sales", "FLOAT", "NULLABLE", "total sales")) client = self._make_client() mock_file_path = "/mocked/file.json" @@ -5242,3 +5285,42 @@ def test__schema_to_json(self): actual = client.schema_to_json(schema_list, mock_file_path) _mock_file.assert_called_once_with(mock_file_path, mode="w") _mock_file().write.assert_called_once_with(file_content) + # This assert is to make sure __exit__ is called in the context + # manager that opens the file in the function + _mock_file().__exit__.assert_called_once_with(None, None, None) + + def test_schema_to_json_with_file_object(self): + from google.cloud.bigquery.schema import SchemaField + + file_content = """[ + { + "description": "quarter", + "mode": "REQUIRED", + "name": "qtr", + "type": "STRING" + }, + { + "description": "sales representative", + "mode": "NULLABLE", + "name": "rep", + "type": "STRING" + }, + { + "description": "total sales", + "mode": "NULLABLE", + "name": "sales", + "type": "FLOAT" + } +]""" + schema_list = list() + schema_list.append(SchemaField("qtr", "STRING", "REQUIRED", "quarter")) + schema_list.append( + SchemaField("rep", "STRING", "NULLABLE", "sales representative") + ) + schema_list.append(SchemaField("sales", "FLOAT", "NULLABLE", "total sales")) + + fake_file = io.StringIO() + client = self._make_client() + + client.schema_to_json(schema_list, fake_file) + assert file_content == fake_file.getvalue() From 092e121b4eb90e7fd0ba4cf1144225c6a4673517 Mon Sep 17 00:00:00 2001 From: Layla Bristol Date: Tue, 23 Apr 2019 22:07:15 -0700 Subject: [PATCH 04/11] Update with review feedback. --- bigquery/google/cloud/bigquery/client.py | 64 +++++++++---------- bigquery/tests/unit/test_client.py | 80 ++++++++++++------------ 2 files changed, 71 insertions(+), 73 deletions(-) diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index ab1ad7b56c0d..8cac934ab364 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -1935,60 +1935,56 @@ def list_rows( ) return row_iterator - def _schema_from_json_file_object(self, file): + def _schema_from_json_file_object(self, file_obj): """Helper function for schema_from_json that takes a - file object that describes a table schema. + file object that describes a table schema. - Returns: - List of schema field objects. + Returns: + List of schema field objects. """ - schema_field_list = list() - json_data = json.load(file) + json_data = json.load(file_obj) + return [SchemaField.from_api_repr(f) for f in json_data] - for field in json_data: - schema_field = SchemaField.from_api_repr(field) - schema_field_list.append(schema_field) - - return schema_field_list + def _schema_to_json_file_object(self, schema_list, file_obj): + """Helper function for schema_to_json that takes a schema list and file + object and writes the schema list to the file object with json.dump + """ + json.dump(schema_list, file_obj, indent=2, sort_keys=True) def schema_from_json(self, file_or_path): """Takes a file object or file path that contains json that describes - a table schema. + a table schema. - Returns: - List of schema field objects. + Returns: + List of schema field objects. """ if isinstance(file_or_path, io.IOBase): return self._schema_from_json_file_object(file_or_path) - else: - try: - with open(file_or_path) as file: - return self._schema_from_json_file_object(file) - except OSError: - raise ValueError(_NEED_JSON_FILE_ARGUMENT) + + try: + with open(file_or_path) as file_obj: + return self._schema_from_json_file_object(file_obj) + except OSError: + raise ValueError(_NEED_JSON_FILE_ARGUMENT) def schema_to_json(self, schema_list, destination): """Takes a list of schema field objects. - Serializes the list of schema field objects as json to a file. + Serializes the list of schema field objects as json to a file. - Destination is a file path or a file object. + Destination is a file path or a file object. """ file_obj = None - json_schema_list = list() - - for field in schema_list: - schema_field = field.to_api_repr() - json_schema_list.append(schema_field) + json_schema_list = [f.to_api_repr() for f in schema_list] if isinstance(destination, io.IOBase): - destination.write(json.dumps(json_schema_list, indent=2, sort_keys=True)) - else: - try: - with open(destination, mode="w") as file: - file.write(json.dumps(json_schema_list, indent=2, sort_keys=True)) - except OSError: - raise ValueError(_NEED_JSON_FILE_ARGUMENT) + return self._schema_to_json_file_object(json_schema_list, destination) + + try: + with open(destination, mode="w") as file_obj: + return self._schema_to_json_file_object(json_schema_list, file_obj) + except OSError: + raise ValueError(_NEED_JSON_FILE_ARGUMENT) # pylint: disable=unused-argument diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index a782ae6c381a..6248ac8bf1b5 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -5205,7 +5205,7 @@ def test_schema_from_json_with_file_path(self): _mock_file.assert_called_once_with(mock_file_path) # This assert is to make sure __exit__ is called in the context # manager that opens the file in the function - _mock_file().__exit__.assert_called_once_with(None, None, None) + _mock_file().__exit__.assert_called_once() assert expected == actual @@ -5250,44 +5250,47 @@ def test_schema_from_json_with_file_object(self): def test_schema_to_json_with_file_path(self): from google.cloud.bigquery.schema import SchemaField - file_content = """[ - { - "description": "quarter", - "mode": "REQUIRED", - "name": "qtr", - "type": "STRING" - }, - { - "description": "sales representative", - "mode": "NULLABLE", - "name": "rep", - "type": "STRING" - }, - { - "description": "total sales", - "mode": "NULLABLE", - "name": "sales", - "type": "FLOAT" - } -]""" - schema_list = list() - schema_list.append(SchemaField("qtr", "STRING", "REQUIRED", "quarter")) - schema_list.append( - SchemaField("rep", "STRING", "NULLABLE", "sales representative") - ) - schema_list.append(SchemaField("sales", "FLOAT", "NULLABLE", "total sales")) + file_content = [ + { + "description": "quarter", + "mode": "REQUIRED", + "name": "qtr", + "type": "STRING", + }, + { + "description": "sales representative", + "mode": "NULLABLE", + "name": "rep", + "type": "STRING", + }, + { + "description": "total sales", + "mode": "NULLABLE", + "name": "sales", + "type": "FLOAT", + }, + ] + + schema_list = [ + SchemaField("qtr", "STRING", "REQUIRED", "quarter"), + SchemaField("rep", "STRING", "NULLABLE", "sales representative"), + SchemaField("sales", "FLOAT", "NULLABLE", "total sales"), + ] client = self._make_client() mock_file_path = "/mocked/file.json" open_patch = mock.patch("builtins.open", mock.mock_open()) with open_patch as _mock_file: - actual = client.schema_to_json(schema_list, mock_file_path) - _mock_file.assert_called_once_with(mock_file_path, mode="w") - _mock_file().write.assert_called_once_with(file_content) - # This assert is to make sure __exit__ is called in the context - # manager that opens the file in the function - _mock_file().__exit__.assert_called_once_with(None, None, None) + with mock.patch("json.dump") as _mock_dump: + client.schema_to_json(schema_list, mock_file_path) + _mock_file.assert_called_once_with(mock_file_path, mode="w") + # This assert is to make sure __exit__ is called in the context + # manager that opens the file in the function + _mock_file().__exit__.assert_called_once() + _mock_dump.assert_called_with( + file_content, _mock_file.return_value, indent=2, sort_keys=True + ) def test_schema_to_json_with_file_object(self): from google.cloud.bigquery.schema import SchemaField @@ -5312,12 +5315,11 @@ def test_schema_to_json_with_file_object(self): "type": "FLOAT" } ]""" - schema_list = list() - schema_list.append(SchemaField("qtr", "STRING", "REQUIRED", "quarter")) - schema_list.append( - SchemaField("rep", "STRING", "NULLABLE", "sales representative") - ) - schema_list.append(SchemaField("sales", "FLOAT", "NULLABLE", "total sales")) + schema_list = [ + SchemaField("qtr", "STRING", "REQUIRED", "quarter"), + SchemaField("rep", "STRING", "NULLABLE", "sales representative"), + SchemaField("sales", "FLOAT", "NULLABLE", "total sales"), + ] fake_file = io.StringIO() client = self._make_client() From 79e965ba36a26093280ad36475832699d9b6894b Mon Sep 17 00:00:00 2001 From: Layla Bristol Date: Tue, 23 Apr 2019 22:40:45 -0700 Subject: [PATCH 05/11] Removed unneeded variable --- bigquery/google/cloud/bigquery/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index 8cac934ab364..9c7f3b33554d 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -1974,7 +1974,6 @@ def schema_to_json(self, schema_list, destination): Destination is a file path or a file object. """ - file_obj = None json_schema_list = [f.to_api_repr() for f in schema_list] if isinstance(destination, io.IOBase): From 22bf1ab469f94eff13ec27b01224d566ba97a532 Mon Sep 17 00:00:00 2001 From: Layla Bristol Date: Tue, 23 Apr 2019 22:50:57 -0700 Subject: [PATCH 06/11] Removed append in test --- bigquery/tests/unit/test_client.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index 6248ac8bf1b5..50c50bbf00c6 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -5186,12 +5186,11 @@ def test_schema_from_json_with_file_path(self): } ]""" - expected = list() - expected.append(SchemaField("qtr", "STRING", "REQUIRED", "quarter")) - expected.append( - SchemaField("rep", "STRING", "NULLABLE", "sales representative") - ) - expected.append(SchemaField("sales", "FLOAT", "NULLABLE", "total sales")) + expected = [ + SchemaField("qtr", "STRING", "REQUIRED", "quarter"), + SchemaField("rep", "STRING", "NULLABLE", "sales representative"), + SchemaField("sales", "FLOAT", "NULLABLE", "total sales"), + ] client = self._make_client() mock_file_path = "/mocked/file.json" @@ -5233,12 +5232,11 @@ def test_schema_from_json_with_file_object(self): } ]""" - expected = list() - expected.append(SchemaField("qtr", "STRING", "REQUIRED", "quarter")) - expected.append( - SchemaField("rep", "STRING", "NULLABLE", "sales representative") - ) - expected.append(SchemaField("sales", "FLOAT", "NULLABLE", "total sales")) + expected = [ + SchemaField("qtr", "STRING", "REQUIRED", "quarter"), + SchemaField("rep", "STRING", "NULLABLE", "sales representative"), + SchemaField("sales", "FLOAT", "NULLABLE", "total sales"), + ] client = self._make_client() From bb2ca7903d3ed23fd25bb2b65cefa309a7100d0b Mon Sep 17 00:00:00 2001 From: Layla Bristol Date: Wed, 24 Apr 2019 10:28:08 -0700 Subject: [PATCH 07/11] Made changes based on feedback --- bigquery/google/cloud/bigquery/client.py | 2 +- bigquery/tests/unit/test_client.py | 19 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index 9c7f3b33554d..06e5459bc854 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -1943,7 +1943,7 @@ def _schema_from_json_file_object(self, file_obj): List of schema field objects. """ json_data = json.load(file_obj) - return [SchemaField.from_api_repr(f) for f in json_data] + return [SchemaField.from_api_repr(field) for field in json_data] def _schema_to_json_file_object(self, schema_list, file_obj): """Helper function for schema_to_json that takes a schema list and file diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index 50c50bbf00c6..6309e77919ea 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -5279,16 +5279,15 @@ def test_schema_to_json_with_file_path(self): mock_file_path = "/mocked/file.json" open_patch = mock.patch("builtins.open", mock.mock_open()) - with open_patch as _mock_file: - with mock.patch("json.dump") as _mock_dump: - client.schema_to_json(schema_list, mock_file_path) - _mock_file.assert_called_once_with(mock_file_path, mode="w") - # This assert is to make sure __exit__ is called in the context - # manager that opens the file in the function - _mock_file().__exit__.assert_called_once() - _mock_dump.assert_called_with( - file_content, _mock_file.return_value, indent=2, sort_keys=True - ) + with open_patch as mock_file, mock.patch("json.dump") as mock_dump: + client.schema_to_json(schema_list, mock_file_path) + mock_file.assert_called_once_with(mock_file_path, mode="w") + # This assert is to make sure __exit__ is called in the context + # manager that opens the file in the function + mock_file().__exit__.assert_called_once() + mock_dump.assert_called_with( + file_content, mock_file.return_value, indent=2, sort_keys=True + ) def test_schema_to_json_with_file_object(self): from google.cloud.bigquery.schema import SchemaField From 54518524dad0bbde6a51b6f0abd694ef67099c22 Mon Sep 17 00:00:00 2001 From: Layla Bristol Date: Wed, 24 Apr 2019 10:36:43 -0700 Subject: [PATCH 08/11] Added change to test per feedback. --- bigquery/tests/unit/test_client.py | 43 +++++++++++++++--------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index 6309e77919ea..47d910eede35 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -5292,26 +5292,27 @@ def test_schema_to_json_with_file_path(self): def test_schema_to_json_with_file_object(self): from google.cloud.bigquery.schema import SchemaField - file_content = """[ - { - "description": "quarter", - "mode": "REQUIRED", - "name": "qtr", - "type": "STRING" - }, - { - "description": "sales representative", - "mode": "NULLABLE", - "name": "rep", - "type": "STRING" - }, - { - "description": "total sales", - "mode": "NULLABLE", - "name": "sales", - "type": "FLOAT" - } -]""" + file_content = [ + { + "description": "quarter", + "mode": "REQUIRED", + "name": "qtr", + "type": "STRING", + }, + { + "description": "sales representative", + "mode": "NULLABLE", + "name": "rep", + "type": "STRING", + }, + { + "description": "total sales", + "mode": "NULLABLE", + "name": "sales", + "type": "FLOAT", + }, + ] + schema_list = [ SchemaField("qtr", "STRING", "REQUIRED", "quarter"), SchemaField("rep", "STRING", "NULLABLE", "sales representative"), @@ -5322,4 +5323,4 @@ def test_schema_to_json_with_file_object(self): client = self._make_client() client.schema_to_json(schema_list, fake_file) - assert file_content == fake_file.getvalue() + assert file_content == json.loads(fake_file.getvalue()) From f4ae0c18e0ba82633e90fc9263debb6f85696795 Mon Sep 17 00:00:00 2001 From: Layla Bristol Date: Wed, 24 Apr 2019 13:03:16 -0700 Subject: [PATCH 09/11] removed try blocks per suggestion --- bigquery/google/cloud/bigquery/client.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index 06e5459bc854..bb6a375975f2 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -74,9 +74,6 @@ _READ_LESS_THAN_SIZE = ( "Size {:d} was specified but the file-like object only had " "{:d} bytes remaining." ) -_NEED_JSON_FILE_ARGUMENT = ( - "The JSON file argument should be a file object or a file path" -) _NEED_TABLE_ARGUMENT = ( "The table argument should be a table ID string, Table, or TableReference" ) @@ -1961,11 +1958,8 @@ def schema_from_json(self, file_or_path): if isinstance(file_or_path, io.IOBase): return self._schema_from_json_file_object(file_or_path) - try: - with open(file_or_path) as file_obj: - return self._schema_from_json_file_object(file_obj) - except OSError: - raise ValueError(_NEED_JSON_FILE_ARGUMENT) + with open(file_or_path) as file_obj: + return self._schema_from_json_file_object(file_obj) def schema_to_json(self, schema_list, destination): """Takes a list of schema field objects. @@ -1979,11 +1973,8 @@ def schema_to_json(self, schema_list, destination): if isinstance(destination, io.IOBase): return self._schema_to_json_file_object(json_schema_list, destination) - try: - with open(destination, mode="w") as file_obj: - return self._schema_to_json_file_object(json_schema_list, file_obj) - except OSError: - raise ValueError(_NEED_JSON_FILE_ARGUMENT) + with open(destination, mode="w") as file_obj: + return self._schema_to_json_file_object(json_schema_list, file_obj) # pylint: disable=unused-argument From 57910495b353eb9ddf78e309bfa46f17e2f19b22 Mon Sep 17 00:00:00 2001 From: Layla Bristol Date: Wed, 24 Apr 2019 15:21:32 -0700 Subject: [PATCH 10/11] Updated test to pass python 2 version --- bigquery/tests/unit/test_client.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index 47d910eede35..4dbe5d22a5da 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -5278,7 +5278,11 @@ def test_schema_to_json_with_file_path(self): client = self._make_client() mock_file_path = "/mocked/file.json" - open_patch = mock.patch("builtins.open", mock.mock_open()) + if six.PY2: + open_patch = mock.patch("__builtins__.open", mock.mock_open()) + else: + open_patch = mock.patch("builtins.open", mock.mock_open()) + with open_patch as mock_file, mock.patch("json.dump") as mock_dump: client.schema_to_json(schema_list, mock_file_path) mock_file.assert_called_once_with(mock_file_path, mode="w") From fa331c42e7f2e330c7f7d974bddbdae00384881c Mon Sep 17 00:00:00 2001 From: Layla Bristol Date: Wed, 24 Apr 2019 15:46:00 -0700 Subject: [PATCH 11/11] fixed tests to work in python 2 --- bigquery/tests/unit/test_client.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index 4dbe5d22a5da..45e80f1a37a3 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -5195,9 +5195,14 @@ def test_schema_from_json_with_file_path(self): client = self._make_client() mock_file_path = "/mocked/file.json" - open_patch = mock.patch( - "builtins.open", new=mock.mock_open(read_data=file_content) - ) + if six.PY2: + open_patch = mock.patch( + "__builtin__.open", mock.mock_open(read_data=file_content) + ) + else: + open_patch = mock.patch( + "builtins.open", new=mock.mock_open(read_data=file_content) + ) with open_patch as _mock_file: actual = client.schema_from_json(mock_file_path) @@ -5240,7 +5245,11 @@ def test_schema_from_json_with_file_object(self): client = self._make_client() - fake_file = io.StringIO(file_content) + if six.PY2: + fake_file = io.BytesIO(file_content) + else: + fake_file = io.StringIO(file_content) + actual = client.schema_from_json(fake_file) assert expected == actual @@ -5279,7 +5288,7 @@ def test_schema_to_json_with_file_path(self): mock_file_path = "/mocked/file.json" if six.PY2: - open_patch = mock.patch("__builtins__.open", mock.mock_open()) + open_patch = mock.patch("__builtin__.open", mock.mock_open()) else: open_patch = mock.patch("builtins.open", mock.mock_open()) @@ -5323,7 +5332,11 @@ def test_schema_to_json_with_file_object(self): SchemaField("sales", "FLOAT", "NULLABLE", "total sales"), ] - fake_file = io.StringIO() + if six.PY2: + fake_file = io.BytesIO() + else: + fake_file = io.StringIO() + client = self._make_client() client.schema_to_json(schema_list, fake_file)