From a95a3c89f5530834c91d01a348f711f2e484bc5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Ignacio=20Ria=C3=B1o=20Chico?= Date: Tue, 3 Oct 2023 22:42:55 +0200 Subject: [PATCH 1/5] feat: add support for Dataset.isCaseInsensitive This commit creates a property named is_case_insensitive (in dataset.py) that allows the usage of the isCaseSensitive field in the Dataset REST API. Fixes: https://github.com/googleapis/python-bigquery/issues/1670 --- google/cloud/bigquery/dataset.py | 20 +++++++++++ tests/system/test_client.py | 60 ++++++++++++++++++++++++++++++-- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index a9c1cd884..6d42b3d23 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -527,6 +527,7 @@ class Dataset(object): "default_table_expiration_ms": "defaultTableExpirationMs", "friendly_name": "friendlyName", "default_encryption_configuration": "defaultEncryptionConfiguration", + "is_case_insensitive": "isCaseInsensitive", "storage_billing_model": "storageBillingModel", } @@ -764,6 +765,25 @@ def default_encryption_configuration(self, value): api_repr = value.to_api_repr() self._properties["defaultEncryptionConfiguration"] = api_repr + @property + def is_case_insensitive(self): + """Optional[bool]: TRUE if the dataset and its table names are case-insensitive, otherwise FALSE. + By default, this is FALSE, which means the dataset and its table names are case-sensitive. + This field does not affect routine references. + + Raises: + ValueError: for invalid value types. + """ + return self._properties.get("isCaseInsensitive") or False + + @is_case_insensitive.setter + def is_case_insensitive(self, value): + if not isinstance(value, bool) and value is not None: + raise ValueError("Pass a boolean value, or None") + if value is None: + value = False + self._properties["isCaseInsensitive"] = value + @property def storage_billing_model(self): """Union[str, None]: StorageBillingModel of the dataset as set by the user diff --git a/tests/system/test_client.py b/tests/system/test_client.py index 8fd532f4c..511ba4ede 100644 --- a/tests/system/test_client.py +++ b/tests/system/test_client.py @@ -237,6 +237,17 @@ def test_create_dataset(self): self.assertTrue(_dataset_exists(dataset)) self.assertEqual(dataset.dataset_id, DATASET_ID) self.assertEqual(dataset.project, Config.CLIENT.project) + self.assertIs(dataset.is_case_insensitive, False) + + def test_create_dataset_case_sensitive(self): + DATASET_ID = _make_dataset_id("create_cs_dataset") + dataset = self.temp_dataset(DATASET_ID, is_case_insensitive=False) + self.assertIs(dataset.is_case_insensitive, False) + + def test_create_dataset_case_insensitive(self): + DATASET_ID = _make_dataset_id("create_ci_dataset") + dataset = self.temp_dataset(DATASET_ID, is_case_insensitive=True) + self.assertIs(dataset.is_case_insensitive, True) def test_get_dataset(self): dataset_id = _make_dataset_id("get_dataset") @@ -271,16 +282,19 @@ def test_update_dataset(self): self.assertIsNone(dataset.friendly_name) self.assertIsNone(dataset.description) self.assertEqual(dataset.labels, {}) + self.assertIs(dataset.is_case_insensitive, False) dataset.friendly_name = "Friendly" dataset.description = "Description" dataset.labels = {"priority": "high", "color": "blue"} + dataset.is_case_insensitive = True ds2 = Config.CLIENT.update_dataset( - dataset, ("friendly_name", "description", "labels") + dataset, ("friendly_name", "description", "labels", "is_case_insensitive") ) self.assertEqual(ds2.friendly_name, "Friendly") self.assertEqual(ds2.description, "Description") self.assertEqual(ds2.labels, {"priority": "high", "color": "blue"}) + self.assertIs(ds2.is_case_insensitive, True) ds2.labels = { "color": "green", # change @@ -335,6 +349,46 @@ def test_create_table(self): self.assertTrue(_table_exists(table)) self.assertEqual(table.table_id, table_id) + def test_create_tables_in_case_insensitive_dataset(self): + ci_dataset = self.temp_dataset( + _make_dataset_id("create_table"), is_case_insensitive=True + ) + table_arg = Table(ci_dataset.table("test_table2"), schema=SCHEMA) + tablemc_arg = Table(ci_dataset.table("Test_taBLe2")) + + table = helpers.retry_403(Config.CLIENT.create_table)(table_arg) + self.to_delete.insert(0, table) + + self.assertTrue(_table_exists(table_arg)) + self.assertTrue(_table_exists(tablemc_arg)) + self.assertIs(ci_dataset.is_case_insensitive, True) + + def test_create_tables_in_case_sensitive_dataset(self): + ci_dataset = self.temp_dataset( + _make_dataset_id("create_table"), is_case_insensitive=False + ) + table_arg = Table(ci_dataset.table("test_table3"), schema=SCHEMA) + tablemc_arg = Table(ci_dataset.table("Test_taBLe3")) + + table = helpers.retry_403(Config.CLIENT.create_table)(table_arg) + self.to_delete.insert(0, table) + + self.assertTrue(_table_exists(table_arg)) + self.assertFalse(_table_exists(tablemc_arg)) + self.assertIs(ci_dataset.is_case_insensitive, False) + + def test_create_tables_in_default_sensitivity_dataset(self): + dataset = self.temp_dataset(_make_dataset_id("create_table")) + table_arg = Table(dataset.table("test_table4"), schema=SCHEMA) + tablemc_arg = Table(dataset.table("Test_taBLe4")) + + table = helpers.retry_403(Config.CLIENT.create_table)(table_arg) + self.to_delete.insert(0, table) + + self.assertTrue(_table_exists(table_arg)) + self.assertFalse(_table_exists(tablemc_arg)) + self.assertIs(dataset.is_case_insensitive, False) + def test_create_table_with_real_custom_policy(self): from google.cloud.bigquery.schema import PolicyTagList @@ -2286,12 +2340,14 @@ def test_nested_table_to_arrow(self): self.assertTrue(pyarrow.types.is_list(record_col[1].type)) self.assertTrue(pyarrow.types.is_int64(record_col[1].type.value_type)) - def temp_dataset(self, dataset_id, location=None): + def temp_dataset(self, dataset_id, location=None, is_case_insensitive=None): project = Config.CLIENT.project dataset_ref = bigquery.DatasetReference(project, dataset_id) dataset = Dataset(dataset_ref) if location: dataset.location = location + if is_case_insensitive is not None: + dataset.is_case_insensitive = is_case_insensitive dataset = helpers.retry_403(Config.CLIENT.create_dataset)(dataset) self.to_delete.append(dataset) return dataset From a47a555db01d9f28b9d14b303a138a858b7333cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Ignacio=20Ria=C3=B1o=20Chico?= Date: Sun, 15 Oct 2023 01:02:45 +0200 Subject: [PATCH 2/5] tests: add unit tests for dataset.is_case_insensitive --- tests/unit/test_dataset.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index 3b1452805..3adb6708c 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -693,7 +693,6 @@ def _verify_access_entry(self, access_entries, resource): self.assertEqual(a_entry.entity_id, r_entry["entity_id"]) def _verify_readonly_resource_properties(self, dataset, resource): - self.assertEqual(dataset.project, self.PROJECT) self.assertEqual(dataset.dataset_id, self.DS_ID) self.assertEqual(dataset.reference.project, self.PROJECT) @@ -717,7 +716,6 @@ def _verify_readonly_resource_properties(self, dataset, resource): self.assertIsNone(dataset.self_link) def _verify_resource_properties(self, dataset, resource): - self._verify_readonly_resource_properties(dataset, resource) if "defaultTableExpirationMs" in resource: @@ -730,6 +728,9 @@ def _verify_resource_properties(self, dataset, resource): self.assertEqual(dataset.description, resource.get("description")) self.assertEqual(dataset.friendly_name, resource.get("friendlyName")) self.assertEqual(dataset.location, resource.get("location")) + self.assertEqual( + dataset.is_case_insensitive, resource.get("isCaseInsensitive") or False + ) if "defaultEncryptionConfiguration" in resource: self.assertEqual( dataset.default_encryption_configuration.kms_key_name, @@ -767,6 +768,7 @@ def test_ctor_defaults(self): self.assertIsNone(dataset.description) self.assertIsNone(dataset.friendly_name) self.assertIsNone(dataset.location) + self.assertEqual(dataset.is_case_insensitive, False) def test_ctor_string(self): dataset = self._make_one("some-project.some_dset") @@ -804,6 +806,7 @@ def test_ctor_explicit(self): self.assertIsNone(dataset.description) self.assertIsNone(dataset.friendly_name) self.assertIsNone(dataset.location) + self.assertEqual(dataset.is_case_insensitive, False) def test_access_entries_setter_non_list(self): dataset = self._make_one(self.DS_REF) @@ -896,6 +899,26 @@ def test_labels_getter_missing_value(self): dataset = self._make_one(self.DS_REF) self.assertEqual(dataset.labels, {}) + def test_is_case_insensitive_setter_bad_value(self): + dataset = self._make_one(self.DS_REF) + with self.assertRaises(ValueError): + dataset.is_case_insensitive = 0 + + def test_is_case_insensitive_setter_true(self): + dataset = self._make_one(self.DS_REF) + dataset.is_case_insensitive = True + self.assertEqual(dataset.is_case_insensitive, True) + + def test_is_case_insensitive_setter_none(self): + dataset = self._make_one(self.DS_REF) + dataset.is_case_insensitive = None + self.assertEqual(dataset.is_case_insensitive, False) + + def test_is_case_insensitive_setter_false(self): + dataset = self._make_one(self.DS_REF) + dataset.is_case_insensitive = False + self.assertEqual(dataset.is_case_insensitive, False) + def test_from_api_repr_missing_identity(self): self._setUpConstants() RESOURCE = {} From cefadd5c440e9bcd3458d644d845159f7b2ace8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Ignacio=20Ria=C3=B1o=20Chico?= Date: Sun, 15 Oct 2023 01:30:55 +0200 Subject: [PATCH 3/5] docs: improve comments for dataset.is_case_sensitive (code and tests) --- google/cloud/bigquery/dataset.py | 4 ++-- tests/system/test_client.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 6d42b3d23..7a3256c05 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -767,8 +767,8 @@ def default_encryption_configuration(self, value): @property def is_case_insensitive(self): - """Optional[bool]: TRUE if the dataset and its table names are case-insensitive, otherwise FALSE. - By default, this is FALSE, which means the dataset and its table names are case-sensitive. + """Optional[bool]: True if the dataset and its table names are case-insensitive, otherwise FALSE. + By default, this is False, which means the dataset and its table names are case-sensitive. This field does not affect routine references. Raises: diff --git a/tests/system/test_client.py b/tests/system/test_client.py index 511ba4ede..a72f49908 100644 --- a/tests/system/test_client.py +++ b/tests/system/test_client.py @@ -354,7 +354,7 @@ def test_create_tables_in_case_insensitive_dataset(self): _make_dataset_id("create_table"), is_case_insensitive=True ) table_arg = Table(ci_dataset.table("test_table2"), schema=SCHEMA) - tablemc_arg = Table(ci_dataset.table("Test_taBLe2")) + tablemc_arg = Table(ci_dataset.table("Test_taBLe2")) # same name, in Mixed Case table = helpers.retry_403(Config.CLIENT.create_table)(table_arg) self.to_delete.insert(0, table) @@ -368,7 +368,7 @@ def test_create_tables_in_case_sensitive_dataset(self): _make_dataset_id("create_table"), is_case_insensitive=False ) table_arg = Table(ci_dataset.table("test_table3"), schema=SCHEMA) - tablemc_arg = Table(ci_dataset.table("Test_taBLe3")) + tablemc_arg = Table(ci_dataset.table("Test_taBLe3")) # same name, in Mixed Case table = helpers.retry_403(Config.CLIENT.create_table)(table_arg) self.to_delete.insert(0, table) @@ -380,7 +380,9 @@ def test_create_tables_in_case_sensitive_dataset(self): def test_create_tables_in_default_sensitivity_dataset(self): dataset = self.temp_dataset(_make_dataset_id("create_table")) table_arg = Table(dataset.table("test_table4"), schema=SCHEMA) - tablemc_arg = Table(dataset.table("Test_taBLe4")) + tablemc_arg = Table( + dataset.table("Test_taBLe4") + ) # same name, in MC (Mixed Case) table = helpers.retry_403(Config.CLIENT.create_table)(table_arg) self.to_delete.insert(0, table) From 819309b3f4cd1426b694a5a8733ea3b896bfbd30 Mon Sep 17 00:00:00 2001 From: Jose Ignacio Riano Date: Thu, 19 Oct 2023 21:27:43 +0200 Subject: [PATCH 4/5] docs: improve docstring of is_case_insensitive Co-authored-by: Lingqing Gan --- google/cloud/bigquery/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index d859fe3f4..532903e65 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -766,7 +766,7 @@ def default_encryption_configuration(self, value): @property def is_case_insensitive(self): - """Optional[bool]: True if the dataset and its table names are case-insensitive, otherwise FALSE. + """Optional[bool]: True if the dataset and its table names are case-insensitive, otherwise False. By default, this is False, which means the dataset and its table names are case-sensitive. This field does not affect routine references. From 852b03e2da9778fd440b683dfbdb196c1ad99945 Mon Sep 17 00:00:00 2001 From: Lingqing Gan Date: Wed, 1 Nov 2023 11:23:26 -0700 Subject: [PATCH 5/5] Update tests/system/test_client.py --- tests/system/test_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/system/test_client.py b/tests/system/test_client.py index 54781cb94..c8ff551ce 100644 --- a/tests/system/test_client.py +++ b/tests/system/test_client.py @@ -2354,7 +2354,6 @@ def test_nested_table_to_arrow(self): self.assertTrue(pyarrow.types.is_list(record_col[1].type)) self.assertTrue(pyarrow.types.is_int64(record_col[1].type.value_type)) - def temp_dataset(self, dataset_id, *args, **kwargs): project = Config.CLIENT.project dataset_ref = bigquery.DatasetReference(project, dataset_id)