Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

BigQuery: address dataset leaks #6099

Merged
merged 5 commits into from
Sep 25, 2018
Merged

BigQuery: address dataset leaks #6099

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ae538f6
Script for scrubbing leaked datasets by prefix.
tseaver Sep 25, 2018
c7643fa
Don't leak dataset after intentionally failing to delete it.
tseaver Sep 25, 2018
c39b194
Make dataset IDs unique for snippets / systests.
tseaver Sep 25, 2018
ed76f14
Lint
tseaver Sep 25, 2018
562e4b5
Merge branch 'master' into 6098-bigquery-dataset_leaks
tseaver Sep 25, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions bigquery/docs/snippets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1212,8 +1212,8 @@ def test_table_insert_rows(client, to_delete):

def test_load_table_from_file(client, to_delete):
"""Upload table data from a CSV file."""
dataset_id = 'table_upload_from_file_dataset_{}'.format(_millis())
table_id = 'table_upload_from_file_table_{}'.format(_millis())
dataset_id = 'load_table_from_file_dataset_{}'.format(_millis())
table_id = 'load_table_from_file_table_{}'.format(_millis())
dataset = bigquery.Dataset(client.dataset(dataset_id))
dataset.location = 'US'
client.create_dataset(dataset)
Expand Down Expand Up @@ -1261,7 +1261,7 @@ def test_load_table_from_file(client, to_delete):


def test_load_table_from_uri_csv(client, to_delete, capsys):
dataset_id = 'load_table_dataset_{}'.format(_millis())
dataset_id = 'load_table_from_uri_csv_{}'.format(_millis())
dataset = bigquery.Dataset(client.dataset(dataset_id))
client.create_dataset(dataset)
to_delete.append(dataset)
Expand Down Expand Up @@ -1300,7 +1300,7 @@ def test_load_table_from_uri_csv(client, to_delete, capsys):


def test_load_table_from_uri_json(client, to_delete, capsys):
dataset_id = 'load_table_dataset_{}'.format(_millis())
dataset_id = 'load_table_from_uri_json_{}'.format(_millis())
dataset = bigquery.Dataset(client.dataset(dataset_id))
dataset.location = 'US'
client.create_dataset(dataset)
Expand Down Expand Up @@ -1381,7 +1381,7 @@ def test_load_table_from_uri_cmek(client, to_delete):


def test_load_table_from_uri_parquet(client, to_delete, capsys):
dataset_id = 'load_table_dataset_{}'.format(_millis())
dataset_id = 'load_table_from_uri_parquet_{}'.format(_millis())
dataset = bigquery.Dataset(client.dataset(dataset_id))
client.create_dataset(dataset)
to_delete.append(dataset)
Expand Down Expand Up @@ -1414,7 +1414,7 @@ def test_load_table_from_uri_parquet(client, to_delete, capsys):


def test_load_table_from_uri_orc(client, to_delete, capsys):
dataset_id = 'load_table_dataset_{}'.format(_millis())
dataset_id = 'load_table_from_uri_orc_{}'.format(_millis())
dataset = bigquery.Dataset(client.dataset(dataset_id))
client.create_dataset(dataset)
to_delete.append(dataset)
Expand Down Expand Up @@ -1458,7 +1458,7 @@ def test_load_table_from_uri_autodetect(client, to_delete, capsys):
followed by more shared code. Note that only the last format in the
format-specific code section will be tested in this test.
"""
dataset_id = 'load_table_dataset_{}'.format(_millis())
dataset_id = 'load_table_from_uri_auto_{}'.format(_millis())
dataset = bigquery.Dataset(client.dataset(dataset_id))
client.create_dataset(dataset)
to_delete.append(dataset)
Expand Down Expand Up @@ -1524,7 +1524,7 @@ def test_load_table_from_uri_truncate(client, to_delete, capsys):
followed by more shared code. Note that only the last format in the
format-specific code section will be tested in this test.
"""
dataset_id = 'load_table_dataset_{}'.format(_millis())
dataset_id = 'load_table_from_uri_trunc_{}'.format(_millis())
dataset = bigquery.Dataset(client.dataset(dataset_id))
client.create_dataset(dataset)
to_delete.append(dataset)
Expand Down Expand Up @@ -2972,7 +2972,7 @@ def test_list_rows_as_dataframe(client):
@pytest.mark.skipif(pandas is None, reason='Requires `pandas`')
@pytest.mark.skipif(pyarrow is None, reason='Requires `pyarrow`')
def test_load_table_from_dataframe(client, to_delete):
dataset_id = 'load_table_dataframe_dataset_{}'.format(_millis())
dataset_id = 'load_table_from_dataframe_{}'.format(_millis())
dataset = bigquery.Dataset(client.dataset(dataset_id))
client.create_dataset(dataset)
to_delete.append(dataset)
Expand Down
26 changes: 26 additions & 0 deletions bigquery/tests/scrub_datasets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import re
import sys

from google.api_core.exceptions import NotFound
from google.cloud.bigquery import Client


def main(prefixes):
client = Client()

pattern = re.compile(
'|'.join('^{}.*$'.format(prefix) for prefix in prefixes))

ds_items = list(client.list_datasets())

This comment was marked as spam.

Sign in to view

This comment was marked as spam.

Sign in to view

This comment was marked as spam.

Sign in to view

for dataset in ds_items:
ds_id = dataset.dataset_id
if pattern.match(ds_id):
print("Deleting dataset: {}".format(ds_id))
try:
client.delete_dataset(dataset.reference, delete_contents=True)
except NotFound:
print(" NOT FOUND")


if __name__ == '__main__':
main(sys.argv[1:])
9 changes: 3 additions & 6 deletions bigquery/tests/system.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,10 +295,7 @@ def test_delete_dataset_delete_contents_true(self):

def test_delete_dataset_delete_contents_false(self):
from google.api_core import exceptions
dataset_id = _make_dataset_id('delete_table_false')
dataset = retry_403(Config.CLIENT.create_dataset)(
Dataset(Config.CLIENT.dataset(dataset_id)))

dataset = self.temp_dataset(_make_dataset_id('delete_table_false'))
table_id = 'test_table'
table_arg = Table(dataset.table(table_id), schema=SCHEMA)

Expand Down Expand Up @@ -1191,7 +1188,7 @@ def _load_table_for_dml(self, rows, dataset_id, table_id):
self._fetch_single_page(table)

def test_query_w_dml(self):
dataset_name = _make_dataset_id('dml_tests')
dataset_name = _make_dataset_id('dml_query')
table_name = 'test_table'
self._load_table_for_dml([('Hello World',)], dataset_name, table_name)
query_template = """UPDATE {}.{}
Expand All @@ -1207,7 +1204,7 @@ def test_query_w_dml(self):
self.assertEqual(query_job.num_dml_affected_rows, 1)

def test_dbapi_w_dml(self):
dataset_name = _make_dataset_id('dml_tests')
dataset_name = _make_dataset_id('dml_dbapi')
table_name = 'test_table'
self._load_table_for_dml([('Hello World',)], dataset_name, table_name)
query_template = """UPDATE {}.{}
Expand Down