From 5cdd5b15fc109992c95de17c2fcccefc6ebcc664 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Wed, 20 Sep 2017 14:45:13 -0400 Subject: [PATCH] bigquery: add client.list_dataset_tables Remove Dataset.list_tables --- bigquery/google/cloud/bigquery/client.py | 53 +++++++++ bigquery/google/cloud/bigquery/dataset.py | 48 -------- bigquery/tests/system.py | 6 +- bigquery/tests/unit/test_client.py | 136 +++++++++++++++++++++ bigquery/tests/unit/test_dataset.py | 139 +--------------------- 5 files changed, 195 insertions(+), 187 deletions(-) diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index 940b016645f3a..9d470259f950b 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -264,6 +264,44 @@ def update_dataset(self, dataset, fields): method='PATCH', path=path, data=partial, headers=headers) return Dataset.from_api_repr(api_response, self) + def list_dataset_tables(self, dataset, max_results=None, page_token=None): + """List tables in the dataset. + + See + https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/list + + :type dataset: One of: + :class:`~google.cloud.bigquery.dataset.Dataset` + :class:`~google.cloud.bigquery.dataset.DatasetReference` + :param dataset: the dataset whose tables to list, or a reference to it. + + :type max_results: int + :param max_results: (Optional) Maximum number of tables to return. + If not passed, defaults to a value set by the API. + + :type page_token: str + :param page_token: (Optional) Opaque marker for the next "page" of + datasets. If not passed, the API will return the + first page of datasets. + + :rtype: :class:`~google.api.core.page_iterator.Iterator` + :returns: Iterator of :class:`~google.cloud.bigquery.table.Table` + contained within the current dataset. + """ + if not isinstance(dataset, (Dataset, DatasetReference)): + raise TypeError('dataset must be a Dataset or a DatasetReference') + path = '%s/tables' % dataset.path + result = page_iterator.HTTPIterator( + client=self, + api_request=self._connection.api_request, + path=path, + item_to_value=_item_to_table, + items_key='tables', + page_token=page_token, + max_results=max_results) + result.dataset = dataset + return result + def delete_dataset(self, dataset): """Delete a dataset. @@ -600,3 +638,18 @@ def _item_to_job(iterator, resource): :returns: The next job in the page. """ return iterator.client.job_from_resource(resource) + + +def _item_to_table(iterator, resource): + """Convert a JSON table to the native object. + + :type iterator: :class:`~google.api.core.page_iterator.Iterator` + :param iterator: The iterator that is currently in use. + + :type resource: dict + :param resource: An item to be converted to a table. + + :rtype: :class:`~google.cloud.bigquery.table.Table` + :returns: The next table in the page. + """ + return Table.from_api_repr(resource, iterator.client) diff --git a/bigquery/google/cloud/bigquery/dataset.py b/bigquery/google/cloud/bigquery/dataset.py index 0d9198809595b..88005fddbbe2d 100644 --- a/bigquery/google/cloud/bigquery/dataset.py +++ b/bigquery/google/cloud/bigquery/dataset.py @@ -18,7 +18,6 @@ import six -from google.api.core import page_iterator from google.cloud._helpers import _datetime_from_microseconds from google.cloud.bigquery.table import Table from google.cloud.bigquery.table import TableReference @@ -473,38 +472,6 @@ def _build_resource(self): return resource - def list_tables(self, max_results=None, page_token=None): - """List tables for the project associated with this client. - - See - https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/list - - :type max_results: int - :param max_results: (Optional) Maximum number of tables to return. - If not passed, defaults to a value set by the API. - - :type page_token: str - :param page_token: (Optional) Opaque marker for the next "page" of - datasets. If not passed, the API will return the - first page of datasets. - - :rtype: :class:`~google.api.core.page_iterator.Iterator` - :returns: Iterator of :class:`~google.cloud.bigquery.table.Table` - contained within the current dataset. - """ - path = '/projects/%s/datasets/%s/tables' % ( - self.project, self.dataset_id) - result = page_iterator.HTTPIterator( - client=self._client, - api_request=self._client._connection.api_request, - path=path, - item_to_value=_item_to_table, - items_key='tables', - page_token=page_token, - max_results=max_results) - result.dataset = self - return result - def table(self, name, schema=()): """Construct a table bound to this dataset. @@ -519,18 +486,3 @@ def table(self, name, schema=()): """ table_ref = TableReference(self, name) return Table(table_ref, schema=schema, client=self._client) - - -def _item_to_table(iterator, resource): - """Convert a JSON table to the native object. - - :type iterator: :class:`~google.api.core.page_iterator.Iterator` - :param iterator: The iterator that is currently in use. - - :type resource: dict - :param resource: An item to be converted to a table. - - :rtype: :class:`~google.cloud.bigquery.table.Table` - :returns: The next table in the page. - """ - return Table.from_api_repr(resource, iterator.dataset) diff --git a/bigquery/tests/system.py b/bigquery/tests/system.py index 117e2bf3f60d8..7630b2bb4d2be 100644 --- a/bigquery/tests/system.py +++ b/bigquery/tests/system.py @@ -200,13 +200,13 @@ def test_get_table_w_public_dataset(self): self.assertEqual( schema_names, ['word', 'word_count', 'corpus', 'corpus_date']) - def test_list_tables(self): + def test_list_dataset_tables(self): DATASET_ID = _make_dataset_id('list_tables') dataset = retry_403(Config.CLIENT.create_dataset)(Dataset(DATASET_ID)) self.to_delete.append(dataset) # Retrieve tables before any are created for the dataset. - iterator = dataset.list_tables() + iterator = Config.CLIENT.list_dataset_tables(dataset) all_tables = list(iterator) self.assertEqual(all_tables, []) self.assertIsNone(iterator.next_page_token) @@ -226,7 +226,7 @@ def test_list_tables(self): self.to_delete.insert(0, created_table) # Retrieve the tables. - iterator = dataset.list_tables() + iterator = Config.CLIENT.list_dataset_tables(dataset) all_tables = list(iterator) self.assertIsNone(iterator.next_page_token) created = [table for table in all_tables diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index ceb530d5a1347..8916e9682ac14 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -480,6 +480,142 @@ def test_update_dataset(self): req = conn._requested[1] self.assertEqual(req['headers']['If-Match'], 'etag') + def test_list_dataset_tables_empty(self): + import six + + PROJECT = 'PROJECT' + DS_ID = 'DATASET_ID' + creds = _make_credentials() + client = self._make_one(project=PROJECT, credentials=creds) + conn = client._connection = _Connection({}) + + dataset = client.dataset(DS_ID) + iterator = client.list_dataset_tables(dataset) + self.assertIs(iterator.dataset, dataset) + page = six.next(iterator.pages) + tables = list(page) + token = iterator.next_page_token + + self.assertEqual(tables, []) + self.assertIsNone(token) + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'GET') + PATH = 'projects/%s/datasets/%s/tables' % (PROJECT, DS_ID) + self.assertEqual(req['path'], '/%s' % PATH) + + def test_list_dataset_tables_defaults(self): + import six + from google.cloud.bigquery.table import Table + + PROJECT = 'PROJECT' + DS_ID = 'DATASET_ID' + TABLE_1 = 'table_one' + TABLE_2 = 'table_two' + PATH = 'projects/%s/datasets/%s/tables' % (PROJECT, DS_ID) + TOKEN = 'TOKEN' + DATA = { + 'nextPageToken': TOKEN, + 'tables': [ + {'kind': 'bigquery#table', + 'id': '%s:%s.%s' % (PROJECT, DS_ID, TABLE_1), + 'tableReference': {'tableId': TABLE_1, + 'datasetId': DS_ID, + 'projectId': PROJECT}, + 'type': 'TABLE'}, + {'kind': 'bigquery#table', + 'id': '%s:%s.%s' % (PROJECT, DS_ID, TABLE_2), + 'tableReference': {'tableId': TABLE_2, + 'datasetId': DS_ID, + 'projectId': PROJECT}, + 'type': 'TABLE'}, + ] + } + + creds = _make_credentials() + client = self._make_one(project=PROJECT, credentials=creds) + conn = client._connection = _Connection(DATA) + dataset = client.dataset(DS_ID) + + iterator = client.list_dataset_tables(dataset) + self.assertIs(iterator.dataset, dataset) + page = six.next(iterator.pages) + tables = list(page) + token = iterator.next_page_token + + self.assertEqual(len(tables), len(DATA['tables'])) + for found, expected in zip(tables, DATA['tables']): + self.assertIsInstance(found, Table) + self.assertEqual(found.full_table_id, expected['id']) + self.assertEqual(found.table_type, expected['type']) + self.assertEqual(token, TOKEN) + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], '/%s' % PATH) + + def test_list_dataset_tables_explicit(self): + import six + from google.cloud.bigquery.table import Table + + PROJECT = 'PROJECT' + DS_ID = 'DATASET_ID' + TABLE_1 = 'table_one' + TABLE_2 = 'table_two' + PATH = 'projects/%s/datasets/%s/tables' % (PROJECT, DS_ID) + TOKEN = 'TOKEN' + DATA = { + 'tables': [ + {'kind': 'bigquery#dataset', + 'id': '%s:%s.%s' % (PROJECT, DS_ID, TABLE_1), + 'tableReference': {'tableId': TABLE_1, + 'datasetId': DS_ID, + 'projectId': PROJECT}, + 'type': 'TABLE'}, + {'kind': 'bigquery#dataset', + 'id': '%s:%s.%s' % (PROJECT, DS_ID, TABLE_2), + 'tableReference': {'tableId': TABLE_2, + 'datasetId': DS_ID, + 'projectId': PROJECT}, + 'type': 'TABLE'}, + ] + } + + creds = _make_credentials() + client = self._make_one(project=PROJECT, credentials=creds) + conn = client._connection = _Connection(DATA) + dataset = client.dataset(DS_ID) + + iterator = client.list_dataset_tables( + dataset, max_results=3, page_token=TOKEN) + self.assertIs(iterator.dataset, dataset) + page = six.next(iterator.pages) + tables = list(page) + token = iterator.next_page_token + + self.assertEqual(len(tables), len(DATA['tables'])) + for found, expected in zip(tables, DATA['tables']): + self.assertIsInstance(found, Table) + self.assertEqual(found.full_table_id, expected['id']) + self.assertEqual(found.table_type, expected['type']) + self.assertIsNone(token) + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['query_params'], + {'maxResults': 3, 'pageToken': TOKEN}) + + def test_list_dataset_tables_wrong_type(self): + PROJECT = 'PROJECT' + DS_ID = 'DATASET_ID' + creds = _make_credentials() + client = self._make_one(project=PROJECT, credentials=creds) + with self.assertRaises(TypeError): + client.list_dataset_tables(client.dataset(DS_ID).table("foo")) + def test_delete_dataset(self): from google.cloud.bigquery.dataset import Dataset diff --git a/bigquery/tests/unit/test_dataset.py b/bigquery/tests/unit/test_dataset.py index ac863d5052d5d..1781aba4b3060 100644 --- a/bigquery/tests/unit/test_dataset.py +++ b/bigquery/tests/unit/test_dataset.py @@ -388,129 +388,10 @@ def test__parse_access_entries_w_extra_keys(self): with self.assertRaises(ValueError): dataset._parse_access_entries(ACCESS) - def test_list_tables_empty(self): - import six - - conn = _Connection({}) - client = _Client(project=self.PROJECT, connection=conn) - dataset = self._make_one(self.DS_ID, client=client) - - iterator = dataset.list_tables() - self.assertIs(iterator.dataset, dataset) - page = six.next(iterator.pages) - tables = list(page) - token = iterator.next_page_token - - self.assertEqual(tables, []) - self.assertIsNone(token) - self.assertEqual(len(conn._requested), 1) - req = conn._requested[0] - self.assertEqual(req['method'], 'GET') - PATH = 'projects/%s/datasets/%s/tables' % (self.PROJECT, self.DS_ID) - self.assertEqual(req['path'], '/%s' % PATH) - - def test_list_tables_defaults(self): - import six - from google.cloud.bigquery.table import Table - - TABLE_1 = 'table_one' - TABLE_2 = 'table_two' - PATH = 'projects/%s/datasets/%s/tables' % (self.PROJECT, self.DS_ID) - TOKEN = 'TOKEN' - DATA = { - 'nextPageToken': TOKEN, - 'tables': [ - {'kind': 'bigquery#table', - 'id': '%s:%s.%s' % (self.PROJECT, self.DS_ID, TABLE_1), - 'tableReference': {'tableId': TABLE_1, - 'datasetId': self.DS_ID, - 'projectId': self.PROJECT}, - 'type': 'TABLE'}, - {'kind': 'bigquery#table', - 'id': '%s:%s.%s' % (self.PROJECT, self.DS_ID, TABLE_2), - 'tableReference': {'tableId': TABLE_2, - 'datasetId': self.DS_ID, - 'projectId': self.PROJECT}, - 'type': 'TABLE'}, - ] - } - - conn = _Connection(DATA) - client = _Client(project=self.PROJECT, connection=conn) - dataset = self._make_one(self.DS_ID, client=client) - - iterator = dataset.list_tables() - self.assertIs(iterator.dataset, dataset) - page = six.next(iterator.pages) - tables = list(page) - token = iterator.next_page_token - - self.assertEqual(len(tables), len(DATA['tables'])) - for found, expected in zip(tables, DATA['tables']): - self.assertIsInstance(found, Table) - self.assertEqual(found.full_table_id, expected['id']) - self.assertEqual(found.table_type, expected['type']) - self.assertEqual(token, TOKEN) - - self.assertEqual(len(conn._requested), 1) - req = conn._requested[0] - self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) - - def test_list_tables_explicit(self): - import six - from google.cloud.bigquery.table import Table - - TABLE_1 = 'table_one' - TABLE_2 = 'table_two' - PATH = 'projects/%s/datasets/%s/tables' % (self.PROJECT, self.DS_ID) - TOKEN = 'TOKEN' - DATA = { - 'tables': [ - {'kind': 'bigquery#dataset', - 'id': '%s:%s.%s' % (self.PROJECT, self.DS_ID, TABLE_1), - 'tableReference': {'tableId': TABLE_1, - 'datasetId': self.DS_ID, - 'projectId': self.PROJECT}, - 'type': 'TABLE'}, - {'kind': 'bigquery#dataset', - 'id': '%s:%s.%s' % (self.PROJECT, self.DS_ID, TABLE_2), - 'tableReference': {'tableId': TABLE_2, - 'datasetId': self.DS_ID, - 'projectId': self.PROJECT}, - 'type': 'TABLE'}, - ] - } - - conn = _Connection(DATA) - client = _Client(project=self.PROJECT, connection=conn) - dataset = self._make_one(self.DS_ID, client=client) - - iterator = dataset.list_tables(max_results=3, page_token=TOKEN) - self.assertIs(iterator.dataset, dataset) - page = six.next(iterator.pages) - tables = list(page) - token = iterator.next_page_token - - self.assertEqual(len(tables), len(DATA['tables'])) - for found, expected in zip(tables, DATA['tables']): - self.assertIsInstance(found, Table) - self.assertEqual(found.full_table_id, expected['id']) - self.assertEqual(found.table_type, expected['type']) - self.assertIsNone(token) - - self.assertEqual(len(conn._requested), 1) - req = conn._requested[0] - self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) - self.assertEqual(req['query_params'], - {'maxResults': 3, 'pageToken': TOKEN}) - def test_table_wo_schema(self): from google.cloud.bigquery.table import Table - conn = _Connection({}) - client = _Client(project=self.PROJECT, connection=conn) + client = _Client(project=self.PROJECT) dataset = self._make_one(self.DS_ID, client=client) table = dataset.table('table_id') self.assertIsInstance(table, Table) @@ -523,8 +404,7 @@ def test_table_w_schema(self): from google.cloud.bigquery.schema import SchemaField from google.cloud.bigquery.table import Table - conn = _Connection({}) - client = _Client(project=self.PROJECT, connection=conn) + client = _Client(project=self.PROJECT) dataset = self._make_one(self.DS_ID, client=client) full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') age = SchemaField('age', 'INTEGER', mode='REQUIRED') @@ -538,18 +418,5 @@ def test_table_w_schema(self): class _Client(object): - def __init__(self, project='project', connection=None): + def __init__(self, project='project'): self.project = project - self._connection = connection - - -class _Connection(object): - - def __init__(self, *responses): - self._responses = responses - self._requested = [] - - def api_request(self, **kw): - self._requested.append(kw) - response, self._responses = self._responses[0], self._responses[1:] - return response