Skip to content

Commit

Permalink
bigquery: add client.list_dataset_tables
Browse files Browse the repository at this point in the history 
Remove Dataset.list_tables
  • Loading branch information
@jba
jba committed Sep 20, 2017
1 parent 4ec7bd2 commit 5cdd5b1
Show file tree
Hide file tree
Showing 5 changed files with 195 additions and 187 deletions.
53 changes: 53 additions & 0 deletions bigquery/google/cloud/bigquery/client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,44 @@ def update_dataset(self, dataset, fields):
method='PATCH', path=path, data=partial, headers=headers)
return Dataset.from_api_repr(api_response, self)

def list_dataset_tables(self, dataset, max_results=None, page_token=None):
"""List tables in the dataset.
See
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/list
:type dataset: One of:
:class:`~google.cloud.bigquery.dataset.Dataset`
:class:`~google.cloud.bigquery.dataset.DatasetReference`
:param dataset: the dataset whose tables to list, or a reference to it.
:type max_results: int
:param max_results: (Optional) Maximum number of tables to return.
If not passed, defaults to a value set by the API.
:type page_token: str
:param page_token: (Optional) Opaque marker for the next "page" of
datasets. If not passed, the API will return the
first page of datasets.
:rtype: :class:`~google.api.core.page_iterator.Iterator`
:returns: Iterator of :class:`~google.cloud.bigquery.table.Table`
contained within the current dataset.
"""
if not isinstance(dataset, (Dataset, DatasetReference)):
raise TypeError('dataset must be a Dataset or a DatasetReference')
path = '%s/tables' % dataset.path
result = page_iterator.HTTPIterator(
client=self,
api_request=self._connection.api_request,
path=path,
item_to_value=_item_to_table,
items_key='tables',
page_token=page_token,
max_results=max_results)
result.dataset = dataset
return result

def delete_dataset(self, dataset):
"""Delete a dataset.
Expand Down Expand Up @@ -600,3 +638,18 @@ def _item_to_job(iterator, resource):
:returns: The next job in the page.
"""
return iterator.client.job_from_resource(resource)


def _item_to_table(iterator, resource):
"""Convert a JSON table to the native object.
:type iterator: :class:`~google.api.core.page_iterator.Iterator`
:param iterator: The iterator that is currently in use.
:type resource: dict
:param resource: An item to be converted to a table.
:rtype: :class:`~google.cloud.bigquery.table.Table`
:returns: The next table in the page.
"""
return Table.from_api_repr(resource, iterator.client)
48 changes: 0 additions & 48 deletions bigquery/google/cloud/bigquery/dataset.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

import six

from google.api.core import page_iterator
from google.cloud._helpers import _datetime_from_microseconds
from google.cloud.bigquery.table import Table
from google.cloud.bigquery.table import TableReference
Expand Down Expand Up @@ -473,38 +472,6 @@ def _build_resource(self):

return resource

def list_tables(self, max_results=None, page_token=None):
"""List tables for the project associated with this client.
See
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/list
:type max_results: int
:param max_results: (Optional) Maximum number of tables to return.
If not passed, defaults to a value set by the API.
:type page_token: str
:param page_token: (Optional) Opaque marker for the next "page" of
datasets. If not passed, the API will return the
first page of datasets.
:rtype: :class:`~google.api.core.page_iterator.Iterator`
:returns: Iterator of :class:`~google.cloud.bigquery.table.Table`
contained within the current dataset.
"""
path = '/projects/%s/datasets/%s/tables' % (
self.project, self.dataset_id)
result = page_iterator.HTTPIterator(
client=self._client,
api_request=self._client._connection.api_request,
path=path,
item_to_value=_item_to_table,
items_key='tables',
page_token=page_token,
max_results=max_results)
result.dataset = self
return result

def table(self, name, schema=()):
"""Construct a table bound to this dataset.
Expand All @@ -519,18 +486,3 @@ def table(self, name, schema=()):
"""
table_ref = TableReference(self, name)
return Table(table_ref, schema=schema, client=self._client)


def _item_to_table(iterator, resource):
"""Convert a JSON table to the native object.
:type iterator: :class:`~google.api.core.page_iterator.Iterator`
:param iterator: The iterator that is currently in use.
:type resource: dict
:param resource: An item to be converted to a table.
:rtype: :class:`~google.cloud.bigquery.table.Table`
:returns: The next table in the page.
"""
return Table.from_api_repr(resource, iterator.dataset)
6 changes: 3 additions & 3 deletions bigquery/tests/system.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,13 +200,13 @@ def test_get_table_w_public_dataset(self):
self.assertEqual(
schema_names, ['word', 'word_count', 'corpus', 'corpus_date'])

def test_list_tables(self):
def test_list_dataset_tables(self):
DATASET_ID = _make_dataset_id('list_tables')
dataset = retry_403(Config.CLIENT.create_dataset)(Dataset(DATASET_ID))
self.to_delete.append(dataset)

# Retrieve tables before any are created for the dataset.
iterator = dataset.list_tables()
iterator = Config.CLIENT.list_dataset_tables(dataset)
all_tables = list(iterator)
self.assertEqual(all_tables, [])
self.assertIsNone(iterator.next_page_token)
Expand All @@ -226,7 +226,7 @@ def test_list_tables(self):
self.to_delete.insert(0, created_table)

# Retrieve the tables.
iterator = dataset.list_tables()
iterator = Config.CLIENT.list_dataset_tables(dataset)
all_tables = list(iterator)
self.assertIsNone(iterator.next_page_token)
created = [table for table in all_tables
Expand Down
136 changes: 136 additions & 0 deletions bigquery/tests/unit/test_client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -480,6 +480,142 @@ def test_update_dataset(self):
req = conn._requested[1]
self.assertEqual(req['headers']['If-Match'], 'etag')

def test_list_dataset_tables_empty(self):
import six

PROJECT = 'PROJECT'
DS_ID = 'DATASET_ID'
creds = _make_credentials()
client = self._make_one(project=PROJECT, credentials=creds)
conn = client._connection = _Connection({})

dataset = client.dataset(DS_ID)
iterator = client.list_dataset_tables(dataset)
self.assertIs(iterator.dataset, dataset)
page = six.next(iterator.pages)
tables = list(page)
token = iterator.next_page_token

self.assertEqual(tables, [])
self.assertIsNone(token)
self.assertEqual(len(conn._requested), 1)
req = conn._requested[0]
self.assertEqual(req['method'], 'GET')
PATH = 'projects/%s/datasets/%s/tables' % (PROJECT, DS_ID)
self.assertEqual(req['path'], '/%s' % PATH)

def test_list_dataset_tables_defaults(self):
import six
from google.cloud.bigquery.table import Table

PROJECT = 'PROJECT'
DS_ID = 'DATASET_ID'
TABLE_1 = 'table_one'
TABLE_2 = 'table_two'
PATH = 'projects/%s/datasets/%s/tables' % (PROJECT, DS_ID)
TOKEN = 'TOKEN'
DATA = {
'nextPageToken': TOKEN,
'tables': [
{'kind': 'bigquery#table',
'id': '%s:%s.%s' % (PROJECT, DS_ID, TABLE_1),
'tableReference': {'tableId': TABLE_1,
'datasetId': DS_ID,
'projectId': PROJECT},
'type': 'TABLE'},
{'kind': 'bigquery#table',
'id': '%s:%s.%s' % (PROJECT, DS_ID, TABLE_2),
'tableReference': {'tableId': TABLE_2,
'datasetId': DS_ID,
'projectId': PROJECT},
'type': 'TABLE'},
]
}

creds = _make_credentials()
client = self._make_one(project=PROJECT, credentials=creds)
conn = client._connection = _Connection(DATA)
dataset = client.dataset(DS_ID)

iterator = client.list_dataset_tables(dataset)
self.assertIs(iterator.dataset, dataset)
page = six.next(iterator.pages)
tables = list(page)
token = iterator.next_page_token

self.assertEqual(len(tables), len(DATA['tables']))
for found, expected in zip(tables, DATA['tables']):
self.assertIsInstance(found, Table)
self.assertEqual(found.full_table_id, expected['id'])
self.assertEqual(found.table_type, expected['type'])
self.assertEqual(token, TOKEN)

self.assertEqual(len(conn._requested), 1)
req = conn._requested[0]
self.assertEqual(req['method'], 'GET')
self.assertEqual(req['path'], '/%s' % PATH)

def test_list_dataset_tables_explicit(self):
import six
from google.cloud.bigquery.table import Table

PROJECT = 'PROJECT'
DS_ID = 'DATASET_ID'
TABLE_1 = 'table_one'
TABLE_2 = 'table_two'
PATH = 'projects/%s/datasets/%s/tables' % (PROJECT, DS_ID)
TOKEN = 'TOKEN'
DATA = {
'tables': [
{'kind': 'bigquery#dataset',
'id': '%s:%s.%s' % (PROJECT, DS_ID, TABLE_1),
'tableReference': {'tableId': TABLE_1,
'datasetId': DS_ID,
'projectId': PROJECT},
'type': 'TABLE'},
{'kind': 'bigquery#dataset',
'id': '%s:%s.%s' % (PROJECT, DS_ID, TABLE_2),
'tableReference': {'tableId': TABLE_2,
'datasetId': DS_ID,
'projectId': PROJECT},
'type': 'TABLE'},
]
}

creds = _make_credentials()
client = self._make_one(project=PROJECT, credentials=creds)
conn = client._connection = _Connection(DATA)
dataset = client.dataset(DS_ID)

iterator = client.list_dataset_tables(
dataset, max_results=3, page_token=TOKEN)
self.assertIs(iterator.dataset, dataset)
page = six.next(iterator.pages)
tables = list(page)
token = iterator.next_page_token

self.assertEqual(len(tables), len(DATA['tables']))
for found, expected in zip(tables, DATA['tables']):
self.assertIsInstance(found, Table)
self.assertEqual(found.full_table_id, expected['id'])
self.assertEqual(found.table_type, expected['type'])
self.assertIsNone(token)

self.assertEqual(len(conn._requested), 1)
req = conn._requested[0]
self.assertEqual(req['method'], 'GET')
self.assertEqual(req['path'], '/%s' % PATH)
self.assertEqual(req['query_params'],
{'maxResults': 3, 'pageToken': TOKEN})

def test_list_dataset_tables_wrong_type(self):
PROJECT = 'PROJECT'
DS_ID = 'DATASET_ID'
creds = _make_credentials()
client = self._make_one(project=PROJECT, credentials=creds)
with self.assertRaises(TypeError):
client.list_dataset_tables(client.dataset(DS_ID).table("foo"))

def test_delete_dataset(self):
from google.cloud.bigquery.dataset import Dataset

Expand Down
Loading

0 comments on commit 5cdd5b1

Please sign in to comment.