diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index da69decd03c8..db1a4b0f2138 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -26,7 +26,7 @@ from google.cloud.bigquery._http import Connection from google.cloud.bigquery.dataset import Dataset from google.cloud.bigquery.dataset import DatasetReference -from google.cloud.bigquery.table import Table +from google.cloud.bigquery.table import Table, _TABLE_HAS_NO_SCHEMA from google.cloud.bigquery.table import TableReference from google.cloud.bigquery.job import CopyJob from google.cloud.bigquery.job import ExtractJob @@ -34,6 +34,8 @@ from google.cloud.bigquery.job import QueryJob from google.cloud.bigquery.job import QueryJobConfig from google.cloud.bigquery.query import QueryResults +from google.cloud.bigquery._helpers import _item_to_row +from google.cloud.bigquery._helpers import _rows_page_start class Project(object): @@ -346,7 +348,6 @@ def delete_table(self, table): :type table: One of: :class:`~google.cloud.bigquery.table.Table` :class:`~google.cloud.bigquery.table.TableReference` - :param table: the table to delete, or a reference to it. """ if not isinstance(table, (Table, TableReference)): @@ -667,6 +668,80 @@ def query_rows(self, query, job_config=None, job_id=None, timeout=None): job.begin() return job.result(timeout=timeout) + def list_rows(self, table, selected_fields=None, max_results=None, + page_token=None, start_index=None): + """List the rows of the table. + + See + https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/list + + .. note:: + + This method assumes that the provided schema is up-to-date with the + schema as defined on the back-end: if the two schemas are not + identical, the values returned may be incomplete. To ensure that the + local copy of the schema is up-to-date, call ``client.get_table``. + + :type table: One of: + :class:`~google.cloud.bigquery.table.Table` + :class:`~google.cloud.bigquery.table.TableReference` + :param table: the table to list, or a reference to it. + + :type selected_fields: list of :class:`SchemaField` + :param selected_fields: + The fields to return. Required if ``table`` is a + :class:`~google.cloud.bigquery.table.TableReference`. + + :type max_results: int + :param max_results: maximum number of rows to return. + + :type page_token: str + :param page_token: (Optional) Token representing a cursor into the + table's rows. + + :type start_index: int + :param page_token: (Optional) The zero-based index of the starting + row to read. + + :rtype: :class:`~google.api.core.page_iterator.Iterator` + :returns: Iterator of row data :class:`tuple`s. During each page, the + iterator will have the ``total_rows`` attribute set, + which counts the total number of rows **in the table** + (this is distinct from the total number of rows in the + current page: ``iterator.page.num_items``). + + """ + if selected_fields is not None: + schema = selected_fields + elif isinstance(table, TableReference): + raise ValueError('need selected_fields with TableReference') + elif isinstance(table, Table): + if len(table._schema) == 0: + raise ValueError(_TABLE_HAS_NO_SCHEMA) + schema = table.schema + else: + raise TypeError('table should be Table or TableReference') + + params = {} + if selected_fields is not None: + params['selectedFields'] = [f.name for f in selected_fields] + if start_index is not None: + params['startIndex'] = start_index + + iterator = page_iterator.HTTPIterator( + client=self, + api_request=self._connection.api_request, + path='%s/data' % (table.path,), + item_to_value=_item_to_row, + items_key='rows', + page_token=page_token, + next_token='pageToken', + max_results=max_results, + page_start=_rows_page_start, + extra_params=params) + iterator.schema = schema + return iterator + # pylint: disable=unused-argument def _item_to_project(iterator, resource): diff --git a/bigquery/google/cloud/bigquery/table.py b/bigquery/google/cloud/bigquery/table.py index a4f1933303a0..dfc31be29745 100644 --- a/bigquery/google/cloud/bigquery/table.py +++ b/bigquery/google/cloud/bigquery/table.py @@ -25,13 +25,10 @@ from google.resumable_media.requests import MultipartUpload from google.resumable_media.requests import ResumableUpload -from google.api.core import page_iterator from google.cloud import exceptions from google.cloud._helpers import _datetime_from_microseconds from google.cloud._helpers import _millis_from_datetime from google.cloud.bigquery.schema import SchemaField -from google.cloud.bigquery._helpers import _item_to_row -from google.cloud.bigquery._helpers import _rows_page_start from google.cloud.bigquery._helpers import _SCALAR_VALUE_TO_JSON_ROW @@ -768,61 +765,6 @@ def update(self, client=None): method='PUT', path=self.path, data=self._build_resource()) self._set_properties(api_response) - def fetch_data(self, max_results=None, page_token=None, client=None): - """API call: fetch the table data via a GET request - - See - https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/list - - .. note:: - - This method assumes that its instance's ``schema`` attribute is - up-to-date with the schema as defined on the back-end: if the - two schemas are not identical, the values returned may be - incomplete. To ensure that the local copy of the schema is - up-to-date, call ``client.get_table``. - - :type max_results: int - :param max_results: (Optional) Maximum number of rows to return. - - :type page_token: str - :param page_token: (Optional) Token representing a cursor into the - table's rows. - - :type client: :class:`~google.cloud.bigquery.client.Client` - :param client: (Optional) The client to use. If not passed, falls - back to the ``client`` stored on the current dataset. - - :rtype: :class:`~google.api.core.page_iterator.Iterator` - :returns: Iterator of row data :class:`tuple`s. During each page, the - iterator will have the ``total_rows`` attribute set, - which counts the total number of rows **in the table** - (this is distinct from the total number of rows in the - current page: ``iterator.page.num_items``). - """ - if len(self._schema) == 0: - raise ValueError(_TABLE_HAS_NO_SCHEMA) - - params = {} - - if max_results is not None: - params['maxResults'] = max_results - - client = self._require_client(client) - path = '%s/data' % (self.path,) - iterator = page_iterator.HTTPIterator( - client=client, - api_request=client._connection.api_request, - path=path, - item_to_value=_item_to_row, - items_key='rows', - page_token=page_token, - page_start=_rows_page_start, - next_token='pageToken', - extra_params=params) - iterator.schema = self._schema - return iterator - def row_from_mapping(self, mapping): """Convert a mapping to a row tuple using the schema. diff --git a/bigquery/tests/system.py b/bigquery/tests/system.py index ada6d92b5050..1936fc435e57 100644 --- a/bigquery/tests/system.py +++ b/bigquery/tests/system.py @@ -295,7 +295,7 @@ def test_update_table(self): @staticmethod def _fetch_single_page(table): - iterator = table.fetch_data() + iterator = Config.CLIENT.list_rows(table) page = six.next(iterator.pages) return list(page) diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index 3f667039f497..806aaa15ca34 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -106,7 +106,6 @@ def test__get_query_results_hit(self): self.assertTrue(query_results.complete) def test_list_projects_defaults(self): - import six from google.cloud.bigquery.client import Project PROJECT_1 = 'PROJECT_ONE' @@ -151,8 +150,6 @@ def test_list_projects_defaults(self): self.assertEqual(req['path'], '/%s' % PATH) def test_list_projects_explicit_response_missing_projects_key(self): - import six - PROJECT = 'PROJECT' PATH = 'projects' TOKEN = 'TOKEN' @@ -177,7 +174,6 @@ def test_list_projects_explicit_response_missing_projects_key(self): {'maxResults': 3, 'pageToken': TOKEN}) def test_list_datasets_defaults(self): - import six from google.cloud.bigquery.dataset import Dataset PROJECT = 'PROJECT' @@ -222,8 +218,6 @@ def test_list_datasets_defaults(self): self.assertEqual(req['path'], '/%s' % PATH) def test_list_datasets_explicit_response_missing_datasets_key(self): - import six - PROJECT = 'PROJECT' PATH = 'projects/%s/datasets' % PROJECT TOKEN = 'TOKEN' @@ -636,8 +630,6 @@ def test_update_dataset(self): self.assertEqual(req['headers']['If-Match'], 'etag') def test_list_dataset_tables_empty(self): - import six - PROJECT = 'PROJECT' DS_ID = 'DATASET_ID' creds = _make_credentials() @@ -660,7 +652,6 @@ def test_list_dataset_tables_empty(self): self.assertEqual(req['path'], '/%s' % PATH) def test_list_dataset_tables_defaults(self): - import six from google.cloud.bigquery.table import Table PROJECT = 'PROJECT' @@ -711,7 +702,6 @@ def test_list_dataset_tables_defaults(self): self.assertEqual(req['path'], '/%s' % PATH) def test_list_dataset_tables_explicit(self): - import six from google.cloud.bigquery.table import Table PROJECT = 'PROJECT' @@ -895,7 +885,6 @@ def test_get_job_hit(self): self.assertEqual(req['query_params'], {'projection': 'full'}) def test_list_jobs_defaults(self): - import six from google.cloud.bigquery.job import LoadJob from google.cloud.bigquery.job import CopyJob from google.cloud.bigquery.job import ExtractJob @@ -1027,7 +1016,6 @@ def test_list_jobs_defaults(self): self.assertEqual(req['query_params'], {'projection': 'full'}) def test_list_jobs_load_job_wo_sourceUris(self): - import six from google.cloud.bigquery.job import LoadJob PROJECT = 'PROJECT' @@ -1084,8 +1072,6 @@ def test_list_jobs_load_job_wo_sourceUris(self): self.assertEqual(req['query_params'], {'projection': 'full'}) def test_list_jobs_explicit_missing(self): - import six - PROJECT = 'PROJECT' PATH = 'projects/%s/jobs' % PROJECT DATA = {} @@ -1546,6 +1532,261 @@ def test_query_rows_w_job_config(self): self.assertEqual(configuration['query']['useLegacySql'], True) self.assertEqual(configuration['dryRun'], True) + def test_list_rows(self): + import datetime + from google.cloud._helpers import UTC + from google.cloud.bigquery.dataset import DatasetReference + from google.cloud.bigquery.table import Table, SchemaField + + PROJECT = 'PROJECT' + DS_ID = 'DS_ID' + TABLE_ID = 'TABLE_ID' + PATH = 'projects/%s/datasets/%s/tables/%s/data' % ( + PROJECT, DS_ID, TABLE_ID) + WHEN_TS = 1437767599.006 + WHEN = datetime.datetime.utcfromtimestamp(WHEN_TS).replace( + tzinfo=UTC) + WHEN_1 = WHEN + datetime.timedelta(seconds=1) + WHEN_2 = WHEN + datetime.timedelta(seconds=2) + ROWS = 1234 + TOKEN = 'TOKEN' + + def _bigquery_timestamp_float_repr(ts_float): + # Preserve microsecond precision for E+09 timestamps + return '%0.15E' % (ts_float,) + + DATA = { + 'totalRows': str(ROWS), + 'pageToken': TOKEN, + 'rows': [ + {'f': [ + {'v': 'Phred Phlyntstone'}, + {'v': '32'}, + {'v': _bigquery_timestamp_float_repr(WHEN_TS)}, + ]}, + {'f': [ + {'v': 'Bharney Rhubble'}, + {'v': '33'}, + {'v': _bigquery_timestamp_float_repr(WHEN_TS + 1)}, + ]}, + {'f': [ + {'v': 'Wylma Phlyntstone'}, + {'v': '29'}, + {'v': _bigquery_timestamp_float_repr(WHEN_TS + 2)}, + ]}, + {'f': [ + {'v': 'Bhettye Rhubble'}, + {'v': None}, + {'v': None}, + ]}, + ] + } + creds = _make_credentials() + http = object() + client = self._make_one(project=PROJECT, credentials=creds, _http=http) + conn = client._connection = _Connection(DATA, DATA) + table_ref = DatasetReference(PROJECT, DS_ID).table(TABLE_ID) + full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') + age = SchemaField('age', 'INTEGER', mode='NULLABLE') + joined = SchemaField('joined', 'TIMESTAMP', mode='NULLABLE') + table = Table(table_ref, schema=[full_name, age, joined]) + + iterator = client.list_rows(table) + page = six.next(iterator.pages) + rows = list(page) + total_rows = iterator.total_rows + page_token = iterator.next_page_token + + self.assertEqual(len(rows), 4) + self.assertEqual(rows[0], ('Phred Phlyntstone', 32, WHEN)) + self.assertEqual(rows[1], ('Bharney Rhubble', 33, WHEN_1)) + self.assertEqual(rows[2], ('Wylma Phlyntstone', 29, WHEN_2)) + self.assertEqual(rows[3], ('Bhettye Rhubble', None, None)) + self.assertEqual(total_rows, ROWS) + self.assertEqual(page_token, TOKEN) + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['query_params'], {}) + + def test_list_rows_query_params(self): + from google.cloud.bigquery.dataset import DatasetReference + from google.cloud.bigquery.table import Table, SchemaField + + PROJECT = 'PROJECT' + DS_ID = 'DS_ID' + TABLE_ID = 'TABLE_ID' + creds = _make_credentials() + http = object() + client = self._make_one(project=PROJECT, credentials=creds, _http=http) + table_ref = DatasetReference(PROJECT, DS_ID).table(TABLE_ID) + table = Table(table_ref, + schema=[SchemaField('age', 'INTEGER', mode='NULLABLE')]) + tests = [ + ({}, {}), + ({'start_index': 1}, {'startIndex': 1}), + ({'max_results': 2}, {'maxResults': 2}), + ({'start_index': 1, 'max_results': 2}, + {'startIndex': 1, 'maxResults': 2}), + ] + conn = client._connection = _Connection(*len(tests)*[{}]) + for i, test in enumerate(tests): + iterator = client.list_rows(table, **test[0]) + six.next(iterator.pages) + req = conn._requested[i] + self.assertEqual(req['query_params'], test[1], + 'for kwargs %s' % test[0]) + + def test_list_rows_repeated_fields(self): + from google.cloud.bigquery.dataset import DatasetReference + from google.cloud.bigquery.table import SchemaField + + PROJECT = 'PROJECT' + DS_ID = 'DS_ID' + TABLE_ID = 'TABLE_ID' + PATH = 'projects/%s/datasets/%s/tables/%s/data' % ( + PROJECT, DS_ID, TABLE_ID) + ROWS = 1234 + TOKEN = 'TOKEN' + DATA = { + 'totalRows': ROWS, + 'pageToken': TOKEN, + 'rows': [ + {'f': [ + {'v': [{'v': 'red'}, {'v': 'green'}]}, + {'v': [{ + 'v': { + 'f': [ + {'v': [{'v': '1'}, {'v': '2'}]}, + {'v': [{'v': '3.1415'}, {'v': '1.414'}]}, + ]} + }]}, + ]}, + ] + } + creds = _make_credentials() + http = object() + client = self._make_one(project=PROJECT, credentials=creds, _http=http) + conn = client._connection = _Connection(DATA) + table_ref = DatasetReference(PROJECT, DS_ID).table(TABLE_ID) + color = SchemaField('color', 'STRING', mode='REPEATED') + index = SchemaField('index', 'INTEGER', 'REPEATED') + score = SchemaField('score', 'FLOAT', 'REPEATED') + struct = SchemaField('struct', 'RECORD', mode='REPEATED', + fields=[index, score]) + + iterator = client.list_rows(table_ref, selected_fields=[color, struct]) + page = six.next(iterator.pages) + rows = list(page) + total_rows = iterator.total_rows + page_token = iterator.next_page_token + + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0][0], ['red', 'green']) + self.assertEqual(rows[0][1], [{'index': [1, 2], + 'score': [3.1415, 1.414]}]) + self.assertEqual(total_rows, ROWS) + self.assertEqual(page_token, TOKEN) + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], '/%s' % PATH) + + def test_list_rows_w_record_schema(self): + from google.cloud.bigquery.dataset import DatasetReference + from google.cloud.bigquery.table import Table, SchemaField + + PROJECT = 'PROJECT' + DS_ID = 'DS_ID' + TABLE_ID = 'TABLE_ID' + PATH = 'projects/%s/datasets/%s/tables/%s/data' % ( + PROJECT, DS_ID, TABLE_ID) + ROWS = 1234 + TOKEN = 'TOKEN' + DATA = { + 'totalRows': ROWS, + 'pageToken': TOKEN, + 'rows': [ + {'f': [ + {'v': 'Phred Phlyntstone'}, + {'v': {'f': [{'v': '800'}, {'v': '555-1212'}, {'v': 1}]}}, + ]}, + {'f': [ + {'v': 'Bharney Rhubble'}, + {'v': {'f': [{'v': '877'}, {'v': '768-5309'}, {'v': 2}]}}, + ]}, + {'f': [ + {'v': 'Wylma Phlyntstone'}, + {'v': None}, + ]}, + ] + } + creds = _make_credentials() + http = object() + client = self._make_one(project=PROJECT, credentials=creds, _http=http) + conn = client._connection = _Connection(DATA) + table_ref = DatasetReference(PROJECT, DS_ID).table(TABLE_ID) + full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') + area_code = SchemaField('area_code', 'STRING', 'REQUIRED') + local_number = SchemaField('local_number', 'STRING', 'REQUIRED') + rank = SchemaField('rank', 'INTEGER', 'REQUIRED') + phone = SchemaField('phone', 'RECORD', mode='NULLABLE', + fields=[area_code, local_number, rank]) + table = Table(table_ref, schema=[full_name, phone]) + + iterator = client.list_rows(table) + page = six.next(iterator.pages) + rows = list(page) + total_rows = iterator.total_rows + page_token = iterator.next_page_token + + self.assertEqual(len(rows), 3) + self.assertEqual(rows[0][0], 'Phred Phlyntstone') + self.assertEqual(rows[0][1], {'area_code': '800', + 'local_number': '555-1212', + 'rank': 1}) + self.assertEqual(rows[1][0], 'Bharney Rhubble') + self.assertEqual(rows[1][1], {'area_code': '877', + 'local_number': '768-5309', + 'rank': 2}) + self.assertEqual(rows[2][0], 'Wylma Phlyntstone') + self.assertIsNone(rows[2][1]) + self.assertEqual(total_rows, ROWS) + self.assertEqual(page_token, TOKEN) + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], '/%s' % PATH) + + def test_list_rows_errors(self): + from google.cloud.bigquery.dataset import DatasetReference + from google.cloud.bigquery.table import Table + + PROJECT = 'PROJECT' + DS_ID = 'DS_ID' + TABLE_ID = 'TABLE_ID' + + creds = _make_credentials() + http = object() + client = self._make_one(project=PROJECT, credentials=creds, _http=http) + table_ref = DatasetReference(PROJECT, DS_ID).table(TABLE_ID) + + # table ref with no selected fields + with self.assertRaises(ValueError): + client.list_rows(table_ref) + + # table with no schema + with self.assertRaises(ValueError): + client.list_rows(Table(table_ref)) + + # neither Table nor tableReference + with self.assertRaises(TypeError): + client.list_rows(1) + class _Connection(object): diff --git a/bigquery/tests/unit/test_table.py b/bigquery/tests/unit/test_table.py index c86c21880bda..fc0ff3370974 100644 --- a/bigquery/tests/unit/test_table.py +++ b/bigquery/tests/unit/test_table.py @@ -945,286 +945,6 @@ def test_update_w_alternate_client(self): self.assertEqual(req['data'], SENT) self._verifyResourceProperties(table, RESOURCE) - def test_fetch_data_wo_schema(self): - from google.cloud.bigquery.table import _TABLE_HAS_NO_SCHEMA - - client = _Client(project=self.PROJECT) - dataset = DatasetReference(self.PROJECT, self.DS_ID) - table_ref = dataset.table(self.TABLE_NAME) - table = self._make_one(table_ref, client=client) - - with self.assertRaises(ValueError) as exc: - table.fetch_data() - - self.assertEqual(exc.exception.args, (_TABLE_HAS_NO_SCHEMA,)) - - def test_fetch_data_w_bound_client(self): - import datetime - import six - from google.cloud._helpers import UTC - from google.cloud.bigquery.table import SchemaField - - PATH = 'projects/%s/datasets/%s/tables/%s/data' % ( - self.PROJECT, self.DS_ID, self.TABLE_NAME) - WHEN_TS = 1437767599.006 - WHEN = datetime.datetime.utcfromtimestamp(WHEN_TS).replace( - tzinfo=UTC) - WHEN_1 = WHEN + datetime.timedelta(seconds=1) - WHEN_2 = WHEN + datetime.timedelta(seconds=2) - ROWS = 1234 - TOKEN = 'TOKEN' - - def _bigquery_timestamp_float_repr(ts_float): - # Preserve microsecond precision for E+09 timestamps - return '%0.15E' % (ts_float,) - - DATA = { - 'totalRows': str(ROWS), - 'pageToken': TOKEN, - 'rows': [ - {'f': [ - {'v': 'Phred Phlyntstone'}, - {'v': '32'}, - {'v': _bigquery_timestamp_float_repr(WHEN_TS)}, - ]}, - {'f': [ - {'v': 'Bharney Rhubble'}, - {'v': '33'}, - {'v': _bigquery_timestamp_float_repr(WHEN_TS + 1)}, - ]}, - {'f': [ - {'v': 'Wylma Phlyntstone'}, - {'v': '29'}, - {'v': _bigquery_timestamp_float_repr(WHEN_TS + 2)}, - ]}, - {'f': [ - {'v': 'Bhettye Rhubble'}, - {'v': None}, - {'v': None}, - ]}, - ] - } - - conn = _Connection(DATA) - client = _Client(project=self.PROJECT, connection=conn) - dataset = DatasetReference(self.PROJECT, self.DS_ID) - table_ref = dataset.table(self.TABLE_NAME) - full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') - age = SchemaField('age', 'INTEGER', mode='NULLABLE') - joined = SchemaField('joined', 'TIMESTAMP', mode='NULLABLE') - table = self._make_one(table_ref, schema=[full_name, age, joined], - client=client) - - iterator = table.fetch_data() - page = six.next(iterator.pages) - rows = list(page) - total_rows = iterator.total_rows - page_token = iterator.next_page_token - - self.assertEqual(len(rows), 4) - self.assertEqual(rows[0], ('Phred Phlyntstone', 32, WHEN)) - self.assertEqual(rows[1], ('Bharney Rhubble', 33, WHEN_1)) - self.assertEqual(rows[2], ('Wylma Phlyntstone', 29, WHEN_2)) - self.assertEqual(rows[3], ('Bhettye Rhubble', None, None)) - self.assertEqual(total_rows, ROWS) - self.assertEqual(page_token, TOKEN) - - self.assertEqual(len(conn._requested), 1) - req = conn._requested[0] - self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) - - def test_fetch_data_w_alternate_client(self): - import six - from google.cloud.bigquery.table import SchemaField - - PATH = 'projects/%s/datasets/%s/tables/%s/data' % ( - self.PROJECT, self.DS_ID, self.TABLE_NAME) - MAX = 10 - TOKEN = 'TOKEN' - DATA = { - 'rows': [ - {'f': [ - {'v': 'Phred Phlyntstone'}, - {'v': '32'}, - {'v': 'true'}, - {'v': '3.1415926'}, - ]}, - {'f': [ - {'v': 'Bharney Rhubble'}, - {'v': '33'}, - {'v': 'false'}, - {'v': '1.414'}, - ]}, - {'f': [ - {'v': 'Wylma Phlyntstone'}, - {'v': '29'}, - {'v': 'true'}, - {'v': '2.71828'}, - ]}, - {'f': [ - {'v': 'Bhettye Rhubble'}, - {'v': '27'}, - {'v': None}, - {'v': None}, - ]}, - ] - } - conn1 = _Connection() - client1 = _Client(project=self.PROJECT, connection=conn1) - conn2 = _Connection(DATA) - client2 = _Client(project=self.PROJECT, connection=conn2) - dataset = DatasetReference(self.PROJECT, self.DS_ID) - table_ref = dataset.table(self.TABLE_NAME) - full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') - age = SchemaField('age', 'INTEGER', mode='REQUIRED') - voter = SchemaField('voter', 'BOOLEAN', mode='NULLABLE') - score = SchemaField('score', 'FLOAT', mode='NULLABLE') - table = self._make_one(table_ref, - schema=[full_name, age, voter, score], - client=client1) - - iterator = table.fetch_data( - client=client2, max_results=MAX, page_token=TOKEN) - page = six.next(iterator.pages) - rows = list(page) - total_rows = iterator.total_rows - page_token = iterator.next_page_token - - self.assertEqual(len(rows), 4) - self.assertEqual(rows[0], ('Phred Phlyntstone', 32, True, 3.1415926)) - self.assertEqual(rows[1], ('Bharney Rhubble', 33, False, 1.414)) - self.assertEqual(rows[2], ('Wylma Phlyntstone', 29, True, 2.71828)) - self.assertEqual(rows[3], ('Bhettye Rhubble', 27, None, None)) - self.assertIsNone(total_rows) - self.assertIsNone(page_token) - - self.assertEqual(len(conn1._requested), 0) - self.assertEqual(len(conn2._requested), 1) - req = conn2._requested[0] - self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) - self.assertEqual(req['query_params'], - {'maxResults': MAX, 'pageToken': TOKEN}) - - def test_fetch_data_w_repeated_fields(self): - import six - from google.cloud.bigquery.table import SchemaField - - PATH = 'projects/%s/datasets/%s/tables/%s/data' % ( - self.PROJECT, self.DS_ID, self.TABLE_NAME) - ROWS = 1234 - TOKEN = 'TOKEN' - DATA = { - 'totalRows': ROWS, - 'pageToken': TOKEN, - 'rows': [ - {'f': [ - {'v': [{'v': 'red'}, {'v': 'green'}]}, - {'v': [{ - 'v': { - 'f': [ - {'v': [{'v': '1'}, {'v': '2'}]}, - {'v': [{'v': '3.1415'}, {'v': '1.414'}]}, - ]} - }]}, - ]}, - ] - } - conn = _Connection(DATA) - client = _Client(project=self.PROJECT, connection=conn) - dataset = DatasetReference(self.PROJECT, self.DS_ID) - table_ref = dataset.table(self.TABLE_NAME) - color = SchemaField('color', 'STRING', mode='REPEATED') - index = SchemaField('index', 'INTEGER', 'REPEATED') - score = SchemaField('score', 'FLOAT', 'REPEATED') - struct = SchemaField('struct', 'RECORD', mode='REPEATED', - fields=[index, score]) - table = self._make_one(table_ref, schema=[color, struct], - client=client) - - iterator = table.fetch_data() - page = six.next(iterator.pages) - rows = list(page) - total_rows = iterator.total_rows - page_token = iterator.next_page_token - - self.assertEqual(len(rows), 1) - self.assertEqual(rows[0][0], ['red', 'green']) - self.assertEqual(rows[0][1], [{'index': [1, 2], - 'score': [3.1415, 1.414]}]) - self.assertEqual(total_rows, ROWS) - self.assertEqual(page_token, TOKEN) - - self.assertEqual(len(conn._requested), 1) - req = conn._requested[0] - self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) - - def test_fetch_data_w_record_schema(self): - import six - from google.cloud.bigquery.table import SchemaField - - PATH = 'projects/%s/datasets/%s/tables/%s/data' % ( - self.PROJECT, self.DS_ID, self.TABLE_NAME) - ROWS = 1234 - TOKEN = 'TOKEN' - DATA = { - 'totalRows': ROWS, - 'pageToken': TOKEN, - 'rows': [ - {'f': [ - {'v': 'Phred Phlyntstone'}, - {'v': {'f': [{'v': '800'}, {'v': '555-1212'}, {'v': 1}]}}, - ]}, - {'f': [ - {'v': 'Bharney Rhubble'}, - {'v': {'f': [{'v': '877'}, {'v': '768-5309'}, {'v': 2}]}}, - ]}, - {'f': [ - {'v': 'Wylma Phlyntstone'}, - {'v': None}, - ]}, - ] - } - conn = _Connection(DATA) - client = _Client(project=self.PROJECT, connection=conn) - dataset = DatasetReference(self.PROJECT, self.DS_ID) - table_ref = dataset.table(self.TABLE_NAME) - full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') - area_code = SchemaField('area_code', 'STRING', 'REQUIRED') - local_number = SchemaField('local_number', 'STRING', 'REQUIRED') - rank = SchemaField('rank', 'INTEGER', 'REQUIRED') - phone = SchemaField('phone', 'RECORD', mode='NULLABLE', - fields=[area_code, local_number, rank]) - table = self._make_one(table_ref, schema=[full_name, phone], - client=client) - - iterator = table.fetch_data() - page = six.next(iterator.pages) - rows = list(page) - total_rows = iterator.total_rows - page_token = iterator.next_page_token - - self.assertEqual(len(rows), 3) - self.assertEqual(rows[0][0], 'Phred Phlyntstone') - self.assertEqual(rows[0][1], {'area_code': '800', - 'local_number': '555-1212', - 'rank': 1}) - self.assertEqual(rows[1][0], 'Bharney Rhubble') - self.assertEqual(rows[1][1], {'area_code': '877', - 'local_number': '768-5309', - 'rank': 2}) - self.assertEqual(rows[2][0], 'Wylma Phlyntstone') - self.assertIsNone(rows[2][1]) - self.assertEqual(total_rows, ROWS) - self.assertEqual(page_token, TOKEN) - - self.assertEqual(len(conn._requested), 1) - req = conn._requested[0] - self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) - def test_row_from_mapping_wo_schema(self): from google.cloud.bigquery.table import _TABLE_HAS_NO_SCHEMA MAPPING = {'full_name': 'Phred Phlyntstone', 'age': 32}