From 2bb9536a1da813d6017ae5dbe7937333cf396d4c Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Wed, 13 Sep 2017 09:32:14 -0700 Subject: [PATCH 1/5] WIP BigQuery client.get_dataset() --- bigquery/google/cloud/bigquery/client.py | 11 ++++++ bigquery/google/cloud/bigquery/dataset.py | 11 +++++- bigquery/tests/unit/test_client.py | 44 +++++++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index 2ae577a51708..091f790288bf 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -162,6 +162,17 @@ def dataset(self, dataset_name, project=None): """ return Dataset(dataset_name, client=self, project=project) + def get_dataset(self, dataset_ref): + """API call: refresh dataset properties via a GET request. + + :type dataset_ref: + :class:`google.cloud.bigquery.dataset.DatasetReference` + :param dataset_ref: the dataset to use. + """ + api_response = self._connection.api_request( + method='GET', path=dataset_ref.path) + Dataset.from_api_repr(api_response, self) + def _get_query_results(self, job_id, project=None, timeout_ms=None): """Get the query results object for a query job. diff --git a/bigquery/google/cloud/bigquery/dataset.py b/bigquery/google/cloud/bigquery/dataset.py index e31b4a2a93b1..bf216aab38c6 100644 --- a/bigquery/google/cloud/bigquery/dataset.py +++ b/bigquery/google/cloud/bigquery/dataset.py @@ -127,6 +127,15 @@ def dataset_id(self): """ return self._dataset_id + @property + def path(self): + """URL path for the dataset's APIs. + + :rtype: str + :returns: the path based on project and dataset name. + """ + return '/projects/%s/datasets/%s' % (self.project_id, self.dataset_id) + def table(self, table_id): """Constructs a TableReference. @@ -181,7 +190,7 @@ def path(self): """URL path for the dataset's APIs. :rtype: str - :returns: the path based on project and dataste name. + :returns: the path based on project and dataset name. """ return '/projects/%s/datasets/%s' % (self.project, self.name) diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index 3cd4a24ceb43..d814e0da0b46 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -259,6 +259,50 @@ def test_dataset(self): self.assertEqual(dataset.name, DATASET) self.assertIs(dataset._client, client) + def test_get_dataset(self): + project = 'PROJECT' + dataset_id = 'dataset_id' + path = 'projects/%s/datasets/%s' % (project, dataset_id) + creds = _make_credentials() + http = object() + client = self._make_one(project=project, credentials=creds, _http=http) + DS_ID = '%s:%s' % (project, dataset_id) + resource = { + 'id' : DS_ID, + 'datasetReference': { + 'projectId': project, + 'datasetId': dataset_id, + }, + } + conn = client._connection = _Connection(resource) + dataset_ref = client.dataset(dataset_id) + + dataset = client.get_dataset(dataset_ref) + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], '/%s' % path) + self.assertEqual(dataset.name, dataset_id) + + # def test_reload_w_alternate_client(self): + # PATH = 'projects/%s/datasets/%s' % (self.PROJECT, self.DS_NAME) + # RESOURCE = self._makeResource() + # conn1 = _Connection() + # CLIENT1 = _Client(project=self.PROJECT, connection=conn1) + # conn2 = _Connection(RESOURCE) + # CLIENT2 = _Client(project=self.PROJECT, connection=conn2) + # dataset = self._make_one(self.DS_NAME, client=CLIENT1) + # + # dataset.reload(client=CLIENT2) + # + # self.assertEqual(len(conn1._requested), 0) + # self.assertEqual(len(conn2._requested), 1) + # req = conn2._requested[0] + # self.assertEqual(req['method'], 'GET') + # self.assertEqual(req['path'], '/%s' % PATH) + # self._verify_resource_properties(dataset, RESOURCE) + def test_job_from_resource_unknown_type(self): PROJECT = 'PROJECT' creds = _make_credentials() From 7f5b4e6bc33cee0290595c775e1ebcd5a6d3ad98 Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Thu, 14 Sep 2017 10:42:16 -0700 Subject: [PATCH 2/5] BigQuery: Adds client.get_dataset() --- bigquery/google/cloud/bigquery/client.py | 5 ++++- bigquery/tests/unit/test_client.py | 18 ------------------ 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index 091f790288bf..317becf67629 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -168,10 +168,13 @@ def get_dataset(self, dataset_ref): :type dataset_ref: :class:`google.cloud.bigquery.dataset.DatasetReference` :param dataset_ref: the dataset to use. + + :rtype: :class:`google.cloud.bigquery.dataset.Dataset` + :returns: a ``Dataset`` instance """ api_response = self._connection.api_request( method='GET', path=dataset_ref.path) - Dataset.from_api_repr(api_response, self) + return Dataset.from_api_repr(api_response, self) def _get_query_results(self, job_id, project=None, timeout_ms=None): """Get the query results object for a query job. diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index d814e0da0b46..22fd9770765f 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -285,24 +285,6 @@ def test_get_dataset(self): self.assertEqual(req['path'], '/%s' % path) self.assertEqual(dataset.name, dataset_id) - # def test_reload_w_alternate_client(self): - # PATH = 'projects/%s/datasets/%s' % (self.PROJECT, self.DS_NAME) - # RESOURCE = self._makeResource() - # conn1 = _Connection() - # CLIENT1 = _Client(project=self.PROJECT, connection=conn1) - # conn2 = _Connection(RESOURCE) - # CLIENT2 = _Client(project=self.PROJECT, connection=conn2) - # dataset = self._make_one(self.DS_NAME, client=CLIENT1) - # - # dataset.reload(client=CLIENT2) - # - # self.assertEqual(len(conn1._requested), 0) - # self.assertEqual(len(conn2._requested), 1) - # req = conn2._requested[0] - # self.assertEqual(req['method'], 'GET') - # self.assertEqual(req['path'], '/%s' % PATH) - # self._verify_resource_properties(dataset, RESOURCE) - def test_job_from_resource_unknown_type(self): PROJECT = 'PROJECT' creds = _make_credentials() From 4ff6cdb73c0e8154e450eb1c19dd82a03693bc4b Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Thu, 14 Sep 2017 12:01:50 -0700 Subject: [PATCH 3/5] BigQuery: Adds client.get_dataset() and removes dataset.reload() --- bigquery/google/cloud/bigquery/dataset.py | 17 ------------ bigquery/tests/system.py | 17 ++++++------ bigquery/tests/unit/test_client.py | 2 +- bigquery/tests/unit/test_dataset.py | 33 ----------------------- 4 files changed, 10 insertions(+), 59 deletions(-) diff --git a/bigquery/google/cloud/bigquery/dataset.py b/bigquery/google/cloud/bigquery/dataset.py index bf216aab38c6..9bba1c112558 100644 --- a/bigquery/google/cloud/bigquery/dataset.py +++ b/bigquery/google/cloud/bigquery/dataset.py @@ -514,23 +514,6 @@ def exists(self, client=None): else: return True - def reload(self, client=None): - """API call: refresh dataset properties via a GET request. - - See - https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets/get - - :type client: :class:`~google.cloud.bigquery.client.Client` or - ``NoneType`` - :param client: the client to use. If not passed, falls back to the - ``client`` stored on the current dataset. - """ - client = self._require_client(client) - - api_response = client._connection.api_request( - method='GET', path=self.path) - self._set_properties(api_response) - def patch(self, client=None, **kw): """API call: update individual dataset properties via a PATCH request. diff --git a/bigquery/tests/system.py b/bigquery/tests/system.py index 1974418461ce..0aae92d73f8a 100644 --- a/bigquery/tests/system.py +++ b/bigquery/tests/system.py @@ -120,19 +120,20 @@ def test_create_dataset(self): self.assertTrue(dataset.exists()) self.assertEqual(dataset.name, DATASET_NAME) - def test_reload_dataset(self): - DATASET_NAME = _make_dataset_name('reload_dataset') - dataset = Config.CLIENT.dataset(DATASET_NAME) + def test_get_dataset(self): + DATASET_NAME = _make_dataset_name('get_dataset') + client = Config.CLIENT + dataset = Dataset(DATASET_NAME, client) dataset.friendly_name = 'Friendly' dataset.description = 'Description' - retry_403(dataset.create)() self.to_delete.append(dataset) + dataset_ref = client.dataset(DATASET_NAME) + + got = client.get_dataset(dataset_ref) - other = Config.CLIENT.dataset(DATASET_NAME) - other.reload() - self.assertEqual(other.friendly_name, 'Friendly') - self.assertEqual(other.description, 'Description') + self.assertEqual(got.friendly_name, 'Friendly') + self.assertEqual(got.description, 'Description') def test_patch_dataset(self): dataset = Config.CLIENT.dataset(_make_dataset_name('patch_dataset')) diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index 22fd9770765f..c500146e1176 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -268,7 +268,7 @@ def test_get_dataset(self): client = self._make_one(project=project, credentials=creds, _http=http) DS_ID = '%s:%s' % (project, dataset_id) resource = { - 'id' : DS_ID, + 'id': DS_ID, 'datasetReference': { 'projectId': project, 'datasetId': dataset_id, diff --git a/bigquery/tests/unit/test_dataset.py b/bigquery/tests/unit/test_dataset.py index c509be6838a1..a060be5e15ad 100644 --- a/bigquery/tests/unit/test_dataset.py +++ b/bigquery/tests/unit/test_dataset.py @@ -520,39 +520,6 @@ def test_exists_hit_w_alternate_client(self): self.assertEqual(req['path'], '/%s' % PATH) self.assertEqual(req['query_params'], {'fields': 'id'}) - def test_reload_w_bound_client(self): - PATH = 'projects/%s/datasets/%s' % (self.PROJECT, self.DS_NAME) - RESOURCE = self._makeResource() - conn = _Connection(RESOURCE) - client = _Client(project=self.PROJECT, connection=conn) - dataset = self._make_one(self.DS_NAME, client=client) - - dataset.reload() - - self.assertEqual(len(conn._requested), 1) - req = conn._requested[0] - self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) - self._verify_resource_properties(dataset, RESOURCE) - - def test_reload_w_alternate_client(self): - PATH = 'projects/%s/datasets/%s' % (self.PROJECT, self.DS_NAME) - RESOURCE = self._makeResource() - conn1 = _Connection() - CLIENT1 = _Client(project=self.PROJECT, connection=conn1) - conn2 = _Connection(RESOURCE) - CLIENT2 = _Client(project=self.PROJECT, connection=conn2) - dataset = self._make_one(self.DS_NAME, client=CLIENT1) - - dataset.reload(client=CLIENT2) - - self.assertEqual(len(conn1._requested), 0) - self.assertEqual(len(conn2._requested), 1) - req = conn2._requested[0] - self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) - self._verify_resource_properties(dataset, RESOURCE) - def test_patch_w_invalid_expiration(self): RESOURCE = self._makeResource() conn = _Connection(RESOURCE) From 238363a3aebc6dcdbbfe4ec591fa61eb3b3f1f28 Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Fri, 15 Sep 2017 10:54:44 -0700 Subject: [PATCH 4/5] BigQuery: changes dataset.name to dataset.dataset_id in test --- bigquery/tests/unit/test_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index 0c3cc1141959..41ab3d21f1d7 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -296,7 +296,7 @@ def test_get_dataset(self): req = conn._requested[0] self.assertEqual(req['method'], 'GET') self.assertEqual(req['path'], '/%s' % path) - self.assertEqual(dataset.name, dataset_id) + self.assertEqual(dataset.dataset_id, dataset_id) def test_job_from_resource_unknown_type(self): PROJECT = 'PROJECT' From 811c9b03d5825d514347e16ff43d8e1948906a2f Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Fri, 15 Sep 2017 13:38:43 -0700 Subject: [PATCH 5/5] fixes client.get_dataset() docstring and removes unnecessary test variable --- bigquery/google/cloud/bigquery/client.py | 2 +- bigquery/tests/unit/test_client.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index 5b12bacc3889..f9a393c0bbb6 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -167,7 +167,7 @@ def dataset(self, dataset_name, project=None): return DatasetReference(project, dataset_name) def get_dataset(self, dataset_ref): - """API call: refresh dataset properties via a GET request. + """Fetch the dataset referenced by ``dataset_ref`` :type dataset_ref: :class:`google.cloud.bigquery.dataset.DatasetReference` diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index 41ab3d21f1d7..55a673c5d1c4 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -279,9 +279,8 @@ def test_get_dataset(self): creds = _make_credentials() http = object() client = self._make_one(project=project, credentials=creds, _http=http) - DS_ID = '%s:%s' % (project, dataset_id) resource = { - 'id': DS_ID, + 'id': '%s:%s' % (project, dataset_id), 'datasetReference': { 'projectId': project, 'datasetId': dataset_id,