From 54560887f01cce32741467495495a59a614ada0c Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Sat, 12 Aug 2017 19:48:21 -0400 Subject: [PATCH 1/6] Allow assigning 'None' to '_TypedProperty' properties. --- bigquery/google/cloud/bigquery/_helpers.py | 26 ++++++++++++ bigquery/tests/unit/test__helpers.py | 47 ++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index 9358229e630a..f5366e395d9a 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -299,6 +299,8 @@ def _validate(self, value): :raises: ValueError on a type mismatch. """ + if value is None: + return if not isinstance(value, self.property_type): raise ValueError('Required type: %s' % (self.property_type,)) @@ -396,6 +398,14 @@ def __init__(self, name, type_, value): self.type_ = type_ self.value = value + def __eq__(self, other): + if not isinstance(other, ScalarQueryParameter): + return NotImplemented + return( + self.name == other.name and + self.type_ == other.type_ and + self.value == other.value) + @classmethod def positional(cls, type_, value): """Factory for positional paramater. @@ -473,6 +483,14 @@ def __init__(self, name, array_type, values): self.array_type = array_type self.values = values + def __eq__(self, other): + if not isinstance(other, ArrayQueryParameter): + return NotImplemented + return( + self.name == other.name and + self.array_type == other.array_type and + self.values == other.values) + @classmethod def positional(cls, array_type, values): """Factory for positional parameters. @@ -566,6 +584,14 @@ def __init__(self, name, *sub_params): types[sub.name] = sub.type_ values[sub.name] = sub.value + def __eq__(self, other): + if not isinstance(other, StructQueryParameter): + return NotImplemented + return( + self.name == other.name and + self.struct_types == other.struct_types and + self.struct_values == other.struct_values) + @classmethod def positional(cls, *sub_params): """Factory for positional parameters. diff --git a/bigquery/tests/unit/test__helpers.py b/bigquery/tests/unit/test__helpers.py index 581b4b9a42fc..86f0ee225a0e 100644 --- a/bigquery/tests/unit/test__helpers.py +++ b/bigquery/tests/unit/test__helpers.py @@ -751,6 +751,14 @@ def __init__(self): self.assertEqual(wrapper.attr, 42) self.assertEqual(wrapper._configuration._attr, 42) + wrapper.attr = None + self.assertIsNone(wrapper.attr) + self.assertIsNone(wrapper._configuration._attr) + + wrapper.attr = 23 + self.assertEqual(wrapper.attr, 23) + self.assertEqual(wrapper._configuration._attr, 23) + del wrapper.attr self.assertIsNone(wrapper.attr) self.assertIsNone(wrapper._configuration._attr) @@ -914,6 +922,17 @@ def test_ctor(self): self.assertEqual(param.type_, 'INT64') self.assertEqual(param.value, 123) + def test___eq__(self): + param = self._make_one(name='foo', type_='INT64', value=123) + self.assertEqual(param, param) + self.assertNotEqual(param, object()) + alias = self._make_one(name='bar', type_='INT64', value=123) + self.assertNotEqual(param, alias) + wrong_type = self._make_one(name='foo', type_='FLOAT64', value=123.0) + self.assertNotEqual(param, wrong_type) + wrong_val = self._make_one(name='foo', type_='INT64', value=234) + self.assertNotEqual(param, wrong_val) + def test_positional(self): klass = self._get_target_class() param = klass.positional(type_='INT64', value=123) @@ -1145,6 +1164,19 @@ def test_ctor(self): self.assertEqual(param.array_type, 'INT64') self.assertEqual(param.values, [1, 2]) + def test___eq__(self): + param = self._make_one(name='foo', array_type='INT64', values=[123]) + self.assertEqual(param, param) + self.assertNotEqual(param, object()) + alias = self._make_one(name='bar', array_type='INT64', values=[123]) + self.assertNotEqual(param, alias) + wrong_type = self._make_one( + name='foo', array_type='FLOAT64', values=[123.0]) + self.assertNotEqual(param, wrong_type) + wrong_val = self._make_one( + name='foo', array_type='INT64', values=[234]) + self.assertNotEqual(param, wrong_val) + def test_positional(self): klass = self._get_target_class() param = klass.positional(array_type='INT64', values=[1, 2]) @@ -1319,6 +1351,21 @@ def test_ctor(self): self.assertEqual(param.struct_types, {'bar': 'INT64', 'baz': 'STRING'}) self.assertEqual(param.struct_values, {'bar': 123, 'baz': 'abc'}) + def test___eq__(self): + sub_1 = _make_subparam('bar', 'INT64', 123) + sub_2 = _make_subparam('baz', 'STRING', 'abc') + sub_3 = _make_subparam('baz', 'STRING', 'def') + sub_1_float = _make_subparam('bar', 'FLOAT64', 123.0) + param = self._make_one('foo', sub_1, sub_2) + self.assertEqual(param, param) + self.assertNotEqual(param, object()) + alias = self._make_one('bar', sub_1, sub_2) + self.assertNotEqual(param, alias) + wrong_type = self._make_one( 'foo', sub_1_float, sub_2) + self.assertNotEqual(param, wrong_type) + wrong_val = self._make_one('foo', sub_2, sub_3) + self.assertNotEqual(param, wrong_val) + def test_positional(self): sub_1 = _make_subparam('bar', 'INT64', 123) sub_2 = _make_subparam('baz', 'STRING', 'abc') From 6059d3469d267c6f1d30f4d974ac96979f27b467 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Sat, 12 Aug 2017 19:49:44 -0400 Subject: [PATCH 2/6] Ensure that configuration properties are copied when (re)loading jobs. --- bigquery/google/cloud/bigquery/_helpers.py | 12 +++ bigquery/google/cloud/bigquery/job.py | 100 ++++++++++++++++++++- bigquery/tests/unit/test__helpers.py | 76 ++++++++++++++++ bigquery/tests/unit/test_job.py | 77 +++++++++++++--- 4 files changed, 250 insertions(+), 15 deletions(-) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index f5366e395d9a..9326bf5e70d5 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -662,6 +662,18 @@ def to_api_repr(self): return resource +def _query_param_from_api_repr(resource): + """Helper: construct concrete query parameter from JSON resource.""" + qp_type = resource['parameterType'] + if 'arrayType' in qp_type: + klass = ArrayQueryParameter + elif 'structTypes' in qp_type: + klass = StructQueryParameter + else: + klass = ScalarQueryParameter + return klass.from_api_repr(resource) + + class QueryParametersProperty(object): """Custom property type, holding query parameter instances.""" diff --git a/bigquery/google/cloud/bigquery/job.py b/bigquery/google/cloud/bigquery/job.py index 663d77501694..077badccc6ef 100644 --- a/bigquery/google/cloud/bigquery/job.py +++ b/bigquery/google/cloud/bigquery/job.py @@ -32,8 +32,10 @@ from google.cloud.bigquery._helpers import QueryParametersProperty from google.cloud.bigquery._helpers import ScalarQueryParameter from google.cloud.bigquery._helpers import StructQueryParameter +from google.cloud.bigquery._helpers import UDFResource from google.cloud.bigquery._helpers import UDFResourcesProperty from google.cloud.bigquery._helpers import _EnumProperty +from google.cloud.bigquery._helpers import _query_param_from_api_repr from google.cloud.bigquery._helpers import _TypedProperty _DONE_STATE = 'DONE' @@ -61,6 +63,22 @@ } +def _bool_or_none(value): + """Helper: deserialize boolean value from JSON string.""" + if isinstance(value, bool): + return value + if value is not None: + return value.lower() in ['t', 'true', '1'] + + +def _int_or_none(value): + """Helper: deserialize int value from JSON string.""" + if isinstance(value, int): + return value + if value is not None: + return int(value) + + def _error_result_to_exception(error_result): """Maps BigQuery error reasons to an exception. @@ -311,6 +329,10 @@ def _scrub_local_properties(self, cleaned): """Helper: handle subclass properties in cleaned.""" pass + def _copy_configuration_properties(self, configuration): + """Helper: assign subclass configuration properties in cleaned.""" + raise NotImplementedError("Abstract") + def _set_properties(self, api_response): """Update properties from resource in body of ``api_response`` @@ -330,6 +352,8 @@ def _set_properties(self, api_response): self._properties.clear() self._properties.update(cleaned) + configuration = cleaned['configuration'][self._JOB_TYPE] + self._copy_configuration_properties(configuration) # For Future interface self._set_future_result() @@ -769,6 +793,28 @@ def _scrub_local_properties(self, cleaned): schema = cleaned.pop('schema', {'fields': ()}) self.schema = _parse_schema_resource(schema) + def _copy_configuration_properties(self, configuration): + """Helper: assign subclass configuration properties in cleaned.""" + self.allow_jagged_rows = _bool_or_none( + configuration.get('allowJaggedRows')) + self.allow_quoted_newlines = _bool_or_none( + configuration.get('allowQuotedNewlines')) + self.autodetect = _bool_or_none( + configuration.get('autodetect')) + self.create_disposition = configuration.get('createDisposition') + self.encoding = configuration.get('encoding') + self.field_delimiter = configuration.get('fieldDelimiter') + self.ignore_unknown_values = _bool_or_none( + configuration.get('ignoreUnknownValues')) + self.max_bad_records = _int_or_none( + configuration.get('maxBadRecords')) + self.null_marker = configuration.get('nullMarker') + self.quote_character = configuration.get('quote') + self.skip_leading_rows = _int_or_none( + configuration.get('skipLeadingRows')) + self.source_format = configuration.get('sourceFormat') + self.write_disposition = configuration.get('writeDisposition') + @classmethod def from_api_repr(cls, resource, client): """Factory: construct a job given its API representation @@ -879,6 +925,11 @@ def _build_resource(self): return resource + def _copy_configuration_properties(self, configuration): + """Helper: assign subclass configuration properties in cleaned.""" + self.create_disposition = configuration.get('createDisposition') + self.write_disposition = configuration.get('writeDisposition') + @classmethod def from_api_repr(cls, resource, client): """Factory: construct a job given its API representation @@ -1012,6 +1063,14 @@ def _build_resource(self): return resource + def _copy_configuration_properties(self, configuration): + """Helper: assign subclass configuration properties in cleaned.""" + self.compression = configuration.get('compression') + self.destination_format = configuration.get('destinationFormat') + self.field_delimiter = configuration.get('fieldDelimiter') + self.print_header = _bool_or_none( + configuration.get('printHeader')) + @classmethod def from_api_repr(cls, resource, client): """Factory: construct a job given its API representation @@ -1257,6 +1316,24 @@ def _scrub_local_properties(self, cleaned): configuration = cleaned['configuration']['query'] self.query = configuration['query'] + + def _copy_configuration_properties(self, configuration): + """Helper: assign subclass configuration properties in cleaned.""" + self.allow_large_results = _bool_or_none( + configuration.get('allowLargeResults')) + self.flatten_results = _bool_or_none( + configuration.get('flattenResults')) + self.use_query_cache = _bool_or_none( + configuration.get('useQueryCache')) + self.use_legacy_sql = _bool_or_none( + configuration.get('useLegacySql')) + + self.create_disposition = configuration.get('createDisposition') + self.priority = configuration.get('priority') + self.write_disposition = configuration.get('writeDisposition') + self.maximum_billing_tier = configuration.get('maximumBillingTier') + self.maximum_bytes_billed = configuration.get('maximumBytesBilled') + dest_remote = configuration.get('destinationTable') if dest_remote is None: @@ -1265,9 +1342,30 @@ def _scrub_local_properties(self, cleaned): else: dest_local = self._destination_table_resource() if dest_remote != dest_local: - dataset = self._client.dataset(dest_remote['datasetId']) + project = dest_remote['projectId'] + dataset = self._client.dataset( + dest_remote['datasetId'], project=project) self.destination = dataset.table(dest_remote['tableId']) + def_ds = configuration.get('defaultDataset') + if def_ds is None: + if self.default_dataset is not None: + del self.default_dataset + else: + project = def_ds['projectId'] + self.default_dataset = self._client.dataset(def_ds['datasetId']) + + udf_resources = [] + for udf_mapping in configuration.get(self._UDF_KEY, ()): + key_val, = udf_mapping.items() + udf_resources.append(UDFResource(key_val[0], key_val[1])) + self._udf_resources = udf_resources + + self._query_parameters = [ + _query_param_from_api_repr(mapping) + for mapping in configuration.get(self._QUERY_PARAMETERS_KEY, ()) + ] + @classmethod def from_api_repr(cls, resource, client): """Factory: construct a job given its API representation diff --git a/bigquery/tests/unit/test__helpers.py b/bigquery/tests/unit/test__helpers.py index 86f0ee225a0e..36efb51a6585 100644 --- a/bigquery/tests/unit/test__helpers.py +++ b/bigquery/tests/unit/test__helpers.py @@ -1528,6 +1528,82 @@ def test_to_api_repr_w_nested_struct(self): self.assertEqual(param.to_api_repr(), EXPECTED) +class Test__query_param_from_api_repr(unittest.TestCase): + + @staticmethod + def _call_fut(resource): + from google.cloud.bigquery._helpers import _query_param_from_api_repr + + return _query_param_from_api_repr(resource) + + def test_w_scalar(self): + from google.cloud.bigquery._helpers import ScalarQueryParameter + + RESOURCE = { + 'name': 'foo', + 'parameterType': {'type': 'INT64'}, + 'parameterValue': {'value': '123'}, + } + + parameter = self._call_fut(RESOURCE) + + self.assertIsInstance(parameter, ScalarQueryParameter) + self.assertEqual(parameter.name, 'foo') + self.assertEqual(parameter.type_, 'INT64') + self.assertEqual(parameter.value, 123) + + def test_w_array(self): + from google.cloud.bigquery._helpers import ArrayQueryParameter + + RESOURCE = { + 'name': 'foo', + 'parameterType': { + 'type': 'ARRAY', + 'arrayType': {'type': 'INT64'}, + }, + 'parameterValue': { + 'arrayValues': [ + {'value': '123'}, + ]}, + } + + parameter = self._call_fut(RESOURCE) + + self.assertIsInstance(parameter, ArrayQueryParameter) + self.assertEqual(parameter.name, 'foo') + self.assertEqual(parameter.array_type, 'INT64') + self.assertEqual(parameter.values, [123]) + + def test_w_struct(self): + from google.cloud.bigquery._helpers import StructQueryParameter + + RESOURCE = { + 'name': 'foo', + 'parameterType': { + 'type': 'STRUCT', + 'structTypes': [ + {'name': 'foo', 'type': {'type': 'STRING'}}, + {'name': 'bar', 'type': {'type': 'INT64'}}, + ], + }, + 'parameterValue': { + 'structValues': { + 'foo': {'value': 'Foo'}, + 'bar': {'value': '123'}, + } + }, + } + + parameter = self._call_fut(RESOURCE) + + self.assertIsInstance(parameter, StructQueryParameter) + self.assertEqual(parameter.name, 'foo') + self.assertEqual( + parameter.struct_types, {'foo': 'STRING', 'bar': 'INT64'}) + self.assertEqual(parameter.struct_values, {'foo': 'Foo', 'bar': 123}) + + + class Test_QueryParametersProperty(unittest.TestCase): @staticmethod diff --git a/bigquery/tests/unit/test_job.py b/bigquery/tests/unit/test_job.py index 01745aa494c9..bd5afd36be94 100644 --- a/bigquery/tests/unit/test_job.py +++ b/bigquery/tests/unit/test_job.py @@ -18,9 +18,49 @@ import unittest +class Test__bool_or_none(unittest.TestCase): + + def _call_fut(self, *args, **kwargs): + from google.cloud.bigquery import job + + return job._bool_or_none(*args, **kwargs) + + def test_w_bool(self): + self.assertTrue(self._call_fut(True)) + self.assertFalse(self._call_fut(False)) + + def test_w_none(self): + self.assertIsNone(self._call_fut(None)) + + def test_w_str(self): + self.assertTrue(self._call_fut('1')) + self.assertTrue(self._call_fut('t')) + self.assertTrue(self._call_fut('true')) + self.assertFalse(self._call_fut('anything else')) + + +class Test__int_or_none(unittest.TestCase): + + def _call_fut(self, *args, **kwargs): + from google.cloud.bigquery import job + + return job._int_or_none(*args, **kwargs) + + def test_w_int(self): + self.assertEqual(self._call_fut(13), 13) + + def test_w_none(self): + self.assertIsNone(self._call_fut(None)) + + def test_w_str(self): + self.assertEqual(self._call_fut('13'), 13) + + class Test__error_result_to_exception(unittest.TestCase): + def _call_fut(self, *args, **kwargs): from google.cloud.bigquery import job + return job._error_result_to_exception(*args, **kwargs) def test_simple(self): @@ -485,10 +525,12 @@ def test_from_api_repr_bare(self): def test_from_api_repr_w_properties(self): client = _Client(self.PROJECT) RESOURCE = self._makeResource() + load_config = RESOURCE['configuration']['load'] + load_config['createDisposition'] = 'CREATE_IF_NEEDED' klass = self._get_target_class() - dataset = klass.from_api_repr(RESOURCE, client=client) - self.assertIs(dataset._client, client) - self._verifyResourceProperties(dataset, RESOURCE) + job = klass.from_api_repr(RESOURCE, client=client) + self.assertIs(job._client, client) + self._verifyResourceProperties(job, RESOURCE) def test_begin_w_already_running(self): conn = _Connection() @@ -941,10 +983,12 @@ def test_from_api_repr_wo_sources(self): def test_from_api_repr_w_properties(self): client = _Client(self.PROJECT) RESOURCE = self._makeResource() + copy_config = RESOURCE['configuration']['copy'] + copy_config['createDisposition'] = 'CREATE_IF_NEEDED' klass = self._get_target_class() - dataset = klass.from_api_repr(RESOURCE, client=client) - self.assertIs(dataset._client, client) - self._verifyResourceProperties(dataset, RESOURCE) + job = klass.from_api_repr(RESOURCE, client=client) + self.assertIs(job._client, client) + self._verifyResourceProperties(job, RESOURCE) def test_begin_w_bound_client(self): PATH = '/projects/%s/jobs' % (self.PROJECT,) @@ -1240,10 +1284,12 @@ def test_from_api_repr_bare(self): def test_from_api_repr_w_properties(self): client = _Client(self.PROJECT) RESOURCE = self._makeResource() + extract_config = RESOURCE['configuration']['extract'] + extract_config['compression'] = 'GZIP' klass = self._get_target_class() - dataset = klass.from_api_repr(RESOURCE, client=client) - self.assertIs(dataset._client, client) - self._verifyResourceProperties(dataset, RESOURCE) + job = klass.from_api_repr(RESOURCE, client=client) + self.assertIs(job._client, client) + self._verifyResourceProperties(job, RESOURCE) def test_begin_w_bound_client(self): PATH = '/projects/%s/jobs' % (self.PROJECT,) @@ -1607,7 +1653,7 @@ def test_from_api_repr_bare(self): 'jobId': self.JOB_NAME, }, 'configuration': { - 'query': {'query': self.QUERY} + 'query': {'query': self.QUERY}, }, } klass = self._get_target_class() @@ -1618,15 +1664,18 @@ def test_from_api_repr_bare(self): def test_from_api_repr_w_properties(self): client = _Client(self.PROJECT) RESOURCE = self._makeResource() - RESOURCE['configuration']['query']['destinationTable'] = { + query_config = RESOURCE['configuration']['query'] + query_config['createDisposition'] = 'CREATE_IF_NEEDED' + query_config['writeDisposition'] = 'WRITE_TRUNCATE' + query_config['destinationTable'] = { 'projectId': self.PROJECT, 'datasetId': self.DS_NAME, 'tableId': self.DESTINATION_TABLE, } klass = self._get_target_class() - dataset = klass.from_api_repr(RESOURCE, client=client) - self.assertIs(dataset._client, client) - self._verifyResourceProperties(dataset, RESOURCE) + job = klass.from_api_repr(RESOURCE, client=client) + self.assertIs(job._client, client) + self._verifyResourceProperties(job, RESOURCE) def test_cancelled(self): client = _Client(self.PROJECT) From f52b00b092f674a26a55fcbe1d19395a411671ab Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Sat, 12 Aug 2017 20:25:25 -0400 Subject: [PATCH 3/6] Add 'Client.get_job' API wrapper. --- bigquery/.coveragerc | 2 + bigquery/google/cloud/bigquery/client.py | 29 ++++++++++ bigquery/google/cloud/bigquery/job.py | 3 -- bigquery/tests/unit/test__helpers.py | 3 +- bigquery/tests/unit/test_client.py | 69 ++++++++++++++++++++++++ bigquery/tests/unit/test_job.py | 13 ++++- 6 files changed, 113 insertions(+), 6 deletions(-) diff --git a/bigquery/.coveragerc b/bigquery/.coveragerc index a54b99aa14b7..d097511c3124 100644 --- a/bigquery/.coveragerc +++ b/bigquery/.coveragerc @@ -9,3 +9,5 @@ exclude_lines = pragma: NO COVER # Ignore debug-only repr def __repr__ + # Ignore abstract methods + raise NotImplementedError diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index dff282e21c43..7826da4ac5e8 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -187,6 +187,35 @@ def job_from_resource(self, resource): return QueryJob.from_api_repr(resource, self) raise ValueError('Cannot parse job resource') + def get_job(self, job_name, project=None): + """Fetch a job for the project associated with this client. + + See + https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/get + + :type job_name: str + :param job_name: Name of the job. + + :type project: str + :param project: + project ID owning the job (defaults to the client's project) + + :rtype: :class:`~google.cloud.bigquery.job._AsyncJob` + :returns: + Concrete job instance, based on the resource returned by the API. + """ + extra_params = {'projection': 'full'} + + if project is None: + project = self.project + + path = '/projects/{}/jobs/{}'.format(project, job_name) + + resource = self._connection.api_request( + method='GET', path=path, query_params=extra_params) + + return self.job_from_resource(resource) + def list_jobs(self, max_results=None, page_token=None, all_users=None, state_filter=None): """List jobs for the project associated with this client. diff --git a/bigquery/google/cloud/bigquery/job.py b/bigquery/google/cloud/bigquery/job.py index 077badccc6ef..8ec8eb8d8c07 100644 --- a/bigquery/google/cloud/bigquery/job.py +++ b/bigquery/google/cloud/bigquery/job.py @@ -28,10 +28,7 @@ from google.cloud.bigquery.table import Table from google.cloud.bigquery.table import _build_schema_resource from google.cloud.bigquery.table import _parse_schema_resource -from google.cloud.bigquery._helpers import ArrayQueryParameter from google.cloud.bigquery._helpers import QueryParametersProperty -from google.cloud.bigquery._helpers import ScalarQueryParameter -from google.cloud.bigquery._helpers import StructQueryParameter from google.cloud.bigquery._helpers import UDFResource from google.cloud.bigquery._helpers import UDFResourcesProperty from google.cloud.bigquery._helpers import _EnumProperty diff --git a/bigquery/tests/unit/test__helpers.py b/bigquery/tests/unit/test__helpers.py index 36efb51a6585..59ce5c8631ac 100644 --- a/bigquery/tests/unit/test__helpers.py +++ b/bigquery/tests/unit/test__helpers.py @@ -1361,7 +1361,7 @@ def test___eq__(self): self.assertNotEqual(param, object()) alias = self._make_one('bar', sub_1, sub_2) self.assertNotEqual(param, alias) - wrong_type = self._make_one( 'foo', sub_1_float, sub_2) + wrong_type = self._make_one('foo', sub_1_float, sub_2) self.assertNotEqual(param, wrong_type) wrong_val = self._make_one('foo', sub_2, sub_3) self.assertNotEqual(param, wrong_val) @@ -1603,7 +1603,6 @@ def test_w_struct(self): self.assertEqual(parameter.struct_values, {'foo': 'Foo', 'bar': 123}) - class Test_QueryParametersProperty(unittest.TestCase): @staticmethod diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index e4a3b8d740f5..d380233ecc64 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -208,6 +208,70 @@ def test_job_from_resource_unknown_type(self): with self.assertRaises(ValueError): client.job_from_resource({'configuration': {'nonesuch': {}}}) + def test_get_job_miss_w_explict_project(self): + from google.cloud.exceptions import NotFound + + PROJECT = 'PROJECT' + OTHER_PROJECT = 'OTHER_PROJECT' + JOB_ID = 'NONESUCH' + creds = _make_credentials() + client = self._make_one(PROJECT, creds) + conn = client._connection = _Connection() + + with self.assertRaises(NotFound): + client.get_job(JOB_ID, project=OTHER_PROJECT) + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], '/projects/OTHER_PROJECT/jobs/NONESUCH') + self.assertEqual(req['query_params'], {'projection': 'full'}) + + def test_get_job_hit(self): + from google.cloud.bigquery.job import QueryJob + + PROJECT = 'PROJECT' + JOB_ID = 'query_job' + DATASET = 'test_dataset' + QUERY_DESTINATION_TABLE = 'query_destination_table' + QUERY = 'SELECT * from test_dataset:test_table' + ASYNC_QUERY_DATA = { + 'id': '{}:{}'.format(PROJECT, JOB_ID), + 'jobReference': { + 'projectId': PROJECT, + 'jobId': 'query_job', + }, + 'state': 'DONE', + 'configuration': { + 'query': { + 'query': QUERY, + 'destinationTable': { + 'projectId': PROJECT, + 'datasetId': DATASET, + 'tableId': QUERY_DESTINATION_TABLE, + }, + 'createDisposition': 'CREATE_IF_NEEDED', + 'writeDisposition': 'WRITE_TRUNCATE', + } + }, + } + creds = _make_credentials() + client = self._make_one(PROJECT, creds) + conn = client._connection = _Connection(ASYNC_QUERY_DATA) + + job = client.get_job(JOB_ID) + + self.assertIsInstance(job, QueryJob) + self.assertEqual(job.name, JOB_ID) + self.assertEqual(job.create_disposition, 'CREATE_IF_NEEDED') + self.assertEqual(job.write_disposition, 'WRITE_TRUNCATE') + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], '/projects/PROJECT/jobs/query_job') + self.assertEqual(req['query_params'], {'projection': 'full'}) + def test_list_jobs_defaults(self): import six from google.cloud.bigquery.job import LoadJob @@ -607,6 +671,11 @@ def __init__(self, *responses): self._requested = [] def api_request(self, **kw): + from google.cloud.exceptions import NotFound self._requested.append(kw) + + if len(self._responses) == 0: + raise NotFound("miss") + response, self._responses = self._responses[0], self._responses[1:] return response diff --git a/bigquery/tests/unit/test_job.py b/bigquery/tests/unit/test_job.py index bd5afd36be94..706f9377a1d9 100644 --- a/bigquery/tests/unit/test_job.py +++ b/bigquery/tests/unit/test_job.py @@ -2030,7 +2030,10 @@ def test_result_error(self): self.assertEqual(exc_info.exception.code, http_client.BAD_REQUEST) def test_begin_w_bound_client(self): + from google.cloud.bigquery.dataset import Dataset + PATH = '/projects/%s/jobs' % (self.PROJECT,) + DS_NAME = 'DATASET' RESOURCE = self._makeResource() # Ensure None for missing server-set props del RESOURCE['statistics']['creationTime'] @@ -2039,9 +2042,13 @@ def test_begin_w_bound_client(self): del RESOURCE['user_email'] conn = _Connection(RESOURCE) client = _Client(project=self.PROJECT, connection=conn) + job = self._make_one(self.JOB_NAME, self.QUERY, client) + job.default_dataset = Dataset(DS_NAME, client) job.begin() + + self.assertIsNone(job.default_dataset) self.assertEqual(job.udf_resources, []) self.assertEqual(len(conn._requested), 1) req = conn._requested[0] @@ -2054,7 +2061,11 @@ def test_begin_w_bound_client(self): }, 'configuration': { 'query': { - 'query': self.QUERY + 'query': self.QUERY, + 'defaultDataset': { + 'projectId': self.PROJECT, + 'datasetId': DS_NAME, + }, }, }, } From 64395abfe388be67f6f933e709eed604e5f86a37 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 24 Aug 2017 09:30:42 -0700 Subject: [PATCH 4/6] Rename job_name to job_id. --- bigquery/google/cloud/bigquery/client.py | 40 ++++++++++++------------ bigquery/google/cloud/bigquery/job.py | 3 ++ 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index 7826da4ac5e8..1d09ef893dc7 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -187,14 +187,14 @@ def job_from_resource(self, resource): return QueryJob.from_api_repr(resource, self) raise ValueError('Cannot parse job resource') - def get_job(self, job_name, project=None): + def get_job(self, job_id, project=None): """Fetch a job for the project associated with this client. See https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/get - :type job_name: str - :param job_name: Name of the job. + :type job_id: str + :param job_id: Name of the job. :type project: str :param project: @@ -209,7 +209,7 @@ def get_job(self, job_name, project=None): if project is None: project = self.project - path = '/projects/{}/jobs/{}'.format(project, job_name) + path = '/projects/{}/jobs/{}'.format(project, job_id) resource = self._connection.api_request( method='GET', path=path, query_params=extra_params) @@ -266,14 +266,14 @@ def list_jobs(self, max_results=None, page_token=None, all_users=None, max_results=max_results, extra_params=extra_params) - def load_table_from_storage(self, job_name, destination, *source_uris): + def load_table_from_storage(self, job_id, destination, *source_uris): """Construct a job for loading data into a table from CloudStorage. See https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load - :type job_name: str - :param job_name: Name of the job. + :type job_id: str + :param job_id: Name of the job. :type destination: :class:`google.cloud.bigquery.table.Table` :param destination: Table into which data is to be loaded. @@ -285,16 +285,16 @@ def load_table_from_storage(self, job_name, destination, *source_uris): :rtype: :class:`google.cloud.bigquery.job.LoadJob` :returns: a new ``LoadJob`` instance """ - return LoadJob(job_name, destination, source_uris, client=self) + return LoadJob(job_id, destination, source_uris, client=self) - def copy_table(self, job_name, destination, *sources): + def copy_table(self, job_id, destination, *sources): """Construct a job for copying one or more tables into another table. See https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.copy - :type job_name: str - :param job_name: Name of the job. + :type job_id: str + :param job_id: Name of the job. :type destination: :class:`google.cloud.bigquery.table.Table` :param destination: Table into which data is to be copied. @@ -305,16 +305,16 @@ def copy_table(self, job_name, destination, *sources): :rtype: :class:`google.cloud.bigquery.job.CopyJob` :returns: a new ``CopyJob`` instance """ - return CopyJob(job_name, destination, sources, client=self) + return CopyJob(job_id, destination, sources, client=self) - def extract_table_to_storage(self, job_name, source, *destination_uris): + def extract_table_to_storage(self, job_id, source, *destination_uris): """Construct a job for extracting a table into Cloud Storage files. See https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.extract - :type job_name: str - :param job_name: Name of the job. + :type job_id: str + :param job_id: Name of the job. :type source: :class:`google.cloud.bigquery.table.Table` :param source: table to be extracted. @@ -327,17 +327,17 @@ def extract_table_to_storage(self, job_name, source, *destination_uris): :rtype: :class:`google.cloud.bigquery.job.ExtractJob` :returns: a new ``ExtractJob`` instance """ - return ExtractJob(job_name, source, destination_uris, client=self) + return ExtractJob(job_id, source, destination_uris, client=self) - def run_async_query(self, job_name, query, + def run_async_query(self, job_id, query, udf_resources=(), query_parameters=()): """Construct a job for running a SQL query asynchronously. See https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query - :type job_name: str - :param job_name: Name of the job. + :type job_id: str + :param job_id: Name of the job. :type query: str :param query: SQL query to be executed @@ -356,7 +356,7 @@ def run_async_query(self, job_name, query, :rtype: :class:`google.cloud.bigquery.job.QueryJob` :returns: a new ``QueryJob`` instance """ - return QueryJob(job_name, query, client=self, + return QueryJob(job_id, query, client=self, udf_resources=udf_resources, query_parameters=query_parameters) diff --git a/bigquery/google/cloud/bigquery/job.py b/bigquery/google/cloud/bigquery/job.py index 8ec8eb8d8c07..077badccc6ef 100644 --- a/bigquery/google/cloud/bigquery/job.py +++ b/bigquery/google/cloud/bigquery/job.py @@ -28,7 +28,10 @@ from google.cloud.bigquery.table import Table from google.cloud.bigquery.table import _build_schema_resource from google.cloud.bigquery.table import _parse_schema_resource +from google.cloud.bigquery._helpers import ArrayQueryParameter from google.cloud.bigquery._helpers import QueryParametersProperty +from google.cloud.bigquery._helpers import ScalarQueryParameter +from google.cloud.bigquery._helpers import StructQueryParameter from google.cloud.bigquery._helpers import UDFResource from google.cloud.bigquery._helpers import UDFResourcesProperty from google.cloud.bigquery._helpers import _EnumProperty From 58787b2e1317d09ffca3ba878e577ade45186d5c Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 24 Aug 2017 10:33:07 -0700 Subject: [PATCH 5/6] Send `maximumBytesBilled` as a string. "maximumBytesBilled": { "type": "string", "description": "[Optional] Limits the bytes billed for this job. Queries that will have bytes billed beyond this limit will fail (without incurring a charge). If unspecified, this will be set to your project default.", "format": "int64" }, From https://www.googleapis.com/discovery/v1/apis/bigquery/v2/rest Since it needs to represent a 64-bit integer the JSON number type is insufficient, so it must be sent as a string. --- bigquery/google/cloud/bigquery/job.py | 6 ++++-- bigquery/tests/unit/test_job.py | 11 ++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/bigquery/google/cloud/bigquery/job.py b/bigquery/google/cloud/bigquery/job.py index 077badccc6ef..e986b72be128 100644 --- a/bigquery/google/cloud/bigquery/job.py +++ b/bigquery/google/cloud/bigquery/job.py @@ -1266,7 +1266,8 @@ def _populate_config_resource(self, configuration): if self.maximum_billing_tier is not None: configuration['maximumBillingTier'] = self.maximum_billing_tier if self.maximum_bytes_billed is not None: - configuration['maximumBytesBilled'] = self.maximum_bytes_billed + configuration['maximumBytesBilled'] = str( + self.maximum_bytes_billed) if len(self._udf_resources) > 0: configuration[self._UDF_KEY] = [ {udf_resource.udf_type: udf_resource.value} @@ -1332,7 +1333,8 @@ def _copy_configuration_properties(self, configuration): self.priority = configuration.get('priority') self.write_disposition = configuration.get('writeDisposition') self.maximum_billing_tier = configuration.get('maximumBillingTier') - self.maximum_bytes_billed = configuration.get('maximumBytesBilled') + self.maximum_bytes_billed = _int_or_none( + configuration.get('maximumBytesBilled')) dest_remote = configuration.get('destinationTable') diff --git a/bigquery/tests/unit/test_job.py b/bigquery/tests/unit/test_job.py index 706f9377a1d9..74a0ed0188d7 100644 --- a/bigquery/tests/unit/test_job.py +++ b/bigquery/tests/unit/test_job.py @@ -1494,13 +1494,14 @@ def _verifyBooleanResourceProperties(self, job, config): def _verifyIntegerResourceProperties(self, job, config): if 'maximumBillingTier' in config: - self.assertEqual(job.maximum_billing_tier, - config['maximumBillingTier']) + self.assertEqual( + job.maximum_billing_tier, config['maximumBillingTier']) else: self.assertIsNone(job.maximum_billing_tier) if 'maximumBytesBilled' in config: - self.assertEqual(job.maximum_bytes_billed, - config['maximumBytesBilled']) + self.assertEqual( + str(job.maximum_bytes_billed), config['maximumBytesBilled']) + self.assertIsInstance(job.maximum_bytes_billed, int) else: self.assertIsNone(job.maximum_bytes_billed) @@ -2099,7 +2100,7 @@ def test_begin_w_alternate_client(self): 'useLegacySql': True, 'writeDisposition': 'WRITE_TRUNCATE', 'maximumBillingTier': 4, - 'maximumBytesBilled': 123456 + 'maximumBytesBilled': '123456' } RESOURCE['configuration']['query'] = QUERY_CONFIGURATION conn1 = _Connection() From 7b2fb70fed78c7dbed283bb2f9cd3546dee6d714 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 8 Sep 2017 09:43:33 -0400 Subject: [PATCH 6/6] Marshall 'skipLeadingRows' to JSON as a string. Addresses: https://github.com/GoogleCloudPlatform/google-cloud-python/pull/3804#discussion_r135085470 --- bigquery/google/cloud/bigquery/job.py | 2 +- bigquery/tests/unit/test_job.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bigquery/google/cloud/bigquery/job.py b/bigquery/google/cloud/bigquery/job.py index e986b72be128..e03765146639 100644 --- a/bigquery/google/cloud/bigquery/job.py +++ b/bigquery/google/cloud/bigquery/job.py @@ -755,7 +755,7 @@ def _populate_config_resource(self, configuration): if self.quote_character is not None: configuration['quote'] = self.quote_character if self.skip_leading_rows is not None: - configuration['skipLeadingRows'] = self.skip_leading_rows + configuration['skipLeadingRows'] = str(self.skip_leading_rows) if self.source_format is not None: configuration['sourceFormat'] = self.source_format if self.write_disposition is not None: diff --git a/bigquery/tests/unit/test_job.py b/bigquery/tests/unit/test_job.py index 74a0ed0188d7..41a359c34925 100644 --- a/bigquery/tests/unit/test_job.py +++ b/bigquery/tests/unit/test_job.py @@ -298,7 +298,7 @@ def _verifyResourceProperties(self, job, resource): else: self.assertIsNone(job.quote_character) if 'skipLeadingRows' in config: - self.assertEqual(job.skip_leading_rows, + self.assertEqual(str(job.skip_leading_rows), config['skipLeadingRows']) else: self.assertIsNone(job.skip_leading_rows) @@ -642,7 +642,7 @@ def test_begin_w_alternate_client(self): 'maxBadRecords': 100, 'nullMarker': r'\N', 'quote': "'", - 'skipLeadingRows': 1, + 'skipLeadingRows': '1', 'sourceFormat': 'CSV', 'writeDisposition': 'WRITE_TRUNCATE', 'schema': {'fields': [