Skip to content

Commit

Permalink
BigQuery: Add properties to job config constructors (#6397)
Browse files Browse the repository at this point in the history 
* add properties to job config constructors

* update properties arg

* update **properties to **kwargs

* fix docstrings
  • Loading branch information
@alixhami @tswast
alixhami authored and tswast committed Nov 6, 2018
1 parent 6ad3c60 commit b7a23bd
Show file tree
Hide file tree
Showing 2 changed files with 74 additions and 32 deletions.
68 changes: 38 additions & 30 deletions bigquery/google/cloud/bigquery/job.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -719,9 +719,11 @@ class _JobConfig(object):
job_type (str): The key to use for the job configuration.
"""

def __init__(self, job_type):
def __init__(self, job_type, **kwargs):
self._job_type = job_type
self._properties = {job_type: {}}
for prop, val in kwargs.items():
setattr(self, prop, val)

@property
def labels(self):
Expand Down Expand Up @@ -762,7 +764,7 @@ def _get_sub_prop(self, key, default=None):
``self._properties[self._job_type]`` dictionary.
default (object):
(Optional) Default value to return if the key is not found.
Defaults to ``None``.
Defaults to :data:`None`.
Returns:
object: The value if present or the default.
Expand Down Expand Up @@ -793,7 +795,7 @@ def _set_sub_prop(self, key, value):
_helpers._set_sub_prop(self._properties, [self._job_type, key], value)

def _del_sub_prop(self, key):
"""Reove ``key`` from the ``self._properties[self._job_type]`` dict.
"""Remove ``key`` from the ``self._properties[self._job_type]`` dict.
Most job properties are inside the dictionary related to the job type
(e.g. 'copy', 'extract', 'load', 'query'). Use this method to clear
Expand Down Expand Up @@ -874,12 +876,13 @@ def from_api_repr(cls, resource):
class LoadJobConfig(_JobConfig):
"""Configuration options for load jobs.
All properties in this class are optional. Values which are ``None`` ->
server defaults.
All properties in this class are optional. Values which are :data:`None` ->
server defaults. Set properties on the constructed configuration by using
the property name as the name of a keyword argument.
"""

def __init__(self):
super(LoadJobConfig, self).__init__('load')
def __init__(self, **kwargs):
super(LoadJobConfig, self).__init__('load', **kwargs)

@property
def allow_jagged_rows(self):
Expand Down Expand Up @@ -967,7 +970,7 @@ def destination_encryption_configuration(self):
"""google.cloud.bigquery.table.EncryptionConfiguration: Custom
encryption configuration for the destination table.
Custom encryption configuration (e.g., Cloud KMS keys) or ``None``
Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None`
if using default encryption.
See
Expand Down Expand Up @@ -1338,7 +1341,7 @@ def destination_encryption_configuration(self):
encryption configuration for the destination table.
Custom encryption configuration (e.g., Cloud KMS keys)
or ``None`` if using default encryption.
or :data:`None` if using default encryption.
See
:attr:`google.cloud.bigquery.job.LoadJobConfig.destination_encryption_configuration`.
Expand Down Expand Up @@ -1469,12 +1472,13 @@ def from_api_repr(cls, resource, client):
class CopyJobConfig(_JobConfig):
"""Configuration options for copy jobs.
All properties in this class are optional. Values which are ``None`` ->
server defaults.
All properties in this class are optional. Values which are :data:`None` ->
server defaults. Set properties on the constructed configuration by using
the property name as the name of a keyword argument.
"""

def __init__(self):
super(CopyJobConfig, self).__init__('copy')
def __init__(self, **kwargs):
super(CopyJobConfig, self).__init__('copy', **kwargs)

@property
def create_disposition(self):
Expand Down Expand Up @@ -1509,7 +1513,7 @@ def destination_encryption_configuration(self):
"""google.cloud.bigquery.table.EncryptionConfiguration: Custom
encryption configuration for the destination table.
Custom encryption configuration (e.g., Cloud KMS keys) or ``None``
Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None`
if using default encryption.
See
Expand Down Expand Up @@ -1579,7 +1583,7 @@ def destination_encryption_configuration(self):
"""google.cloud.bigquery.table.EncryptionConfiguration: Custom
encryption configuration for the destination table.
Custom encryption configuration (e.g., Cloud KMS keys) or ``None``
Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None`
if using default encryption.
See
Expand Down Expand Up @@ -1662,12 +1666,13 @@ def from_api_repr(cls, resource, client):
class ExtractJobConfig(_JobConfig):
"""Configuration options for extract jobs.
All properties in this class are optional. Values which are ``None`` ->
server defaults.
All properties in this class are optional. Values which are :data:`None` ->
server defaults. Set properties on the constructed configuration by using
the property name as the name of a keyword argument.
"""

def __init__(self):
super(ExtractJobConfig, self).__init__('extract')
def __init__(self, **kwargs):
super(ExtractJobConfig, self).__init__('extract', **kwargs)

@property
def compression(self):
Expand Down Expand Up @@ -1906,19 +1911,20 @@ def _to_api_repr_table_defs(value):
class QueryJobConfig(_JobConfig):
"""Configuration options for query jobs.
All properties in this class are optional. Values which are ``None`` ->
server defaults.
All properties in this class are optional. Values which are :data:`None` ->
server defaults. Set properties on the constructed configuration by using
the property name as the name of a keyword argument.
"""

def __init__(self):
super(QueryJobConfig, self).__init__('query')
def __init__(self, **kwargs):
super(QueryJobConfig, self).__init__('query', **kwargs)

@property
def destination_encryption_configuration(self):
"""google.cloud.bigquery.table.EncryptionConfiguration: Custom
encryption configuration for the destination table.
Custom encryption configuration (e.g., Cloud KMS keys) or ``None``
Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None`
if using default encryption.
See
Expand Down Expand Up @@ -1966,7 +1972,8 @@ def create_disposition(self, value):
@property
def default_dataset(self):
"""google.cloud.bigquery.dataset.DatasetReference: the default dataset
to use for unqualified table names in the query or ``None`` if not set.
to use for unqualified table names in the query or :data:`None` if not
set.
See
https://g.co/cloud/bigquery/docs/reference/v2/jobs#configuration.query.defaultDataset
Expand All @@ -1986,7 +1993,7 @@ def default_dataset(self, value):
@property
def destination(self):
"""google.cloud.bigquery.table.TableReference: table where results are
written or ``None`` if not set.
written or :data:`None` if not set.
See
https://g.co/cloud/bigquery/docs/reference/rest/v2/jobs#configuration.query.destinationTable
Expand All @@ -2005,7 +2012,8 @@ def destination(self, value):

@property
def dry_run(self):
"""bool: ``True`` if this query should be a dry run to estimate costs.
"""bool: :data:`True` if this query should be a dry run to estimate
costs.
See
https://g.co/cloud/bigquery/docs/reference/v2/jobs#configuration.dryRun
Expand Down Expand Up @@ -2045,7 +2053,7 @@ def maximum_billing_tier(self, value):

@property
def maximum_bytes_billed(self):
"""int: Maximum bytes to be billed for this job or ``None`` if not set.
"""int: Maximum bytes to be billed for this job or :data:`None` if not set.
See
https://g.co/cloud/bigquery/docs/reference/rest/v2/jobs#configuration.query.maximumBytesBilled
Expand Down Expand Up @@ -2147,7 +2155,7 @@ def write_disposition(self, value):
@property
def table_definitions(self):
"""Dict[str, google.cloud.bigquery.external_config.ExternalConfig]:
Definitions for external tables or ``None`` if not set.
Definitions for external tables or :data:`None` if not set.
See
https://g.co/cloud/bigquery/docs/reference/rest/v2/jobs#configuration.query.tableDefinitions
Expand Down Expand Up @@ -2305,7 +2313,7 @@ def destination_encryption_configuration(self):
"""google.cloud.bigquery.table.EncryptionConfiguration: Custom
encryption configuration for the destination table.
Custom encryption configuration (e.g., Cloud KMS keys) or ``None``
Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None`
if using default encryption.
See
Expand Down
38 changes: 36 additions & 2 deletions bigquery/tests/unit/test_job.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1152,6 +1152,13 @@ def _get_target_class():
from google.cloud.bigquery.job import LoadJobConfig
return LoadJobConfig

def test_ctor_w_properties(self):
config = self._get_target_class()(
allow_jagged_rows=True, allow_quoted_newlines=True)

self.assertTrue(config.allow_jagged_rows)
self.assertTrue(config.allow_quoted_newlines)

def test_allow_jagged_rows_missing(self):
config = self._get_target_class()()
self.assertIsNone(config.allow_jagged_rows)
Expand Down Expand Up @@ -2482,6 +2489,20 @@ def _get_target_class():
from google.cloud.bigquery.job import CopyJobConfig
return CopyJobConfig

def test_ctor_w_properties(self):
from google.cloud.bigquery.job import CreateDisposition
from google.cloud.bigquery.job import WriteDisposition

create_disposition = CreateDisposition.CREATE_NEVER
write_disposition = WriteDisposition.WRITE_TRUNCATE
config = self._get_target_class()(
create_disposition=create_disposition,
write_disposition=write_disposition
)

self.assertEqual(config.create_disposition, create_disposition)
self.assertEqual(config.write_disposition, write_disposition)

def test_to_api_repr_with_encryption(self):
from google.cloud.bigquery.table import EncryptionConfiguration

Expand Down Expand Up @@ -2916,6 +2937,13 @@ def _get_target_class():
from google.cloud.bigquery.job import ExtractJobConfig
return ExtractJobConfig

def test_ctor_w_properties(self):
config = self._get_target_class()(
field_delimiter='\t', print_header=True)

self.assertEqual(config.field_delimiter, '\t')
self.assertTrue(config.print_header)

def test_to_api_repr(self):
from google.cloud.bigquery import job
config = self._make_one()
Expand Down Expand Up @@ -3299,6 +3327,13 @@ def test_ctor_w_none(self):
self.assertIsNone(config.default_dataset)
self.assertIsNone(config.destination)

def test_ctor_w_properties(self):
config = self._get_target_class()(
use_query_cache=False, use_legacy_sql=True)

self.assertFalse(config.use_query_cache)
self.assertTrue(config.use_legacy_sql)

def test_time_partitioning(self):
from google.cloud.bigquery import table

Expand Down Expand Up @@ -3637,8 +3672,7 @@ def test_ctor_w_query_parameters(self):

query_parameters = [ScalarQueryParameter("foo", 'INT64', 123)]
client = _make_client(project=self.PROJECT)
config = QueryJobConfig()
config.query_parameters = query_parameters
config = QueryJobConfig(query_parameters=query_parameters)
job = self._make_one(
self.JOB_ID, self.QUERY, client, job_config=config)
self.assertEqual(job.query_parameters, query_parameters)
Expand Down

0 comments on commit b7a23bd

Please sign in to comment.