diff --git a/docs/bigquery_v2/types.rst b/docs/bigquery/standard_sql.rst similarity index 72% rename from docs/bigquery_v2/types.rst rename to docs/bigquery/standard_sql.rst index c36a83e0b..bd52bb78f 100644 --- a/docs/bigquery_v2/types.rst +++ b/docs/bigquery/standard_sql.rst @@ -1,7 +1,7 @@ Types for Google Cloud Bigquery v2 API ====================================== -.. automodule:: google.cloud.bigquery_v2.types +.. automodule:: google.cloud.bigquery.standard_sql :members: :undoc-members: :show-inheritance: diff --git a/docs/conf.py b/docs/conf.py index 07e5d8c30..b8ddbd8c8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -114,7 +114,6 @@ "samples/AUTHORING_GUIDE.md", "samples/CONTRIBUTING.md", "samples/snippets/README.rst", - "bigquery_v2/services.rst", # generated by the code generator ] # The reST default role (used for this markup: `text`) to use for all @@ -364,8 +363,6 @@ "google-auth": ("https://googleapis.dev/python/google-auth/latest/", None), "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None,), "grpc": ("https://grpc.github.io/grpc/python/", None), - "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None), - "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), "pandas": ("http://pandas.pydata.org/pandas-docs/dev", None), "geopandas": ("https://geopandas.org/", None), } diff --git a/docs/reference.rst b/docs/reference.rst index d8738e67b..128dee718 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -197,9 +197,9 @@ Encryption Configuration Additional Types ================ -Protocol buffer classes for working with the Models API. +Helper SQL type classes. .. toctree:: :maxdepth: 2 - bigquery_v2/types + bigquery/standard_sql diff --git a/google/cloud/bigquery/__init__.py b/google/cloud/bigquery/__init__.py index 5529f9b2e..660a660b4 100644 --- a/google/cloud/bigquery/__init__.py +++ b/google/cloud/bigquery/__init__.py @@ -41,7 +41,7 @@ from google.cloud.bigquery.enums import DecimalTargetType from google.cloud.bigquery.enums import KeyResultStatementKind from google.cloud.bigquery.enums import SqlTypeNames -from google.cloud.bigquery.enums import StandardSqlDataTypes +from google.cloud.bigquery.enums import StandardSqlTypeNames from google.cloud.bigquery.external_config import ExternalConfig from google.cloud.bigquery.external_config import BigtableOptions from google.cloud.bigquery.external_config import BigtableColumnFamily @@ -77,6 +77,7 @@ from google.cloud.bigquery.query import ArrayQueryParameterType from google.cloud.bigquery.query import ScalarQueryParameter from google.cloud.bigquery.query import ScalarQueryParameterType +from google.cloud.bigquery.query import SqlParameterScalarTypes from google.cloud.bigquery.query import StructQueryParameter from google.cloud.bigquery.query import StructQueryParameterType from google.cloud.bigquery.query import UDFResource @@ -87,6 +88,10 @@ from google.cloud.bigquery.routine import RoutineReference from google.cloud.bigquery.routine import RoutineType from google.cloud.bigquery.schema import SchemaField +from google.cloud.bigquery.standard_sql import StandardSqlDataType +from google.cloud.bigquery.standard_sql import StandardSqlField +from google.cloud.bigquery.standard_sql import StandardSqlStructType +from google.cloud.bigquery.standard_sql import StandardSqlTableType from google.cloud.bigquery.table import PartitionRange from google.cloud.bigquery.table import RangePartitioning from google.cloud.bigquery.table import Row @@ -108,6 +113,7 @@ "StructQueryParameter", "ArrayQueryParameterType", "ScalarQueryParameterType", + "SqlParameterScalarTypes", "StructQueryParameterType", # Datasets "Dataset", @@ -151,6 +157,11 @@ "ScriptOptions", "TransactionInfo", "DEFAULT_RETRY", + # Standard SQL types + "StandardSqlDataType", + "StandardSqlField", + "StandardSqlStructType", + "StandardSqlTableType", # Enum Constants "enums", "AutoRowIDs", @@ -168,7 +179,7 @@ "SchemaUpdateOption", "SourceFormat", "SqlTypeNames", - "StandardSqlDataTypes", + "StandardSqlTypeNames", "WriteDisposition", # EncryptionConfiguration "EncryptionConfiguration", diff --git a/google/cloud/bigquery/dbapi/_helpers.py b/google/cloud/bigquery/dbapi/_helpers.py index 72e711bcf..c2daf4076 100644 --- a/google/cloud/bigquery/dbapi/_helpers.py +++ b/google/cloud/bigquery/dbapi/_helpers.py @@ -22,7 +22,7 @@ import typing from google.cloud import bigquery -from google.cloud.bigquery import table, enums, query +from google.cloud.bigquery import table, query from google.cloud.bigquery.dbapi import exceptions @@ -48,7 +48,7 @@ def _parameter_type(name, value, query_parameter_type=None, value_doc=""): query_parameter_type = type_parameters_re.sub("", query_parameter_type) try: parameter_type = getattr( - enums.SqlParameterScalarTypes, query_parameter_type.upper() + query.SqlParameterScalarTypes, query_parameter_type.upper() )._type except AttributeError: raise exceptions.ProgrammingError( @@ -185,7 +185,7 @@ def _parse_type( # Strip type parameters type_ = type_parameters_re.sub("", type_).strip() try: - type_ = getattr(enums.SqlParameterScalarTypes, type_.upper()) + type_ = getattr(query.SqlParameterScalarTypes, type_.upper()) except AttributeError: raise exceptions.ProgrammingError( f"The given parameter type, {type_}," diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index d67cebd4c..cecdaa503 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -12,13 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import re - import enum -import itertools - -from google.cloud.bigquery_v2 import types as gapic_types -from google.cloud.bigquery.query import ScalarQueryParameterType class AutoRowIDs(enum.Enum): @@ -180,56 +174,27 @@ class KeyResultStatementKind: FIRST_SELECT = "FIRST_SELECT" -_SQL_SCALAR_TYPES = frozenset( - ( - "INT64", - "BOOL", - "FLOAT64", - "STRING", - "BYTES", - "TIMESTAMP", - "DATE", - "TIME", - "DATETIME", - "INTERVAL", - "GEOGRAPHY", - "NUMERIC", - "BIGNUMERIC", - "JSON", - ) -) - -_SQL_NONSCALAR_TYPES = frozenset(("TYPE_KIND_UNSPECIFIED", "ARRAY", "STRUCT")) - - -def _make_sql_scalars_enum(): - """Create an enum based on a gapic enum containing only SQL scalar types.""" - - new_enum = enum.Enum( - "StandardSqlDataTypes", - ( - (member.name, member.value) - for member in gapic_types.StandardSqlDataType.TypeKind - if member.name in _SQL_SCALAR_TYPES - ), - ) - - # make sure the docstring for the new enum is also correct - orig_doc = gapic_types.StandardSqlDataType.TypeKind.__doc__ - skip_pattern = re.compile( - "|".join(_SQL_NONSCALAR_TYPES) - + "|because a JSON object" # the second description line of STRUCT member - ) - - new_doc = "\n".join( - itertools.filterfalse(skip_pattern.search, orig_doc.splitlines()) - ) - new_enum.__doc__ = "An Enum of scalar SQL types.\n" + new_doc - - return new_enum - - -StandardSqlDataTypes = _make_sql_scalars_enum() +class StandardSqlTypeNames(str, enum.Enum): + def _generate_next_value_(name, start, count, last_values): + return name + + TYPE_KIND_UNSPECIFIED = enum.auto() + INT64 = enum.auto() + BOOL = enum.auto() + FLOAT64 = enum.auto() + STRING = enum.auto() + BYTES = enum.auto() + TIMESTAMP = enum.auto() + DATE = enum.auto() + TIME = enum.auto() + DATETIME = enum.auto() + INTERVAL = enum.auto() + GEOGRAPHY = enum.auto() + NUMERIC = enum.auto() + BIGNUMERIC = enum.auto() + JSON = enum.auto() + ARRAY = enum.auto() + STRUCT = enum.auto() # See also: https://cloud.google.com/bigquery/data-types#legacy_sql_data_types @@ -256,28 +221,6 @@ class SqlTypeNames(str, enum.Enum): DATETIME = "DATETIME" -class SqlParameterScalarTypes: - """Supported scalar SQL query parameter types as type objects.""" - - BOOL = ScalarQueryParameterType("BOOL") - BOOLEAN = ScalarQueryParameterType("BOOL") - BIGDECIMAL = ScalarQueryParameterType("BIGNUMERIC") - BIGNUMERIC = ScalarQueryParameterType("BIGNUMERIC") - BYTES = ScalarQueryParameterType("BYTES") - DATE = ScalarQueryParameterType("DATE") - DATETIME = ScalarQueryParameterType("DATETIME") - DECIMAL = ScalarQueryParameterType("NUMERIC") - FLOAT = ScalarQueryParameterType("FLOAT64") - FLOAT64 = ScalarQueryParameterType("FLOAT64") - GEOGRAPHY = ScalarQueryParameterType("GEOGRAPHY") - INT64 = ScalarQueryParameterType("INT64") - INTEGER = ScalarQueryParameterType("INT64") - NUMERIC = ScalarQueryParameterType("NUMERIC") - STRING = ScalarQueryParameterType("STRING") - TIME = ScalarQueryParameterType("TIME") - TIMESTAMP = ScalarQueryParameterType("TIMESTAMP") - - class WriteDisposition(object): """Specifies the action that occurs if destination table already exists. diff --git a/google/cloud/bigquery/model.py b/google/cloud/bigquery/model.py index 2d3f6660f..18b7b13ec 100644 --- a/google/cloud/bigquery/model.py +++ b/google/cloud/bigquery/model.py @@ -17,24 +17,23 @@ """Define resources for the BigQuery ML Models API.""" import copy - -from google.protobuf import json_format +import datetime +from typing import Any, Dict, Optional, Sequence, Union import google.cloud._helpers -from google.api_core import datetime_helpers from google.cloud.bigquery import _helpers -from google.cloud.bigquery_v2 import types +from google.cloud.bigquery import standard_sql from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration -class Model(object): +class Model: """Model represents a machine learning model resource. See https://cloud.google.com/bigquery/docs/reference/rest/v2/models Args: - model_ref (Union[google.cloud.bigquery.model.ModelReference, str]): + model_ref: A pointer to a model. If ``model_ref`` is a string, it must included a project ID, dataset ID, and model ID, each separated by ``.``. @@ -51,11 +50,7 @@ class Model(object): "encryption_configuration": "encryptionConfiguration", } - def __init__(self, model_ref): - # Use _proto on read-only properties to use it's built-in type - # conversion. - self._proto = types.Model()._pb - + def __init__(self, model_ref: Union["ModelReference", str, None]): # Use _properties on read-write properties to match the REST API # semantics. The BigQuery API makes a distinction between an unset # value, a null value, and a default value (0 or ""), but the protocol @@ -66,140 +61,125 @@ def __init__(self, model_ref): model_ref = ModelReference.from_string(model_ref) if model_ref: - self._proto.model_reference.CopyFrom(model_ref._proto) + self._properties["modelReference"] = model_ref.to_api_repr() @property - def reference(self): - """A :class:`~google.cloud.bigquery.model.ModelReference` pointing to - this model. + def reference(self) -> Optional["ModelReference"]: + """A model reference pointing to this model. Read-only. - - Returns: - google.cloud.bigquery.model.ModelReference: pointer to this model. """ - ref = ModelReference() - ref._proto = self._proto.model_reference - return ref + resource = self._properties.get("modelReference") + if resource is not None: + return ModelReference.from_api_repr(resource) @property - def project(self): - """str: Project bound to the model""" + def project(self) -> str: + """Project bound to the model.""" return self.reference.project @property - def dataset_id(self): - """str: ID of dataset containing the model.""" + def dataset_id(self) -> str: + """ID of dataset containing the model.""" return self.reference.dataset_id @property - def model_id(self): - """str: The model ID.""" + def model_id(self) -> str: + """The model ID.""" return self.reference.model_id @property - def path(self): - """str: URL path for the model's APIs.""" + def path(self) -> str: + """URL path for the model's APIs.""" return self.reference.path @property - def location(self): - """str: The geographic location where the model resides. This value - is inherited from the dataset. + def location(self) -> str: + """The geographic location where the model resides. + + This value is inherited from the dataset. Read-only. """ - return self._proto.location + return self._properties.get("location") @property - def etag(self): - """str: ETag for the model resource (:data:`None` until - set from the server). + def etag(self) -> str: + """ETag for the model resource (:data:`None` until set from the server). Read-only. """ - return self._proto.etag + return self._properties.get("etag") @property - def created(self): - """Union[datetime.datetime, None]: Datetime at which the model was - created (:data:`None` until set from the server). + def created(self) -> Optional[datetime.datetime]: + """Datetime at which the model was created (:data:`None` until set from the server). Read-only. """ - value = self._proto.creation_time - if value is not None and value != 0: + value = self._properties.get("creationTime") + if value is not None: # value will be in milliseconds. return google.cloud._helpers._datetime_from_microseconds( 1000.0 * float(value) ) @property - def modified(self): - """Union[datetime.datetime, None]: Datetime at which the model was last - modified (:data:`None` until set from the server). + def modified(self) -> Optional[datetime.datetime]: + """Datetime at which the model was last modified (:data:`None` until set from the server). Read-only. """ - value = self._proto.last_modified_time - if value is not None and value != 0: + value = value = self._properties.get("lastModifiedTime") + if value is not None: # value will be in milliseconds. return google.cloud._helpers._datetime_from_microseconds( 1000.0 * float(value) ) @property - def model_type(self): - """google.cloud.bigquery_v2.types.Model.ModelType: Type of the - model resource. + def model_type(self) -> str: + """Type of the model resource. Read-only. - - The value is one of elements of the - :class:`~google.cloud.bigquery_v2.types.Model.ModelType` - enumeration. """ - return self._proto.model_type + return self._properties.get("modelType", "MODEL_TYPE_UNSPECIFIED") @property - def training_runs(self): - """Sequence[google.cloud.bigquery_v2.types.Model.TrainingRun]: Information - for all training runs in increasing order of start time. + def training_runs(self) -> Sequence[Dict[str, Any]]: + """Information for all training runs in increasing order of start time. - Read-only. + Dictionaries are in REST API format. See: + https://cloud.google.com/bigquery/docs/reference/rest/v2/models#trainingrun - An iterable of :class:`~google.cloud.bigquery_v2.types.Model.TrainingRun`. + Read-only. """ - return self._proto.training_runs + return self._properties.get("trainingRuns", []) @property - def feature_columns(self): - """Sequence[google.cloud.bigquery_v2.types.StandardSqlField]: Input - feature columns that were used to train this model. + def feature_columns(self) -> Sequence[standard_sql.StandardSqlField]: + """Input feature columns that were used to train this model. Read-only. - - An iterable of :class:`~google.cloud.bigquery_v2.types.StandardSqlField`. """ - return self._proto.feature_columns + return self._properties.get("featureColumns", []) @property - def label_columns(self): - """Sequence[google.cloud.bigquery_v2.types.StandardSqlField]: Label - columns that were used to train this model. The output of the model - will have a ``predicted_`` prefix to these columns. + def label_columns(self) -> Sequence[standard_sql.StandardSqlField]: + """Label columns that were used to train this model. - Read-only. + The output of the model will have a ``predicted_`` prefix to these columns. - An iterable of :class:`~google.cloud.bigquery_v2.types.StandardSqlField`. + Read-only. """ - return self._proto.label_columns + return self._properties.get("labelColumns", []) @property - def expires(self): - """Union[datetime.datetime, None]: The datetime when this model - expires. If not present, the model will persist indefinitely. Expired - models will be deleted and their storage reclaimed. + def expires(self) -> Optional[datetime.datetime]: + """The datetime when this model expires. + + If not present, the model will persist indefinitely. Expired models will be + deleted and their storage reclaimed. """ value = self._properties.get("expirationTime") if value is not None: @@ -209,55 +189,48 @@ def expires(self): ) @expires.setter - def expires(self, value): + def expires(self, value: Optional[datetime.datetime]): if value is not None: value = str(google.cloud._helpers._millis_from_datetime(value)) self._properties["expirationTime"] = value @property - def description(self): - """Optional[str]: Description of the model (defaults to - :data:`None`). - """ + def description(self) -> Optional[str]: + """Description of the model (defaults to :data:`None`).""" return self._properties.get("description") @description.setter - def description(self, value): + def description(self, value: Optional[str]): self._properties["description"] = value @property - def friendly_name(self): - """Optional[str]: Title of the table (defaults to :data:`None`). - - Raises: - ValueError: For invalid value types. - """ + def friendly_name(self) -> Optional[str]: + """Title of the table (defaults to :data:`None`).""" return self._properties.get("friendlyName") @friendly_name.setter - def friendly_name(self, value): + def friendly_name(self, value: Optional[str]): self._properties["friendlyName"] = value @property - def labels(self): - """Optional[Dict[str, str]]: Labels for the table. + def labels(self) -> Dict[str, str]: + """Labels for the table. - This method always returns a dict. To change a model's labels, - modify the dict, then call ``Client.update_model``. To delete a - label, set its value to :data:`None` before updating. + This method always returns a dict. To change a model's labels, modify the dict, + then call ``Client.update_model``. To delete a label, set its value to + :data:`None` before updating. """ return self._properties.setdefault("labels", {}) @labels.setter - def labels(self, value): + def labels(self, value: Optional[Dict[str, str]]): if value is None: value = {} self._properties["labels"] = value @property - def encryption_configuration(self): - """Optional[google.cloud.bigquery.encryption_configuration.EncryptionConfiguration]: Custom - encryption configuration for the model. + def encryption_configuration(self) -> Optional[EncryptionConfiguration]: + """Custom encryption configuration for the model. Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None` if using default encryption. @@ -272,47 +245,26 @@ def encryption_configuration(self): return prop @encryption_configuration.setter - def encryption_configuration(self, value): + def encryption_configuration(self, value: Optional[EncryptionConfiguration]): api_repr = value if value: api_repr = value.to_api_repr() self._properties["encryptionConfiguration"] = api_repr @classmethod - def from_api_repr(cls, resource: dict) -> "Model": + def from_api_repr(cls, resource: Dict[str, Any]) -> "Model": """Factory: construct a model resource given its API representation Args: - resource (Dict[str, object]): + resource: Model resource representation from the API Returns: - google.cloud.bigquery.model.Model: Model parsed from ``resource``. + Model parsed from ``resource``. """ this = cls(None) - # Keep a reference to the resource as a workaround to find unknown - # field values. - this._properties = resource - - # Convert from millis-from-epoch to timestamp well-known type. - # TODO: Remove this hack once CL 238585470 hits prod. resource = copy.deepcopy(resource) - for training_run in resource.get("trainingRuns", ()): - start_time = training_run.get("startTime") - if not start_time or "-" in start_time: # Already right format? - continue - start_time = datetime_helpers.from_microseconds(1e3 * float(start_time)) - training_run["startTime"] = datetime_helpers.to_rfc3339(start_time) - - try: - this._proto = json_format.ParseDict( - resource, types.Model()._pb, ignore_unknown_fields=True - ) - except json_format.ParseError: - resource["modelType"] = "MODEL_TYPE_UNSPECIFIED" - this._proto = json_format.ParseDict( - resource, types.Model()._pb, ignore_unknown_fields=True - ) + this._properties = resource return this def _build_resource(self, filter_fields): @@ -320,18 +272,18 @@ def _build_resource(self, filter_fields): return _helpers._build_resource_from_properties(self, filter_fields) def __repr__(self): - return "Model(reference={})".format(repr(self.reference)) + return f"Model(reference={self.reference!r})" - def to_api_repr(self) -> dict: + def to_api_repr(self) -> Dict[str, Any]: """Construct the API resource representation of this model. Returns: - Dict[str, object]: Model reference represented as an API resource + Model reference represented as an API resource """ - return json_format.MessageToDict(self._proto) + return copy.deepcopy(self._properties) -class ModelReference(object): +class ModelReference: """ModelReferences are pointers to models. See @@ -339,73 +291,60 @@ class ModelReference(object): """ def __init__(self): - self._proto = types.ModelReference()._pb self._properties = {} @property def project(self): """str: Project bound to the model""" - return self._proto.project_id + return self._properties.get("projectId") @property def dataset_id(self): """str: ID of dataset containing the model.""" - return self._proto.dataset_id + return self._properties.get("datasetId") @property def model_id(self): """str: The model ID.""" - return self._proto.model_id + return self._properties.get("modelId") @property - def path(self): - """str: URL path for the model's APIs.""" - return "/projects/%s/datasets/%s/models/%s" % ( - self._proto.project_id, - self._proto.dataset_id, - self._proto.model_id, - ) + def path(self) -> str: + """URL path for the model's APIs.""" + return f"/projects/{self.project}/datasets/{self.dataset_id}/models/{self.model_id}" @classmethod - def from_api_repr(cls, resource): - """Factory: construct a model reference given its API representation + def from_api_repr(cls, resource: Dict[str, Any]) -> "ModelReference": + """Factory: construct a model reference given its API representation. Args: - resource (Dict[str, object]): + resource: Model reference representation returned from the API Returns: - google.cloud.bigquery.model.ModelReference: - Model reference parsed from ``resource``. + Model reference parsed from ``resource``. """ ref = cls() - # Keep a reference to the resource as a workaround to find unknown - # field values. ref._properties = resource - ref._proto = json_format.ParseDict( - resource, types.ModelReference()._pb, ignore_unknown_fields=True - ) - return ref @classmethod def from_string( - cls, model_id: str, default_project: str = None + cls, model_id: str, default_project: Optional[str] = None ) -> "ModelReference": """Construct a model reference from model ID string. Args: - model_id (str): + model_id: A model ID in standard SQL format. If ``default_project`` is not specified, this must included a project ID, dataset ID, and model ID, each separated by ``.``. - default_project (Optional[str]): + default_project: The project ID to use when ``model_id`` does not include a project ID. Returns: - google.cloud.bigquery.model.ModelReference: - Model reference parsed from ``model_id``. + Model reference parsed from ``model_id``. Raises: ValueError: @@ -419,13 +358,13 @@ def from_string( {"projectId": proj, "datasetId": dset, "modelId": model} ) - def to_api_repr(self) -> dict: + def to_api_repr(self) -> Dict[str, Any]: """Construct the API resource representation of this model reference. Returns: - Dict[str, object]: Model reference represented as an API resource + Model reference represented as an API resource. """ - return json_format.MessageToDict(self._proto) + return copy.deepcopy(self._properties) def _key(self): """Unique key for this model. @@ -437,7 +376,7 @@ def _key(self): def __eq__(self, other): if not isinstance(other, ModelReference): return NotImplemented - return self._proto == other._proto + return self._properties == other._properties def __ne__(self, other): return not self == other diff --git a/google/cloud/bigquery/query.py b/google/cloud/bigquery/query.py index 1f449f189..d58d46fd9 100644 --- a/google/cloud/bigquery/query.py +++ b/google/cloud/bigquery/query.py @@ -339,7 +339,7 @@ class ScalarQueryParameter(_AbstractQueryParameter): type_: Name of parameter type. See :class:`google.cloud.bigquery.enums.SqlTypeNames` and - :class:`google.cloud.bigquery.enums.SqlParameterScalarTypes` for + :class:`google.cloud.bigquery.query.SqlParameterScalarTypes` for supported types. value: @@ -750,6 +750,28 @@ def __repr__(self): return "StructQueryParameter{}".format(self._key()) +class SqlParameterScalarTypes: + """Supported scalar SQL query parameter types as type objects.""" + + BOOL = ScalarQueryParameterType("BOOL") + BOOLEAN = ScalarQueryParameterType("BOOL") + BIGDECIMAL = ScalarQueryParameterType("BIGNUMERIC") + BIGNUMERIC = ScalarQueryParameterType("BIGNUMERIC") + BYTES = ScalarQueryParameterType("BYTES") + DATE = ScalarQueryParameterType("DATE") + DATETIME = ScalarQueryParameterType("DATETIME") + DECIMAL = ScalarQueryParameterType("NUMERIC") + FLOAT = ScalarQueryParameterType("FLOAT64") + FLOAT64 = ScalarQueryParameterType("FLOAT64") + GEOGRAPHY = ScalarQueryParameterType("GEOGRAPHY") + INT64 = ScalarQueryParameterType("INT64") + INTEGER = ScalarQueryParameterType("INT64") + NUMERIC = ScalarQueryParameterType("NUMERIC") + STRING = ScalarQueryParameterType("STRING") + TIME = ScalarQueryParameterType("TIME") + TIMESTAMP = ScalarQueryParameterType("TIMESTAMP") + + class _QueryResults(object): """Results of a query. diff --git a/google/cloud/bigquery/routine/routine.py b/google/cloud/bigquery/routine/routine.py index a776212c3..677fb1178 100644 --- a/google/cloud/bigquery/routine/routine.py +++ b/google/cloud/bigquery/routine/routine.py @@ -16,12 +16,12 @@ """Define resources for the BigQuery Routines API.""" -from google.protobuf import json_format +from typing import Optional import google.cloud._helpers from google.cloud.bigquery import _helpers -import google.cloud.bigquery_v2.types -from google.cloud.bigquery_v2.types import StandardSqlTableType +from google.cloud.bigquery.standard_sql import StandardSqlDataType +from google.cloud.bigquery.standard_sql import StandardSqlTableType class RoutineType: @@ -190,7 +190,7 @@ def arguments(self, value): @property def return_type(self): - """google.cloud.bigquery_v2.types.StandardSqlDataType: Return type of + """google.cloud.bigquery.StandardSqlDataType: Return type of the routine. If absent, the return type is inferred from @@ -206,16 +206,12 @@ def return_type(self): if not resource: return resource - output = google.cloud.bigquery_v2.types.StandardSqlDataType() - raw_protobuf = json_format.ParseDict( - resource, output._pb, ignore_unknown_fields=True - ) - return type(output).wrap(raw_protobuf) + return StandardSqlDataType.from_api_repr(resource) @return_type.setter - def return_type(self, value): + def return_type(self, value: StandardSqlDataType): if value: - resource = json_format.MessageToDict(value._pb) + resource = value.to_api_repr() else: resource = None self._properties[self._PROPERTY_TO_API_FIELD["return_type"]] = resource @@ -232,20 +228,14 @@ def return_table_type(self) -> StandardSqlTableType: if not resource: return resource - output = google.cloud.bigquery_v2.types.StandardSqlTableType() - raw_protobuf = json_format.ParseDict( - resource, output._pb, ignore_unknown_fields=True - ) - return type(output).wrap(raw_protobuf) + return StandardSqlTableType.from_api_repr(resource) @return_table_type.setter - def return_table_type(self, value): + def return_table_type(self, value: Optional[StandardSqlTableType]): if not value: resource = None else: - resource = { - "columns": [json_format.MessageToDict(col._pb) for col in value.columns] - } + resource = value.to_api_repr() self._properties[self._PROPERTY_TO_API_FIELD["return_table_type"]] = resource @@ -407,7 +397,7 @@ def mode(self, value): @property def data_type(self): - """Optional[google.cloud.bigquery_v2.types.StandardSqlDataType]: Type + """Optional[google.cloud.bigquery.StandardSqlDataType]: Type of a variable, e.g., a function argument. See: @@ -417,16 +407,12 @@ def data_type(self): if not resource: return resource - output = google.cloud.bigquery_v2.types.StandardSqlDataType() - raw_protobuf = json_format.ParseDict( - resource, output._pb, ignore_unknown_fields=True - ) - return type(output).wrap(raw_protobuf) + return StandardSqlDataType.from_api_repr(resource) @data_type.setter def data_type(self, value): if value: - resource = json_format.MessageToDict(value._pb) + resource = value.to_api_repr() else: resource = None self._properties[self._PROPERTY_TO_API_FIELD["data_type"]] = resource diff --git a/google/cloud/bigquery/schema.py b/google/cloud/bigquery/schema.py index 157db7ce6..b52e288f4 100644 --- a/google/cloud/bigquery/schema.py +++ b/google/cloud/bigquery/schema.py @@ -17,7 +17,8 @@ import collections from typing import Optional -from google.cloud.bigquery_v2 import types +from google.cloud.bigquery import standard_sql +from google.cloud.bigquery.enums import StandardSqlTypeNames _DEFAULT_VALUE = object() @@ -27,26 +28,26 @@ # https://cloud.google.com/bigquery/data-types#legacy_sql_data_types # https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types LEGACY_TO_STANDARD_TYPES = { - "STRING": types.StandardSqlDataType.TypeKind.STRING, - "BYTES": types.StandardSqlDataType.TypeKind.BYTES, - "INTEGER": types.StandardSqlDataType.TypeKind.INT64, - "INT64": types.StandardSqlDataType.TypeKind.INT64, - "FLOAT": types.StandardSqlDataType.TypeKind.FLOAT64, - "FLOAT64": types.StandardSqlDataType.TypeKind.FLOAT64, - "NUMERIC": types.StandardSqlDataType.TypeKind.NUMERIC, - "BIGNUMERIC": types.StandardSqlDataType.TypeKind.BIGNUMERIC, - "BOOLEAN": types.StandardSqlDataType.TypeKind.BOOL, - "BOOL": types.StandardSqlDataType.TypeKind.BOOL, - "GEOGRAPHY": types.StandardSqlDataType.TypeKind.GEOGRAPHY, - "RECORD": types.StandardSqlDataType.TypeKind.STRUCT, - "STRUCT": types.StandardSqlDataType.TypeKind.STRUCT, - "TIMESTAMP": types.StandardSqlDataType.TypeKind.TIMESTAMP, - "DATE": types.StandardSqlDataType.TypeKind.DATE, - "TIME": types.StandardSqlDataType.TypeKind.TIME, - "DATETIME": types.StandardSqlDataType.TypeKind.DATETIME, + "STRING": StandardSqlTypeNames.STRING, + "BYTES": StandardSqlTypeNames.BYTES, + "INTEGER": StandardSqlTypeNames.INT64, + "INT64": StandardSqlTypeNames.INT64, + "FLOAT": StandardSqlTypeNames.FLOAT64, + "FLOAT64": StandardSqlTypeNames.FLOAT64, + "NUMERIC": StandardSqlTypeNames.NUMERIC, + "BIGNUMERIC": StandardSqlTypeNames.BIGNUMERIC, + "BOOLEAN": StandardSqlTypeNames.BOOL, + "BOOL": StandardSqlTypeNames.BOOL, + "GEOGRAPHY": StandardSqlTypeNames.GEOGRAPHY, + "RECORD": StandardSqlTypeNames.STRUCT, + "STRUCT": StandardSqlTypeNames.STRUCT, + "TIMESTAMP": StandardSqlTypeNames.TIMESTAMP, + "DATE": StandardSqlTypeNames.DATE, + "TIME": StandardSqlTypeNames.TIME, + "DATETIME": StandardSqlTypeNames.DATETIME, # no direct conversion from ARRAY, the latter is represented by mode="REPEATED" } -"""String names of the legacy SQL types to integer codes of Standard SQL types.""" +"""String names of the legacy SQL types to integer codes of Standard SQL standard_sql.""" class SchemaField(object): @@ -285,48 +286,37 @@ def _key(self): policy_tags, ) - def to_standard_sql(self) -> types.StandardSqlField: - """Return the field as the standard SQL field representation object. - - Returns: - An instance of :class:`~google.cloud.bigquery_v2.types.StandardSqlField`. - """ - sql_type = types.StandardSqlDataType() + def to_standard_sql(self) -> standard_sql.StandardSqlField: + """Return the field as the standard SQL field representation object.""" + sql_type = standard_sql.StandardSqlDataType() if self.mode == "REPEATED": - sql_type.type_kind = types.StandardSqlDataType.TypeKind.ARRAY + sql_type.type_kind = StandardSqlTypeNames.ARRAY else: sql_type.type_kind = LEGACY_TO_STANDARD_TYPES.get( - self.field_type, - types.StandardSqlDataType.TypeKind.TYPE_KIND_UNSPECIFIED, + self.field_type, StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED, ) - if sql_type.type_kind == types.StandardSqlDataType.TypeKind.ARRAY: # noqa: E721 + if sql_type.type_kind == StandardSqlTypeNames.ARRAY: # noqa: E721 array_element_type = LEGACY_TO_STANDARD_TYPES.get( - self.field_type, - types.StandardSqlDataType.TypeKind.TYPE_KIND_UNSPECIFIED, + self.field_type, StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED, + ) + sql_type.array_element_type = standard_sql.StandardSqlDataType( + type_kind=array_element_type ) - sql_type.array_element_type.type_kind = array_element_type # ARRAY cannot directly contain other arrays, only scalar types and STRUCTs # https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#array-type - if ( - array_element_type - == types.StandardSqlDataType.TypeKind.STRUCT # noqa: E721 - ): - sql_type.array_element_type.struct_type.fields.extend( - field.to_standard_sql() for field in self.fields + if array_element_type == StandardSqlTypeNames.STRUCT: # noqa: E721 + sql_type.array_element_type.struct_type = standard_sql.StandardSqlStructType( + fields=(field.to_standard_sql() for field in self.fields) ) - - elif ( - sql_type.type_kind - == types.StandardSqlDataType.TypeKind.STRUCT # noqa: E721 - ): - sql_type.struct_type.fields.extend( - field.to_standard_sql() for field in self.fields + elif sql_type.type_kind == StandardSqlTypeNames.STRUCT: # noqa: E721 + sql_type.struct_type = standard_sql.StandardSqlStructType( + fields=(field.to_standard_sql() for field in self.fields) ) - return types.StandardSqlField(name=self.name, type=sql_type) + return standard_sql.StandardSqlField(name=self.name, type=sql_type) def __eq__(self, other): if not isinstance(other, SchemaField): diff --git a/google/cloud/bigquery/standard_sql.py b/google/cloud/bigquery/standard_sql.py new file mode 100644 index 000000000..479929c74 --- /dev/null +++ b/google/cloud/bigquery/standard_sql.py @@ -0,0 +1,363 @@ +# Copyright 2021 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# https://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +from typing import Any, Dict, Iterable, List, Optional + +from google.cloud.bigquery.enums import StandardSqlTypeNames + + +class StandardSqlDataType: + """The type of a variable, e.g., a function argument. + + See: + https://cloud.google.com/bigquery/docs/reference/rest/v2/StandardSqlDataType + + Examples: + + .. code-block:: text + + INT64: {type_kind="INT64"} + ARRAY: {type_kind="ARRAY", array_element_type="STRING"} + STRUCT: { + type_kind="STRUCT", + struct_type={ + fields=[ + {name="x", type={type_kind="STRING"}}, + { + name="y", + type={type_kind="ARRAY", array_element_type="DATE"} + } + ] + } + } + + Args: + type_kind: + The top level type of this field. Can be any standard SQL data type, + e.g. INT64, DATE, ARRAY. + array_element_type: + The type of the array's elements, if type_kind is ARRAY. + struct_type: + The fields of this struct, in order, if type_kind is STRUCT. + """ + + def __init__( + self, + type_kind: Optional[ + StandardSqlTypeNames + ] = StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED, + array_element_type: Optional["StandardSqlDataType"] = None, + struct_type: Optional["StandardSqlStructType"] = None, + ): + self._properties = {} + + self.type_kind = type_kind + self.array_element_type = array_element_type + self.struct_type = struct_type + + @property + def type_kind(self) -> StandardSqlTypeNames: + """The top level type of this field. + + Can be any standard SQL data type, e.g. INT64, DATE, ARRAY. + """ + kind = self._properties["typeKind"] + return StandardSqlTypeNames[kind] # pytype: disable=missing-parameter + + @type_kind.setter + def type_kind(self, value: Optional[StandardSqlTypeNames]): + if not value: + kind = StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED.value + else: + kind = value.value + self._properties["typeKind"] = kind + + @property + def array_element_type(self) -> Optional["StandardSqlDataType"]: + """The type of the array's elements, if type_kind is ARRAY.""" + element_type = self._properties.get("arrayElementType") + + if element_type is None: + return None + + result = StandardSqlDataType() + result._properties = element_type # We do not use a copy on purpose. + return result + + @array_element_type.setter + def array_element_type(self, value: Optional["StandardSqlDataType"]): + element_type = None if value is None else value.to_api_repr() + + if element_type is None: + self._properties.pop("arrayElementType", None) + else: + self._properties["arrayElementType"] = element_type + + @property + def struct_type(self) -> Optional["StandardSqlStructType"]: + """The fields of this struct, in order, if type_kind is STRUCT.""" + struct_info = self._properties.get("structType") + + if struct_info is None: + return None + + result = StandardSqlStructType() + result._properties = struct_info # We do not use a copy on purpose. + return result + + @struct_type.setter + def struct_type(self, value: Optional["StandardSqlStructType"]): + struct_type = None if value is None else value.to_api_repr() + + if struct_type is None: + self._properties.pop("structType", None) + else: + self._properties["structType"] = struct_type + + def to_api_repr(self) -> Dict[str, Any]: + """Construct the API resource representation of this SQL data type.""" + return copy.deepcopy(self._properties) + + @classmethod + def from_api_repr(cls, resource: Dict[str, Any]): + """Construct an SQL data type instance given its API representation.""" + type_kind = resource.get("typeKind") + if type_kind not in StandardSqlTypeNames.__members__: + type_kind = StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED + else: + # Convert string to an enum member. + type_kind = StandardSqlTypeNames[ # pytype: disable=missing-parameter + type_kind + ] + + array_element_type = None + if type_kind == StandardSqlTypeNames.ARRAY: + element_type = resource.get("arrayElementType") + if element_type: + array_element_type = cls.from_api_repr(element_type) + + struct_type = None + if type_kind == StandardSqlTypeNames.STRUCT: + struct_info = resource.get("structType") + if struct_info: + struct_type = StandardSqlStructType.from_api_repr(struct_info) + + return cls(type_kind, array_element_type, struct_type) + + def __eq__(self, other): + if not isinstance(other, StandardSqlDataType): + return NotImplemented + else: + return ( + self.type_kind == other.type_kind + and self.array_element_type == other.array_element_type + and self.struct_type == other.struct_type + ) + + __hash__ = None + + def __str__(self): + result = f"{self.__class__.__name__}(type_kind={self.type_kind!r}, ...)" + return result + + +class StandardSqlField: + """A field or a column. + + See: + https://cloud.google.com/bigquery/docs/reference/rest/v2/StandardSqlField + + Args: + name: + The name of this field. Can be absent for struct fields. + type: + The type of this parameter. Absent if not explicitly specified. + + For example, CREATE FUNCTION statement can omit the return type; in this + case the output parameter does not have this "type" field). + """ + + def __init__( + self, name: Optional[str] = None, type: Optional[StandardSqlDataType] = None + ): + if type is not None: + type = type.to_api_repr() + + self._properties = {"name": name, "type": type} + + @property + def name(self) -> Optional[str]: + """The name of this field. Can be absent for struct fields.""" + return self._properties["name"] + + @name.setter + def name(self, value: Optional[str]): + self._properties["name"] = value + + @property + def type(self) -> Optional[StandardSqlDataType]: + """The type of this parameter. Absent if not explicitly specified. + + For example, CREATE FUNCTION statement can omit the return type; in this + case the output parameter does not have this "type" field). + """ + type_info = self._properties["type"] + + if type_info is None: + return None + + result = StandardSqlDataType() + result._properties = type_info # We do not use a copy on purpose. + return result + + @type.setter + def type(self, value: Optional[StandardSqlDataType]): + if value is not None: + value = value.to_api_repr() + self._properties["type"] = value + + def to_api_repr(self) -> Dict[str, Any]: + """Construct the API resource representation of this SQL field.""" + return copy.deepcopy(self._properties) + + @classmethod + def from_api_repr(cls, resource: Dict[str, Any]): + """Construct an SQL field instance given its API representation.""" + result = cls( + name=resource.get("name"), + type=StandardSqlDataType.from_api_repr(resource.get("type", {})), + ) + return result + + def __eq__(self, other): + if not isinstance(other, StandardSqlField): + return NotImplemented + else: + return self.name == other.name and self.type == other.type + + __hash__ = None + + +class StandardSqlStructType: + """Type of a struct field. + + See: + https://cloud.google.com/bigquery/docs/reference/rest/v2/StandardSqlDataType#StandardSqlStructType + + Args: + fields: The fields in this struct. + """ + + def __init__(self, fields: Optional[Iterable[StandardSqlField]] = None): + if fields is None: + fields = [] + self._properties = {"fields": [field.to_api_repr() for field in fields]} + + @property + def fields(self) -> List[StandardSqlField]: + """The fields in this struct.""" + result = [] + + for field_resource in self._properties.get("fields", []): + field = StandardSqlField() + field._properties = field_resource # We do not use a copy on purpose. + result.append(field) + + return result + + @fields.setter + def fields(self, value: Iterable[StandardSqlField]): + self._properties["fields"] = [field.to_api_repr() for field in value] + + def to_api_repr(self) -> Dict[str, Any]: + """Construct the API resource representation of this SQL struct type.""" + return copy.deepcopy(self._properties) + + @classmethod + def from_api_repr(cls, resource: Dict[str, Any]) -> "StandardSqlStructType": + """Construct an SQL struct type instance given its API representation.""" + fields = ( + StandardSqlField.from_api_repr(field_resource) + for field_resource in resource.get("fields", []) + ) + return cls(fields=fields) + + def __eq__(self, other): + if not isinstance(other, StandardSqlStructType): + return NotImplemented + else: + return self.fields == other.fields + + __hash__ = None + + +class StandardSqlTableType: + """A table type. + + See: + https://cloud.google.com/workflows/docs/reference/googleapis/bigquery/v2/Overview#StandardSqlTableType + + Args: + columns: The columns in this table type. + """ + + def __init__(self, columns: Iterable[StandardSqlField]): + self._properties = {"columns": [col.to_api_repr() for col in columns]} + + @property + def columns(self) -> List[StandardSqlField]: + """The columns in this table type.""" + result = [] + + for column_resource in self._properties.get("columns", []): + column = StandardSqlField() + column._properties = column_resource # We do not use a copy on purpose. + result.append(column) + + return result + + @columns.setter + def columns(self, value: Iterable[StandardSqlField]): + self._properties["columns"] = [col.to_api_repr() for col in value] + + def to_api_repr(self) -> Dict[str, Any]: + """Construct the API resource representation of this SQL table type.""" + return copy.deepcopy(self._properties) + + @classmethod + def from_api_repr(cls, resource: Dict[str, Any]) -> "StandardSqlTableType": + """Construct an SQL table type instance given its API representation.""" + columns = [] + + for column_resource in resource.get("columns", []): + type_ = column_resource.get("type") + if type_ is None: + type_ = {} + + column = StandardSqlField( + name=column_resource.get("name"), + type=StandardSqlDataType.from_api_repr(type_), + ) + columns.append(column) + + return cls(columns=columns) + + def __eq__(self, other): + if not isinstance(other, StandardSqlTableType): + return NotImplemented + else: + return self.columns == other.columns + + __hash__ = None diff --git a/google/cloud/bigquery_v2/__init__.py b/google/cloud/bigquery_v2/__init__.py deleted file mode 100644 index f9957efa9..000000000 --- a/google/cloud/bigquery_v2/__init__.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -from .types.encryption_config import EncryptionConfiguration -from .types.model import DeleteModelRequest -from .types.model import GetModelRequest -from .types.model import ListModelsRequest -from .types.model import ListModelsResponse -from .types.model import Model -from .types.model import PatchModelRequest -from .types.model_reference import ModelReference -from .types.standard_sql import StandardSqlDataType -from .types.standard_sql import StandardSqlField -from .types.standard_sql import StandardSqlStructType -from .types.standard_sql import StandardSqlTableType -from .types.table_reference import TableReference - -__all__ = ( - "DeleteModelRequest", - "EncryptionConfiguration", - "GetModelRequest", - "ListModelsRequest", - "ListModelsResponse", - "Model", - "ModelReference", - "PatchModelRequest", - "StandardSqlDataType", - "StandardSqlField", - "StandardSqlStructType", - "StandardSqlTableType", - "TableReference", -) diff --git a/google/cloud/bigquery_v2/gapic_metadata.json b/google/cloud/bigquery_v2/gapic_metadata.json deleted file mode 100644 index 3251a2630..000000000 --- a/google/cloud/bigquery_v2/gapic_metadata.json +++ /dev/null @@ -1,63 +0,0 @@ - { - "comment": "This file maps proto services/RPCs to the corresponding library clients/methods", - "language": "python", - "libraryPackage": "google.cloud.bigquery_v2", - "protoPackage": "google.cloud.bigquery.v2", - "schema": "1.0", - "services": { - "ModelService": { - "clients": { - "grpc": { - "libraryClient": "ModelServiceClient", - "rpcs": { - "DeleteModel": { - "methods": [ - "delete_model" - ] - }, - "GetModel": { - "methods": [ - "get_model" - ] - }, - "ListModels": { - "methods": [ - "list_models" - ] - }, - "PatchModel": { - "methods": [ - "patch_model" - ] - } - } - }, - "grpc-async": { - "libraryClient": "ModelServiceAsyncClient", - "rpcs": { - "DeleteModel": { - "methods": [ - "delete_model" - ] - }, - "GetModel": { - "methods": [ - "get_model" - ] - }, - "ListModels": { - "methods": [ - "list_models" - ] - }, - "PatchModel": { - "methods": [ - "patch_model" - ] - } - } - } - } - } - } -} diff --git a/google/cloud/bigquery_v2/py.typed b/google/cloud/bigquery_v2/py.typed deleted file mode 100644 index e73777993..000000000 --- a/google/cloud/bigquery_v2/py.typed +++ /dev/null @@ -1,2 +0,0 @@ -# Marker file for PEP 561. -# The google-cloud-bigquery package uses inline types. diff --git a/google/cloud/bigquery_v2/types/__init__.py b/google/cloud/bigquery_v2/types/__init__.py deleted file mode 100644 index 83bbb3a54..000000000 --- a/google/cloud/bigquery_v2/types/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from .encryption_config import EncryptionConfiguration -from .model import ( - DeleteModelRequest, - GetModelRequest, - ListModelsRequest, - ListModelsResponse, - Model, - PatchModelRequest, -) -from .model_reference import ModelReference -from .standard_sql import ( - StandardSqlDataType, - StandardSqlField, - StandardSqlStructType, - StandardSqlTableType, -) -from .table_reference import TableReference - -__all__ = ( - "EncryptionConfiguration", - "DeleteModelRequest", - "GetModelRequest", - "ListModelsRequest", - "ListModelsResponse", - "Model", - "PatchModelRequest", - "ModelReference", - "StandardSqlDataType", - "StandardSqlField", - "StandardSqlStructType", - "StandardSqlTableType", - "TableReference", -) diff --git a/google/cloud/bigquery_v2/types/encryption_config.py b/google/cloud/bigquery_v2/types/encryption_config.py deleted file mode 100644 index 4b9139733..000000000 --- a/google/cloud/bigquery_v2/types/encryption_config.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import proto # type: ignore - -from google.protobuf import wrappers_pb2 # type: ignore - - -__protobuf__ = proto.module( - package="google.cloud.bigquery.v2", manifest={"EncryptionConfiguration",}, -) - - -class EncryptionConfiguration(proto.Message): - r""" - Attributes: - kms_key_name (google.protobuf.wrappers_pb2.StringValue): - Optional. Describes the Cloud KMS encryption - key that will be used to protect destination - BigQuery table. The BigQuery Service Account - associated with your project requires access to - this encryption key. - """ - - kms_key_name = proto.Field( - proto.MESSAGE, number=1, message=wrappers_pb2.StringValue, - ) - - -__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/bigquery_v2/types/model.py b/google/cloud/bigquery_v2/types/model.py deleted file mode 100644 index 706418401..000000000 --- a/google/cloud/bigquery_v2/types/model.py +++ /dev/null @@ -1,1507 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import proto # type: ignore - -from google.cloud.bigquery_v2.types import encryption_config -from google.cloud.bigquery_v2.types import model_reference as gcb_model_reference -from google.cloud.bigquery_v2.types import standard_sql -from google.cloud.bigquery_v2.types import table_reference -from google.protobuf import timestamp_pb2 # type: ignore -from google.protobuf import wrappers_pb2 # type: ignore - - -__protobuf__ = proto.module( - package="google.cloud.bigquery.v2", - manifest={ - "Model", - "GetModelRequest", - "PatchModelRequest", - "DeleteModelRequest", - "ListModelsRequest", - "ListModelsResponse", - }, -) - - -class Model(proto.Message): - r""" - Attributes: - etag (str): - Output only. A hash of this resource. - model_reference (google.cloud.bigquery_v2.types.ModelReference): - Required. Unique identifier for this model. - creation_time (int): - Output only. The time when this model was - created, in millisecs since the epoch. - last_modified_time (int): - Output only. The time when this model was - last modified, in millisecs since the epoch. - description (str): - Optional. A user-friendly description of this - model. - friendly_name (str): - Optional. A descriptive name for this model. - labels (Sequence[google.cloud.bigquery_v2.types.Model.LabelsEntry]): - The labels associated with this model. You - can use these to organize and group your models. - Label keys and values can be no longer than 63 - characters, can only contain lowercase letters, - numeric characters, underscores and dashes. - International characters are allowed. Label - values are optional. Label keys must start with - a letter and each label in the list must have a - different key. - expiration_time (int): - Optional. The time when this model expires, - in milliseconds since the epoch. If not present, - the model will persist indefinitely. Expired - models will be deleted and their storage - reclaimed. The defaultTableExpirationMs - property of the encapsulating dataset can be - used to set a default expirationTime on newly - created models. - location (str): - Output only. The geographic location where - the model resides. This value is inherited from - the dataset. - encryption_configuration (google.cloud.bigquery_v2.types.EncryptionConfiguration): - Custom encryption configuration (e.g., Cloud - KMS keys). This shows the encryption - configuration of the model data while stored in - BigQuery storage. This field can be used with - PatchModel to update encryption key for an - already encrypted model. - model_type (google.cloud.bigquery_v2.types.Model.ModelType): - Output only. Type of the model resource. - training_runs (Sequence[google.cloud.bigquery_v2.types.Model.TrainingRun]): - Output only. Information for all training runs in increasing - order of start_time. - feature_columns (Sequence[google.cloud.bigquery_v2.types.StandardSqlField]): - Output only. Input feature columns that were - used to train this model. - label_columns (Sequence[google.cloud.bigquery_v2.types.StandardSqlField]): - Output only. Label columns that were used to train this - model. The output of the model will have a `predicted_` - prefix to these columns. - best_trial_id (int): - The best trial_id across all training runs. - """ - - class ModelType(proto.Enum): - r"""Indicates the type of the Model.""" - MODEL_TYPE_UNSPECIFIED = 0 - LINEAR_REGRESSION = 1 - LOGISTIC_REGRESSION = 2 - KMEANS = 3 - MATRIX_FACTORIZATION = 4 - DNN_CLASSIFIER = 5 - TENSORFLOW = 6 - DNN_REGRESSOR = 7 - BOOSTED_TREE_REGRESSOR = 9 - BOOSTED_TREE_CLASSIFIER = 10 - ARIMA = 11 - AUTOML_REGRESSOR = 12 - AUTOML_CLASSIFIER = 13 - ARIMA_PLUS = 19 - - class LossType(proto.Enum): - r"""Loss metric to evaluate model training performance.""" - LOSS_TYPE_UNSPECIFIED = 0 - MEAN_SQUARED_LOSS = 1 - MEAN_LOG_LOSS = 2 - - class DistanceType(proto.Enum): - r"""Distance metric used to compute the distance between two - points. - """ - DISTANCE_TYPE_UNSPECIFIED = 0 - EUCLIDEAN = 1 - COSINE = 2 - - class DataSplitMethod(proto.Enum): - r"""Indicates the method to split input data into multiple - tables. - """ - DATA_SPLIT_METHOD_UNSPECIFIED = 0 - RANDOM = 1 - CUSTOM = 2 - SEQUENTIAL = 3 - NO_SPLIT = 4 - AUTO_SPLIT = 5 - - class DataFrequency(proto.Enum): - r"""Type of supported data frequency for time series forecasting - models. - """ - DATA_FREQUENCY_UNSPECIFIED = 0 - AUTO_FREQUENCY = 1 - YEARLY = 2 - QUARTERLY = 3 - MONTHLY = 4 - WEEKLY = 5 - DAILY = 6 - HOURLY = 7 - PER_MINUTE = 8 - - class HolidayRegion(proto.Enum): - r"""Type of supported holiday regions for time series forecasting - models. - """ - HOLIDAY_REGION_UNSPECIFIED = 0 - GLOBAL = 1 - NA = 2 - JAPAC = 3 - EMEA = 4 - LAC = 5 - AE = 6 - AR = 7 - AT = 8 - AU = 9 - BE = 10 - BR = 11 - CA = 12 - CH = 13 - CL = 14 - CN = 15 - CO = 16 - CS = 17 - CZ = 18 - DE = 19 - DK = 20 - DZ = 21 - EC = 22 - EE = 23 - EG = 24 - ES = 25 - FI = 26 - FR = 27 - GB = 28 - GR = 29 - HK = 30 - HU = 31 - ID = 32 - IE = 33 - IL = 34 - IN = 35 - IR = 36 - IT = 37 - JP = 38 - KR = 39 - LV = 40 - MA = 41 - MX = 42 - MY = 43 - NG = 44 - NL = 45 - NO = 46 - NZ = 47 - PE = 48 - PH = 49 - PK = 50 - PL = 51 - PT = 52 - RO = 53 - RS = 54 - RU = 55 - SA = 56 - SE = 57 - SG = 58 - SI = 59 - SK = 60 - TH = 61 - TR = 62 - TW = 63 - UA = 64 - US = 65 - VE = 66 - VN = 67 - ZA = 68 - - class LearnRateStrategy(proto.Enum): - r"""Indicates the learning rate optimization strategy to use.""" - LEARN_RATE_STRATEGY_UNSPECIFIED = 0 - LINE_SEARCH = 1 - CONSTANT = 2 - - class OptimizationStrategy(proto.Enum): - r"""Indicates the optimization strategy used for training.""" - OPTIMIZATION_STRATEGY_UNSPECIFIED = 0 - BATCH_GRADIENT_DESCENT = 1 - NORMAL_EQUATION = 2 - - class FeedbackType(proto.Enum): - r"""Indicates the training algorithm to use for matrix - factorization models. - """ - FEEDBACK_TYPE_UNSPECIFIED = 0 - IMPLICIT = 1 - EXPLICIT = 2 - - class SeasonalPeriod(proto.Message): - r""" """ - - class SeasonalPeriodType(proto.Enum): - r"""""" - SEASONAL_PERIOD_TYPE_UNSPECIFIED = 0 - NO_SEASONALITY = 1 - DAILY = 2 - WEEKLY = 3 - MONTHLY = 4 - QUARTERLY = 5 - YEARLY = 6 - - class KmeansEnums(proto.Message): - r""" """ - - class KmeansInitializationMethod(proto.Enum): - r"""Indicates the method used to initialize the centroids for - KMeans clustering algorithm. - """ - KMEANS_INITIALIZATION_METHOD_UNSPECIFIED = 0 - RANDOM = 1 - CUSTOM = 2 - KMEANS_PLUS_PLUS = 3 - - class RegressionMetrics(proto.Message): - r"""Evaluation metrics for regression and explicit feedback type - matrix factorization models. - - Attributes: - mean_absolute_error (google.protobuf.wrappers_pb2.DoubleValue): - Mean absolute error. - mean_squared_error (google.protobuf.wrappers_pb2.DoubleValue): - Mean squared error. - mean_squared_log_error (google.protobuf.wrappers_pb2.DoubleValue): - Mean squared log error. - median_absolute_error (google.protobuf.wrappers_pb2.DoubleValue): - Median absolute error. - r_squared (google.protobuf.wrappers_pb2.DoubleValue): - R^2 score. This corresponds to r2_score in ML.EVALUATE. - """ - - mean_absolute_error = proto.Field( - proto.MESSAGE, number=1, message=wrappers_pb2.DoubleValue, - ) - mean_squared_error = proto.Field( - proto.MESSAGE, number=2, message=wrappers_pb2.DoubleValue, - ) - mean_squared_log_error = proto.Field( - proto.MESSAGE, number=3, message=wrappers_pb2.DoubleValue, - ) - median_absolute_error = proto.Field( - proto.MESSAGE, number=4, message=wrappers_pb2.DoubleValue, - ) - r_squared = proto.Field( - proto.MESSAGE, number=5, message=wrappers_pb2.DoubleValue, - ) - - class AggregateClassificationMetrics(proto.Message): - r"""Aggregate metrics for classification/classifier models. For - multi-class models, the metrics are either macro-averaged or - micro-averaged. When macro-averaged, the metrics are calculated - for each label and then an unweighted average is taken of those - values. When micro-averaged, the metric is calculated globally - by counting the total number of correctly predicted rows. - - Attributes: - precision (google.protobuf.wrappers_pb2.DoubleValue): - Precision is the fraction of actual positive - predictions that had positive actual labels. For - multiclass this is a macro-averaged metric - treating each class as a binary classifier. - recall (google.protobuf.wrappers_pb2.DoubleValue): - Recall is the fraction of actual positive - labels that were given a positive prediction. - For multiclass this is a macro-averaged metric. - accuracy (google.protobuf.wrappers_pb2.DoubleValue): - Accuracy is the fraction of predictions given - the correct label. For multiclass this is a - micro-averaged metric. - threshold (google.protobuf.wrappers_pb2.DoubleValue): - Threshold at which the metrics are computed. - For binary classification models this is the - positive class threshold. For multi-class - classfication models this is the confidence - threshold. - f1_score (google.protobuf.wrappers_pb2.DoubleValue): - The F1 score is an average of recall and - precision. For multiclass this is a macro- - averaged metric. - log_loss (google.protobuf.wrappers_pb2.DoubleValue): - Logarithmic Loss. For multiclass this is a - macro-averaged metric. - roc_auc (google.protobuf.wrappers_pb2.DoubleValue): - Area Under a ROC Curve. For multiclass this - is a macro-averaged metric. - """ - - precision = proto.Field( - proto.MESSAGE, number=1, message=wrappers_pb2.DoubleValue, - ) - recall = proto.Field(proto.MESSAGE, number=2, message=wrappers_pb2.DoubleValue,) - accuracy = proto.Field( - proto.MESSAGE, number=3, message=wrappers_pb2.DoubleValue, - ) - threshold = proto.Field( - proto.MESSAGE, number=4, message=wrappers_pb2.DoubleValue, - ) - f1_score = proto.Field( - proto.MESSAGE, number=5, message=wrappers_pb2.DoubleValue, - ) - log_loss = proto.Field( - proto.MESSAGE, number=6, message=wrappers_pb2.DoubleValue, - ) - roc_auc = proto.Field( - proto.MESSAGE, number=7, message=wrappers_pb2.DoubleValue, - ) - - class BinaryClassificationMetrics(proto.Message): - r"""Evaluation metrics for binary classification/classifier - models. - - Attributes: - aggregate_classification_metrics (google.cloud.bigquery_v2.types.Model.AggregateClassificationMetrics): - Aggregate classification metrics. - binary_confusion_matrix_list (Sequence[google.cloud.bigquery_v2.types.Model.BinaryClassificationMetrics.BinaryConfusionMatrix]): - Binary confusion matrix at multiple - thresholds. - positive_label (str): - Label representing the positive class. - negative_label (str): - Label representing the negative class. - """ - - class BinaryConfusionMatrix(proto.Message): - r"""Confusion matrix for binary classification models. - Attributes: - positive_class_threshold (google.protobuf.wrappers_pb2.DoubleValue): - Threshold value used when computing each of - the following metric. - true_positives (google.protobuf.wrappers_pb2.Int64Value): - Number of true samples predicted as true. - false_positives (google.protobuf.wrappers_pb2.Int64Value): - Number of false samples predicted as true. - true_negatives (google.protobuf.wrappers_pb2.Int64Value): - Number of true samples predicted as false. - false_negatives (google.protobuf.wrappers_pb2.Int64Value): - Number of false samples predicted as false. - precision (google.protobuf.wrappers_pb2.DoubleValue): - The fraction of actual positive predictions - that had positive actual labels. - recall (google.protobuf.wrappers_pb2.DoubleValue): - The fraction of actual positive labels that - were given a positive prediction. - f1_score (google.protobuf.wrappers_pb2.DoubleValue): - The equally weighted average of recall and - precision. - accuracy (google.protobuf.wrappers_pb2.DoubleValue): - The fraction of predictions given the correct - label. - """ - - positive_class_threshold = proto.Field( - proto.MESSAGE, number=1, message=wrappers_pb2.DoubleValue, - ) - true_positives = proto.Field( - proto.MESSAGE, number=2, message=wrappers_pb2.Int64Value, - ) - false_positives = proto.Field( - proto.MESSAGE, number=3, message=wrappers_pb2.Int64Value, - ) - true_negatives = proto.Field( - proto.MESSAGE, number=4, message=wrappers_pb2.Int64Value, - ) - false_negatives = proto.Field( - proto.MESSAGE, number=5, message=wrappers_pb2.Int64Value, - ) - precision = proto.Field( - proto.MESSAGE, number=6, message=wrappers_pb2.DoubleValue, - ) - recall = proto.Field( - proto.MESSAGE, number=7, message=wrappers_pb2.DoubleValue, - ) - f1_score = proto.Field( - proto.MESSAGE, number=8, message=wrappers_pb2.DoubleValue, - ) - accuracy = proto.Field( - proto.MESSAGE, number=9, message=wrappers_pb2.DoubleValue, - ) - - aggregate_classification_metrics = proto.Field( - proto.MESSAGE, number=1, message="Model.AggregateClassificationMetrics", - ) - binary_confusion_matrix_list = proto.RepeatedField( - proto.MESSAGE, - number=2, - message="Model.BinaryClassificationMetrics.BinaryConfusionMatrix", - ) - positive_label = proto.Field(proto.STRING, number=3,) - negative_label = proto.Field(proto.STRING, number=4,) - - class MultiClassClassificationMetrics(proto.Message): - r"""Evaluation metrics for multi-class classification/classifier - models. - - Attributes: - aggregate_classification_metrics (google.cloud.bigquery_v2.types.Model.AggregateClassificationMetrics): - Aggregate classification metrics. - confusion_matrix_list (Sequence[google.cloud.bigquery_v2.types.Model.MultiClassClassificationMetrics.ConfusionMatrix]): - Confusion matrix at different thresholds. - """ - - class ConfusionMatrix(proto.Message): - r"""Confusion matrix for multi-class classification models. - Attributes: - confidence_threshold (google.protobuf.wrappers_pb2.DoubleValue): - Confidence threshold used when computing the - entries of the confusion matrix. - rows (Sequence[google.cloud.bigquery_v2.types.Model.MultiClassClassificationMetrics.ConfusionMatrix.Row]): - One row per actual label. - """ - - class Entry(proto.Message): - r"""A single entry in the confusion matrix. - Attributes: - predicted_label (str): - The predicted label. For confidence_threshold > 0, we will - also add an entry indicating the number of items under the - confidence threshold. - item_count (google.protobuf.wrappers_pb2.Int64Value): - Number of items being predicted as this - label. - """ - - predicted_label = proto.Field(proto.STRING, number=1,) - item_count = proto.Field( - proto.MESSAGE, number=2, message=wrappers_pb2.Int64Value, - ) - - class Row(proto.Message): - r"""A single row in the confusion matrix. - Attributes: - actual_label (str): - The original label of this row. - entries (Sequence[google.cloud.bigquery_v2.types.Model.MultiClassClassificationMetrics.ConfusionMatrix.Entry]): - Info describing predicted label distribution. - """ - - actual_label = proto.Field(proto.STRING, number=1,) - entries = proto.RepeatedField( - proto.MESSAGE, - number=2, - message="Model.MultiClassClassificationMetrics.ConfusionMatrix.Entry", - ) - - confidence_threshold = proto.Field( - proto.MESSAGE, number=1, message=wrappers_pb2.DoubleValue, - ) - rows = proto.RepeatedField( - proto.MESSAGE, - number=2, - message="Model.MultiClassClassificationMetrics.ConfusionMatrix.Row", - ) - - aggregate_classification_metrics = proto.Field( - proto.MESSAGE, number=1, message="Model.AggregateClassificationMetrics", - ) - confusion_matrix_list = proto.RepeatedField( - proto.MESSAGE, - number=2, - message="Model.MultiClassClassificationMetrics.ConfusionMatrix", - ) - - class ClusteringMetrics(proto.Message): - r"""Evaluation metrics for clustering models. - Attributes: - davies_bouldin_index (google.protobuf.wrappers_pb2.DoubleValue): - Davies-Bouldin index. - mean_squared_distance (google.protobuf.wrappers_pb2.DoubleValue): - Mean of squared distances between each sample - to its cluster centroid. - clusters (Sequence[google.cloud.bigquery_v2.types.Model.ClusteringMetrics.Cluster]): - Information for all clusters. - """ - - class Cluster(proto.Message): - r"""Message containing the information about one cluster. - Attributes: - centroid_id (int): - Centroid id. - feature_values (Sequence[google.cloud.bigquery_v2.types.Model.ClusteringMetrics.Cluster.FeatureValue]): - Values of highly variant features for this - cluster. - count (google.protobuf.wrappers_pb2.Int64Value): - Count of training data rows that were - assigned to this cluster. - """ - - class FeatureValue(proto.Message): - r"""Representative value of a single feature within the cluster. - Attributes: - feature_column (str): - The feature column name. - numerical_value (google.protobuf.wrappers_pb2.DoubleValue): - The numerical feature value. This is the - centroid value for this feature. - categorical_value (google.cloud.bigquery_v2.types.Model.ClusteringMetrics.Cluster.FeatureValue.CategoricalValue): - The categorical feature value. - """ - - class CategoricalValue(proto.Message): - r"""Representative value of a categorical feature. - Attributes: - category_counts (Sequence[google.cloud.bigquery_v2.types.Model.ClusteringMetrics.Cluster.FeatureValue.CategoricalValue.CategoryCount]): - Counts of all categories for the categorical feature. If - there are more than ten categories, we return top ten (by - count) and return one more CategoryCount with category - "*OTHER*" and count as aggregate counts of remaining - categories. - """ - - class CategoryCount(proto.Message): - r"""Represents the count of a single category within the cluster. - Attributes: - category (str): - The name of category. - count (google.protobuf.wrappers_pb2.Int64Value): - The count of training samples matching the - category within the cluster. - """ - - category = proto.Field(proto.STRING, number=1,) - count = proto.Field( - proto.MESSAGE, number=2, message=wrappers_pb2.Int64Value, - ) - - category_counts = proto.RepeatedField( - proto.MESSAGE, - number=1, - message="Model.ClusteringMetrics.Cluster.FeatureValue.CategoricalValue.CategoryCount", - ) - - feature_column = proto.Field(proto.STRING, number=1,) - numerical_value = proto.Field( - proto.MESSAGE, - number=2, - oneof="value", - message=wrappers_pb2.DoubleValue, - ) - categorical_value = proto.Field( - proto.MESSAGE, - number=3, - oneof="value", - message="Model.ClusteringMetrics.Cluster.FeatureValue.CategoricalValue", - ) - - centroid_id = proto.Field(proto.INT64, number=1,) - feature_values = proto.RepeatedField( - proto.MESSAGE, - number=2, - message="Model.ClusteringMetrics.Cluster.FeatureValue", - ) - count = proto.Field( - proto.MESSAGE, number=3, message=wrappers_pb2.Int64Value, - ) - - davies_bouldin_index = proto.Field( - proto.MESSAGE, number=1, message=wrappers_pb2.DoubleValue, - ) - mean_squared_distance = proto.Field( - proto.MESSAGE, number=2, message=wrappers_pb2.DoubleValue, - ) - clusters = proto.RepeatedField( - proto.MESSAGE, number=3, message="Model.ClusteringMetrics.Cluster", - ) - - class RankingMetrics(proto.Message): - r"""Evaluation metrics used by weighted-ALS models specified by - feedback_type=implicit. - - Attributes: - mean_average_precision (google.protobuf.wrappers_pb2.DoubleValue): - Calculates a precision per user for all the - items by ranking them and then averages all the - precisions across all the users. - mean_squared_error (google.protobuf.wrappers_pb2.DoubleValue): - Similar to the mean squared error computed in - regression and explicit recommendation models - except instead of computing the rating directly, - the output from evaluate is computed against a - preference which is 1 or 0 depending on if the - rating exists or not. - normalized_discounted_cumulative_gain (google.protobuf.wrappers_pb2.DoubleValue): - A metric to determine the goodness of a - ranking calculated from the predicted confidence - by comparing it to an ideal rank measured by the - original ratings. - average_rank (google.protobuf.wrappers_pb2.DoubleValue): - Determines the goodness of a ranking by - computing the percentile rank from the predicted - confidence and dividing it by the original rank. - """ - - mean_average_precision = proto.Field( - proto.MESSAGE, number=1, message=wrappers_pb2.DoubleValue, - ) - mean_squared_error = proto.Field( - proto.MESSAGE, number=2, message=wrappers_pb2.DoubleValue, - ) - normalized_discounted_cumulative_gain = proto.Field( - proto.MESSAGE, number=3, message=wrappers_pb2.DoubleValue, - ) - average_rank = proto.Field( - proto.MESSAGE, number=4, message=wrappers_pb2.DoubleValue, - ) - - class ArimaForecastingMetrics(proto.Message): - r"""Model evaluation metrics for ARIMA forecasting models. - Attributes: - non_seasonal_order (Sequence[google.cloud.bigquery_v2.types.Model.ArimaOrder]): - Non-seasonal order. - arima_fitting_metrics (Sequence[google.cloud.bigquery_v2.types.Model.ArimaFittingMetrics]): - Arima model fitting metrics. - seasonal_periods (Sequence[google.cloud.bigquery_v2.types.Model.SeasonalPeriod.SeasonalPeriodType]): - Seasonal periods. Repeated because multiple - periods are supported for one time series. - has_drift (Sequence[bool]): - Whether Arima model fitted with drift or not. - It is always false when d is not 1. - time_series_id (Sequence[str]): - Id to differentiate different time series for - the large-scale case. - arima_single_model_forecasting_metrics (Sequence[google.cloud.bigquery_v2.types.Model.ArimaForecastingMetrics.ArimaSingleModelForecastingMetrics]): - Repeated as there can be many metric sets - (one for each model) in auto-arima and the - large-scale case. - """ - - class ArimaSingleModelForecastingMetrics(proto.Message): - r"""Model evaluation metrics for a single ARIMA forecasting - model. - - Attributes: - non_seasonal_order (google.cloud.bigquery_v2.types.Model.ArimaOrder): - Non-seasonal order. - arima_fitting_metrics (google.cloud.bigquery_v2.types.Model.ArimaFittingMetrics): - Arima fitting metrics. - has_drift (bool): - Is arima model fitted with drift or not. It - is always false when d is not 1. - time_series_id (str): - The time_series_id value for this time series. It will be - one of the unique values from the time_series_id_column - specified during ARIMA model training. Only present when - time_series_id_column training option was used. - time_series_ids (Sequence[str]): - The tuple of time_series_ids identifying this time series. - It will be one of the unique tuples of values present in the - time_series_id_columns specified during ARIMA model - training. Only present when time_series_id_columns training - option was used and the order of values here are same as the - order of time_series_id_columns. - seasonal_periods (Sequence[google.cloud.bigquery_v2.types.Model.SeasonalPeriod.SeasonalPeriodType]): - Seasonal periods. Repeated because multiple - periods are supported for one time series. - has_holiday_effect (google.protobuf.wrappers_pb2.BoolValue): - If true, holiday_effect is a part of time series - decomposition result. - has_spikes_and_dips (google.protobuf.wrappers_pb2.BoolValue): - If true, spikes_and_dips is a part of time series - decomposition result. - has_step_changes (google.protobuf.wrappers_pb2.BoolValue): - If true, step_changes is a part of time series decomposition - result. - """ - - non_seasonal_order = proto.Field( - proto.MESSAGE, number=1, message="Model.ArimaOrder", - ) - arima_fitting_metrics = proto.Field( - proto.MESSAGE, number=2, message="Model.ArimaFittingMetrics", - ) - has_drift = proto.Field(proto.BOOL, number=3,) - time_series_id = proto.Field(proto.STRING, number=4,) - time_series_ids = proto.RepeatedField(proto.STRING, number=9,) - seasonal_periods = proto.RepeatedField( - proto.ENUM, number=5, enum="Model.SeasonalPeriod.SeasonalPeriodType", - ) - has_holiday_effect = proto.Field( - proto.MESSAGE, number=6, message=wrappers_pb2.BoolValue, - ) - has_spikes_and_dips = proto.Field( - proto.MESSAGE, number=7, message=wrappers_pb2.BoolValue, - ) - has_step_changes = proto.Field( - proto.MESSAGE, number=8, message=wrappers_pb2.BoolValue, - ) - - non_seasonal_order = proto.RepeatedField( - proto.MESSAGE, number=1, message="Model.ArimaOrder", - ) - arima_fitting_metrics = proto.RepeatedField( - proto.MESSAGE, number=2, message="Model.ArimaFittingMetrics", - ) - seasonal_periods = proto.RepeatedField( - proto.ENUM, number=3, enum="Model.SeasonalPeriod.SeasonalPeriodType", - ) - has_drift = proto.RepeatedField(proto.BOOL, number=4,) - time_series_id = proto.RepeatedField(proto.STRING, number=5,) - arima_single_model_forecasting_metrics = proto.RepeatedField( - proto.MESSAGE, - number=6, - message="Model.ArimaForecastingMetrics.ArimaSingleModelForecastingMetrics", - ) - - class EvaluationMetrics(proto.Message): - r"""Evaluation metrics of a model. These are either computed on - all training data or just the eval data based on whether eval - data was used during training. These are not present for - imported models. - - Attributes: - regression_metrics (google.cloud.bigquery_v2.types.Model.RegressionMetrics): - Populated for regression models and explicit - feedback type matrix factorization models. - binary_classification_metrics (google.cloud.bigquery_v2.types.Model.BinaryClassificationMetrics): - Populated for binary - classification/classifier models. - multi_class_classification_metrics (google.cloud.bigquery_v2.types.Model.MultiClassClassificationMetrics): - Populated for multi-class - classification/classifier models. - clustering_metrics (google.cloud.bigquery_v2.types.Model.ClusteringMetrics): - Populated for clustering models. - ranking_metrics (google.cloud.bigquery_v2.types.Model.RankingMetrics): - Populated for implicit feedback type matrix - factorization models. - arima_forecasting_metrics (google.cloud.bigquery_v2.types.Model.ArimaForecastingMetrics): - Populated for ARIMA models. - """ - - regression_metrics = proto.Field( - proto.MESSAGE, number=1, oneof="metrics", message="Model.RegressionMetrics", - ) - binary_classification_metrics = proto.Field( - proto.MESSAGE, - number=2, - oneof="metrics", - message="Model.BinaryClassificationMetrics", - ) - multi_class_classification_metrics = proto.Field( - proto.MESSAGE, - number=3, - oneof="metrics", - message="Model.MultiClassClassificationMetrics", - ) - clustering_metrics = proto.Field( - proto.MESSAGE, number=4, oneof="metrics", message="Model.ClusteringMetrics", - ) - ranking_metrics = proto.Field( - proto.MESSAGE, number=5, oneof="metrics", message="Model.RankingMetrics", - ) - arima_forecasting_metrics = proto.Field( - proto.MESSAGE, - number=6, - oneof="metrics", - message="Model.ArimaForecastingMetrics", - ) - - class DataSplitResult(proto.Message): - r"""Data split result. This contains references to the training - and evaluation data tables that were used to train the model. - - Attributes: - training_table (google.cloud.bigquery_v2.types.TableReference): - Table reference of the training data after - split. - evaluation_table (google.cloud.bigquery_v2.types.TableReference): - Table reference of the evaluation data after - split. - """ - - training_table = proto.Field( - proto.MESSAGE, number=1, message=table_reference.TableReference, - ) - evaluation_table = proto.Field( - proto.MESSAGE, number=2, message=table_reference.TableReference, - ) - - class ArimaOrder(proto.Message): - r"""Arima order, can be used for both non-seasonal and seasonal - parts. - - Attributes: - p (int): - Order of the autoregressive part. - d (int): - Order of the differencing part. - q (int): - Order of the moving-average part. - """ - - p = proto.Field(proto.INT64, number=1,) - d = proto.Field(proto.INT64, number=2,) - q = proto.Field(proto.INT64, number=3,) - - class ArimaFittingMetrics(proto.Message): - r"""ARIMA model fitting metrics. - Attributes: - log_likelihood (float): - Log-likelihood. - aic (float): - AIC. - variance (float): - Variance. - """ - - log_likelihood = proto.Field(proto.DOUBLE, number=1,) - aic = proto.Field(proto.DOUBLE, number=2,) - variance = proto.Field(proto.DOUBLE, number=3,) - - class GlobalExplanation(proto.Message): - r"""Global explanations containing the top most important - features after training. - - Attributes: - explanations (Sequence[google.cloud.bigquery_v2.types.Model.GlobalExplanation.Explanation]): - A list of the top global explanations. Sorted - by absolute value of attribution in descending - order. - class_label (str): - Class label for this set of global - explanations. Will be empty/null for binary - logistic and linear regression models. Sorted - alphabetically in descending order. - """ - - class Explanation(proto.Message): - r"""Explanation for a single feature. - Attributes: - feature_name (str): - Full name of the feature. For non-numerical features, will - be formatted like .. - Overall size of feature name will always be truncated to - first 120 characters. - attribution (google.protobuf.wrappers_pb2.DoubleValue): - Attribution of feature. - """ - - feature_name = proto.Field(proto.STRING, number=1,) - attribution = proto.Field( - proto.MESSAGE, number=2, message=wrappers_pb2.DoubleValue, - ) - - explanations = proto.RepeatedField( - proto.MESSAGE, number=1, message="Model.GlobalExplanation.Explanation", - ) - class_label = proto.Field(proto.STRING, number=2,) - - class TrainingRun(proto.Message): - r"""Information about a single training query run for the model. - Attributes: - training_options (google.cloud.bigquery_v2.types.Model.TrainingRun.TrainingOptions): - Options that were used for this training run, - includes user specified and default options that - were used. - start_time (google.protobuf.timestamp_pb2.Timestamp): - The start time of this training run. - results (Sequence[google.cloud.bigquery_v2.types.Model.TrainingRun.IterationResult]): - Output of each iteration run, results.size() <= - max_iterations. - evaluation_metrics (google.cloud.bigquery_v2.types.Model.EvaluationMetrics): - The evaluation metrics over training/eval - data that were computed at the end of training. - data_split_result (google.cloud.bigquery_v2.types.Model.DataSplitResult): - Data split result of the training run. Only - set when the input data is actually split. - global_explanations (Sequence[google.cloud.bigquery_v2.types.Model.GlobalExplanation]): - Global explanations for important features of - the model. For multi-class models, there is one - entry for each label class. For other models, - there is only one entry in the list. - """ - - class TrainingOptions(proto.Message): - r"""Options used in model training. - Attributes: - max_iterations (int): - The maximum number of iterations in training. - Used only for iterative training algorithms. - loss_type (google.cloud.bigquery_v2.types.Model.LossType): - Type of loss function used during training - run. - learn_rate (float): - Learning rate in training. Used only for - iterative training algorithms. - l1_regularization (google.protobuf.wrappers_pb2.DoubleValue): - L1 regularization coefficient. - l2_regularization (google.protobuf.wrappers_pb2.DoubleValue): - L2 regularization coefficient. - min_relative_progress (google.protobuf.wrappers_pb2.DoubleValue): - When early_stop is true, stops training when accuracy - improvement is less than 'min_relative_progress'. Used only - for iterative training algorithms. - warm_start (google.protobuf.wrappers_pb2.BoolValue): - Whether to train a model from the last - checkpoint. - early_stop (google.protobuf.wrappers_pb2.BoolValue): - Whether to stop early when the loss doesn't improve - significantly any more (compared to min_relative_progress). - Used only for iterative training algorithms. - input_label_columns (Sequence[str]): - Name of input label columns in training data. - data_split_method (google.cloud.bigquery_v2.types.Model.DataSplitMethod): - The data split type for training and - evaluation, e.g. RANDOM. - data_split_eval_fraction (float): - The fraction of evaluation data over the - whole input data. The rest of data will be used - as training data. The format should be double. - Accurate to two decimal places. - Default value is 0.2. - data_split_column (str): - The column to split data with. This column won't be used as - a feature. - - 1. When data_split_method is CUSTOM, the corresponding - column should be boolean. The rows with true value tag - are eval data, and the false are training data. - 2. When data_split_method is SEQ, the first - DATA_SPLIT_EVAL_FRACTION rows (from smallest to largest) - in the corresponding column are used as training data, - and the rest are eval data. It respects the order in - Orderable data types: - https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#data-type-properties - learn_rate_strategy (google.cloud.bigquery_v2.types.Model.LearnRateStrategy): - The strategy to determine learn rate for the - current iteration. - initial_learn_rate (float): - Specifies the initial learning rate for the - line search learn rate strategy. - label_class_weights (Sequence[google.cloud.bigquery_v2.types.Model.TrainingRun.TrainingOptions.LabelClassWeightsEntry]): - Weights associated with each label class, for - rebalancing the training data. Only applicable - for classification models. - user_column (str): - User column specified for matrix - factorization models. - item_column (str): - Item column specified for matrix - factorization models. - distance_type (google.cloud.bigquery_v2.types.Model.DistanceType): - Distance type for clustering models. - num_clusters (int): - Number of clusters for clustering models. - model_uri (str): - Google Cloud Storage URI from which the model - was imported. Only applicable for imported - models. - optimization_strategy (google.cloud.bigquery_v2.types.Model.OptimizationStrategy): - Optimization strategy for training linear - regression models. - hidden_units (Sequence[int]): - Hidden units for dnn models. - batch_size (int): - Batch size for dnn models. - dropout (google.protobuf.wrappers_pb2.DoubleValue): - Dropout probability for dnn models. - max_tree_depth (int): - Maximum depth of a tree for boosted tree - models. - subsample (float): - Subsample fraction of the training data to - grow tree to prevent overfitting for boosted - tree models. - min_split_loss (google.protobuf.wrappers_pb2.DoubleValue): - Minimum split loss for boosted tree models. - num_factors (int): - Num factors specified for matrix - factorization models. - feedback_type (google.cloud.bigquery_v2.types.Model.FeedbackType): - Feedback type that specifies which algorithm - to run for matrix factorization. - wals_alpha (google.protobuf.wrappers_pb2.DoubleValue): - Hyperparameter for matrix factoration when - implicit feedback type is specified. - kmeans_initialization_method (google.cloud.bigquery_v2.types.Model.KmeansEnums.KmeansInitializationMethod): - The method used to initialize the centroids - for kmeans algorithm. - kmeans_initialization_column (str): - The column used to provide the initial centroids for kmeans - algorithm when kmeans_initialization_method is CUSTOM. - time_series_timestamp_column (str): - Column to be designated as time series - timestamp for ARIMA model. - time_series_data_column (str): - Column to be designated as time series data - for ARIMA model. - auto_arima (bool): - Whether to enable auto ARIMA or not. - non_seasonal_order (google.cloud.bigquery_v2.types.Model.ArimaOrder): - A specification of the non-seasonal part of - the ARIMA model: the three components (p, d, q) - are the AR order, the degree of differencing, - and the MA order. - data_frequency (google.cloud.bigquery_v2.types.Model.DataFrequency): - The data frequency of a time series. - include_drift (bool): - Include drift when fitting an ARIMA model. - holiday_region (google.cloud.bigquery_v2.types.Model.HolidayRegion): - The geographical region based on which the - holidays are considered in time series modeling. - If a valid value is specified, then holiday - effects modeling is enabled. - time_series_id_column (str): - The time series id column that was used - during ARIMA model training. - time_series_id_columns (Sequence[str]): - The time series id columns that were used - during ARIMA model training. - horizon (int): - The number of periods ahead that need to be - forecasted. - preserve_input_structs (bool): - Whether to preserve the input structs in output feature - names. Suppose there is a struct A with field b. When false - (default), the output feature name is A_b. When true, the - output feature name is A.b. - auto_arima_max_order (int): - The max value of non-seasonal p and q. - decompose_time_series (google.protobuf.wrappers_pb2.BoolValue): - If true, perform decompose time series and - save the results. - clean_spikes_and_dips (google.protobuf.wrappers_pb2.BoolValue): - If true, clean spikes and dips in the input - time series. - adjust_step_changes (google.protobuf.wrappers_pb2.BoolValue): - If true, detect step changes and make data - adjustment in the input time series. - """ - - max_iterations = proto.Field(proto.INT64, number=1,) - loss_type = proto.Field(proto.ENUM, number=2, enum="Model.LossType",) - learn_rate = proto.Field(proto.DOUBLE, number=3,) - l1_regularization = proto.Field( - proto.MESSAGE, number=4, message=wrappers_pb2.DoubleValue, - ) - l2_regularization = proto.Field( - proto.MESSAGE, number=5, message=wrappers_pb2.DoubleValue, - ) - min_relative_progress = proto.Field( - proto.MESSAGE, number=6, message=wrappers_pb2.DoubleValue, - ) - warm_start = proto.Field( - proto.MESSAGE, number=7, message=wrappers_pb2.BoolValue, - ) - early_stop = proto.Field( - proto.MESSAGE, number=8, message=wrappers_pb2.BoolValue, - ) - input_label_columns = proto.RepeatedField(proto.STRING, number=9,) - data_split_method = proto.Field( - proto.ENUM, number=10, enum="Model.DataSplitMethod", - ) - data_split_eval_fraction = proto.Field(proto.DOUBLE, number=11,) - data_split_column = proto.Field(proto.STRING, number=12,) - learn_rate_strategy = proto.Field( - proto.ENUM, number=13, enum="Model.LearnRateStrategy", - ) - initial_learn_rate = proto.Field(proto.DOUBLE, number=16,) - label_class_weights = proto.MapField(proto.STRING, proto.DOUBLE, number=17,) - user_column = proto.Field(proto.STRING, number=18,) - item_column = proto.Field(proto.STRING, number=19,) - distance_type = proto.Field( - proto.ENUM, number=20, enum="Model.DistanceType", - ) - num_clusters = proto.Field(proto.INT64, number=21,) - model_uri = proto.Field(proto.STRING, number=22,) - optimization_strategy = proto.Field( - proto.ENUM, number=23, enum="Model.OptimizationStrategy", - ) - hidden_units = proto.RepeatedField(proto.INT64, number=24,) - batch_size = proto.Field(proto.INT64, number=25,) - dropout = proto.Field( - proto.MESSAGE, number=26, message=wrappers_pb2.DoubleValue, - ) - max_tree_depth = proto.Field(proto.INT64, number=27,) - subsample = proto.Field(proto.DOUBLE, number=28,) - min_split_loss = proto.Field( - proto.MESSAGE, number=29, message=wrappers_pb2.DoubleValue, - ) - num_factors = proto.Field(proto.INT64, number=30,) - feedback_type = proto.Field( - proto.ENUM, number=31, enum="Model.FeedbackType", - ) - wals_alpha = proto.Field( - proto.MESSAGE, number=32, message=wrappers_pb2.DoubleValue, - ) - kmeans_initialization_method = proto.Field( - proto.ENUM, - number=33, - enum="Model.KmeansEnums.KmeansInitializationMethod", - ) - kmeans_initialization_column = proto.Field(proto.STRING, number=34,) - time_series_timestamp_column = proto.Field(proto.STRING, number=35,) - time_series_data_column = proto.Field(proto.STRING, number=36,) - auto_arima = proto.Field(proto.BOOL, number=37,) - non_seasonal_order = proto.Field( - proto.MESSAGE, number=38, message="Model.ArimaOrder", - ) - data_frequency = proto.Field( - proto.ENUM, number=39, enum="Model.DataFrequency", - ) - include_drift = proto.Field(proto.BOOL, number=41,) - holiday_region = proto.Field( - proto.ENUM, number=42, enum="Model.HolidayRegion", - ) - time_series_id_column = proto.Field(proto.STRING, number=43,) - time_series_id_columns = proto.RepeatedField(proto.STRING, number=51,) - horizon = proto.Field(proto.INT64, number=44,) - preserve_input_structs = proto.Field(proto.BOOL, number=45,) - auto_arima_max_order = proto.Field(proto.INT64, number=46,) - decompose_time_series = proto.Field( - proto.MESSAGE, number=50, message=wrappers_pb2.BoolValue, - ) - clean_spikes_and_dips = proto.Field( - proto.MESSAGE, number=52, message=wrappers_pb2.BoolValue, - ) - adjust_step_changes = proto.Field( - proto.MESSAGE, number=53, message=wrappers_pb2.BoolValue, - ) - - class IterationResult(proto.Message): - r"""Information about a single iteration of the training run. - Attributes: - index (google.protobuf.wrappers_pb2.Int32Value): - Index of the iteration, 0 based. - duration_ms (google.protobuf.wrappers_pb2.Int64Value): - Time taken to run the iteration in - milliseconds. - training_loss (google.protobuf.wrappers_pb2.DoubleValue): - Loss computed on the training data at the end - of iteration. - eval_loss (google.protobuf.wrappers_pb2.DoubleValue): - Loss computed on the eval data at the end of - iteration. - learn_rate (float): - Learn rate used for this iteration. - cluster_infos (Sequence[google.cloud.bigquery_v2.types.Model.TrainingRun.IterationResult.ClusterInfo]): - Information about top clusters for clustering - models. - arima_result (google.cloud.bigquery_v2.types.Model.TrainingRun.IterationResult.ArimaResult): - - """ - - class ClusterInfo(proto.Message): - r"""Information about a single cluster for clustering model. - Attributes: - centroid_id (int): - Centroid id. - cluster_radius (google.protobuf.wrappers_pb2.DoubleValue): - Cluster radius, the average distance from - centroid to each point assigned to the cluster. - cluster_size (google.protobuf.wrappers_pb2.Int64Value): - Cluster size, the total number of points - assigned to the cluster. - """ - - centroid_id = proto.Field(proto.INT64, number=1,) - cluster_radius = proto.Field( - proto.MESSAGE, number=2, message=wrappers_pb2.DoubleValue, - ) - cluster_size = proto.Field( - proto.MESSAGE, number=3, message=wrappers_pb2.Int64Value, - ) - - class ArimaResult(proto.Message): - r"""(Auto-)arima fitting result. Wrap everything in ArimaResult - for easier refactoring if we want to use model-specific - iteration results. - - Attributes: - arima_model_info (Sequence[google.cloud.bigquery_v2.types.Model.TrainingRun.IterationResult.ArimaResult.ArimaModelInfo]): - This message is repeated because there are - multiple arima models fitted in auto-arima. For - non-auto-arima model, its size is one. - seasonal_periods (Sequence[google.cloud.bigquery_v2.types.Model.SeasonalPeriod.SeasonalPeriodType]): - Seasonal periods. Repeated because multiple - periods are supported for one time series. - """ - - class ArimaCoefficients(proto.Message): - r"""Arima coefficients. - Attributes: - auto_regressive_coefficients (Sequence[float]): - Auto-regressive coefficients, an array of - double. - moving_average_coefficients (Sequence[float]): - Moving-average coefficients, an array of - double. - intercept_coefficient (float): - Intercept coefficient, just a double not an - array. - """ - - auto_regressive_coefficients = proto.RepeatedField( - proto.DOUBLE, number=1, - ) - moving_average_coefficients = proto.RepeatedField( - proto.DOUBLE, number=2, - ) - intercept_coefficient = proto.Field(proto.DOUBLE, number=3,) - - class ArimaModelInfo(proto.Message): - r"""Arima model information. - Attributes: - non_seasonal_order (google.cloud.bigquery_v2.types.Model.ArimaOrder): - Non-seasonal order. - arima_coefficients (google.cloud.bigquery_v2.types.Model.TrainingRun.IterationResult.ArimaResult.ArimaCoefficients): - Arima coefficients. - arima_fitting_metrics (google.cloud.bigquery_v2.types.Model.ArimaFittingMetrics): - Arima fitting metrics. - has_drift (bool): - Whether Arima model fitted with drift or not. - It is always false when d is not 1. - time_series_id (str): - The time_series_id value for this time series. It will be - one of the unique values from the time_series_id_column - specified during ARIMA model training. Only present when - time_series_id_column training option was used. - time_series_ids (Sequence[str]): - The tuple of time_series_ids identifying this time series. - It will be one of the unique tuples of values present in the - time_series_id_columns specified during ARIMA model - training. Only present when time_series_id_columns training - option was used and the order of values here are same as the - order of time_series_id_columns. - seasonal_periods (Sequence[google.cloud.bigquery_v2.types.Model.SeasonalPeriod.SeasonalPeriodType]): - Seasonal periods. Repeated because multiple - periods are supported for one time series. - has_holiday_effect (google.protobuf.wrappers_pb2.BoolValue): - If true, holiday_effect is a part of time series - decomposition result. - has_spikes_and_dips (google.protobuf.wrappers_pb2.BoolValue): - If true, spikes_and_dips is a part of time series - decomposition result. - has_step_changes (google.protobuf.wrappers_pb2.BoolValue): - If true, step_changes is a part of time series decomposition - result. - """ - - non_seasonal_order = proto.Field( - proto.MESSAGE, number=1, message="Model.ArimaOrder", - ) - arima_coefficients = proto.Field( - proto.MESSAGE, - number=2, - message="Model.TrainingRun.IterationResult.ArimaResult.ArimaCoefficients", - ) - arima_fitting_metrics = proto.Field( - proto.MESSAGE, number=3, message="Model.ArimaFittingMetrics", - ) - has_drift = proto.Field(proto.BOOL, number=4,) - time_series_id = proto.Field(proto.STRING, number=5,) - time_series_ids = proto.RepeatedField(proto.STRING, number=10,) - seasonal_periods = proto.RepeatedField( - proto.ENUM, - number=6, - enum="Model.SeasonalPeriod.SeasonalPeriodType", - ) - has_holiday_effect = proto.Field( - proto.MESSAGE, number=7, message=wrappers_pb2.BoolValue, - ) - has_spikes_and_dips = proto.Field( - proto.MESSAGE, number=8, message=wrappers_pb2.BoolValue, - ) - has_step_changes = proto.Field( - proto.MESSAGE, number=9, message=wrappers_pb2.BoolValue, - ) - - arima_model_info = proto.RepeatedField( - proto.MESSAGE, - number=1, - message="Model.TrainingRun.IterationResult.ArimaResult.ArimaModelInfo", - ) - seasonal_periods = proto.RepeatedField( - proto.ENUM, - number=2, - enum="Model.SeasonalPeriod.SeasonalPeriodType", - ) - - index = proto.Field( - proto.MESSAGE, number=1, message=wrappers_pb2.Int32Value, - ) - duration_ms = proto.Field( - proto.MESSAGE, number=4, message=wrappers_pb2.Int64Value, - ) - training_loss = proto.Field( - proto.MESSAGE, number=5, message=wrappers_pb2.DoubleValue, - ) - eval_loss = proto.Field( - proto.MESSAGE, number=6, message=wrappers_pb2.DoubleValue, - ) - learn_rate = proto.Field(proto.DOUBLE, number=7,) - cluster_infos = proto.RepeatedField( - proto.MESSAGE, - number=8, - message="Model.TrainingRun.IterationResult.ClusterInfo", - ) - arima_result = proto.Field( - proto.MESSAGE, - number=9, - message="Model.TrainingRun.IterationResult.ArimaResult", - ) - - training_options = proto.Field( - proto.MESSAGE, number=1, message="Model.TrainingRun.TrainingOptions", - ) - start_time = proto.Field( - proto.MESSAGE, number=8, message=timestamp_pb2.Timestamp, - ) - results = proto.RepeatedField( - proto.MESSAGE, number=6, message="Model.TrainingRun.IterationResult", - ) - evaluation_metrics = proto.Field( - proto.MESSAGE, number=7, message="Model.EvaluationMetrics", - ) - data_split_result = proto.Field( - proto.MESSAGE, number=9, message="Model.DataSplitResult", - ) - global_explanations = proto.RepeatedField( - proto.MESSAGE, number=10, message="Model.GlobalExplanation", - ) - - etag = proto.Field(proto.STRING, number=1,) - model_reference = proto.Field( - proto.MESSAGE, number=2, message=gcb_model_reference.ModelReference, - ) - creation_time = proto.Field(proto.INT64, number=5,) - last_modified_time = proto.Field(proto.INT64, number=6,) - description = proto.Field(proto.STRING, number=12,) - friendly_name = proto.Field(proto.STRING, number=14,) - labels = proto.MapField(proto.STRING, proto.STRING, number=15,) - expiration_time = proto.Field(proto.INT64, number=16,) - location = proto.Field(proto.STRING, number=13,) - encryption_configuration = proto.Field( - proto.MESSAGE, number=17, message=encryption_config.EncryptionConfiguration, - ) - model_type = proto.Field(proto.ENUM, number=7, enum=ModelType,) - training_runs = proto.RepeatedField(proto.MESSAGE, number=9, message=TrainingRun,) - feature_columns = proto.RepeatedField( - proto.MESSAGE, number=10, message=standard_sql.StandardSqlField, - ) - label_columns = proto.RepeatedField( - proto.MESSAGE, number=11, message=standard_sql.StandardSqlField, - ) - best_trial_id = proto.Field(proto.INT64, number=19,) - - -class GetModelRequest(proto.Message): - r""" - Attributes: - project_id (str): - Required. Project ID of the requested model. - dataset_id (str): - Required. Dataset ID of the requested model. - model_id (str): - Required. Model ID of the requested model. - """ - - project_id = proto.Field(proto.STRING, number=1,) - dataset_id = proto.Field(proto.STRING, number=2,) - model_id = proto.Field(proto.STRING, number=3,) - - -class PatchModelRequest(proto.Message): - r""" - Attributes: - project_id (str): - Required. Project ID of the model to patch. - dataset_id (str): - Required. Dataset ID of the model to patch. - model_id (str): - Required. Model ID of the model to patch. - model (google.cloud.bigquery_v2.types.Model): - Required. Patched model. - Follows RFC5789 patch semantics. Missing fields - are not updated. To clear a field, explicitly - set to default value. - """ - - project_id = proto.Field(proto.STRING, number=1,) - dataset_id = proto.Field(proto.STRING, number=2,) - model_id = proto.Field(proto.STRING, number=3,) - model = proto.Field(proto.MESSAGE, number=4, message="Model",) - - -class DeleteModelRequest(proto.Message): - r""" - Attributes: - project_id (str): - Required. Project ID of the model to delete. - dataset_id (str): - Required. Dataset ID of the model to delete. - model_id (str): - Required. Model ID of the model to delete. - """ - - project_id = proto.Field(proto.STRING, number=1,) - dataset_id = proto.Field(proto.STRING, number=2,) - model_id = proto.Field(proto.STRING, number=3,) - - -class ListModelsRequest(proto.Message): - r""" - Attributes: - project_id (str): - Required. Project ID of the models to list. - dataset_id (str): - Required. Dataset ID of the models to list. - max_results (google.protobuf.wrappers_pb2.UInt32Value): - The maximum number of results to return in a - single response page. Leverage the page tokens - to iterate through the entire collection. - page_token (str): - Page token, returned by a previous call to - request the next page of results - """ - - project_id = proto.Field(proto.STRING, number=1,) - dataset_id = proto.Field(proto.STRING, number=2,) - max_results = proto.Field( - proto.MESSAGE, number=3, message=wrappers_pb2.UInt32Value, - ) - page_token = proto.Field(proto.STRING, number=4,) - - -class ListModelsResponse(proto.Message): - r""" - Attributes: - models (Sequence[google.cloud.bigquery_v2.types.Model]): - Models in the requested dataset. Only the following fields - are populated: model_reference, model_type, creation_time, - last_modified_time and labels. - next_page_token (str): - A token to request the next page of results. - """ - - @property - def raw_page(self): - return self - - models = proto.RepeatedField(proto.MESSAGE, number=1, message="Model",) - next_page_token = proto.Field(proto.STRING, number=2,) - - -__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/bigquery_v2/types/model_reference.py b/google/cloud/bigquery_v2/types/model_reference.py deleted file mode 100644 index a9ebad613..000000000 --- a/google/cloud/bigquery_v2/types/model_reference.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import proto # type: ignore - - -__protobuf__ = proto.module( - package="google.cloud.bigquery.v2", manifest={"ModelReference",}, -) - - -class ModelReference(proto.Message): - r"""Id path of a model. - Attributes: - project_id (str): - Required. The ID of the project containing - this model. - dataset_id (str): - Required. The ID of the dataset containing - this model. - model_id (str): - Required. The ID of the model. The ID must contain only - letters (a-z, A-Z), numbers (0-9), or underscores (_). The - maximum length is 1,024 characters. - """ - - project_id = proto.Field(proto.STRING, number=1,) - dataset_id = proto.Field(proto.STRING, number=2,) - model_id = proto.Field(proto.STRING, number=3,) - - -__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/bigquery_v2/types/standard_sql.py b/google/cloud/bigquery_v2/types/standard_sql.py deleted file mode 100644 index 7a845fc48..000000000 --- a/google/cloud/bigquery_v2/types/standard_sql.py +++ /dev/null @@ -1,117 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import proto # type: ignore - - -__protobuf__ = proto.module( - package="google.cloud.bigquery.v2", - manifest={ - "StandardSqlDataType", - "StandardSqlField", - "StandardSqlStructType", - "StandardSqlTableType", - }, -) - - -class StandardSqlDataType(proto.Message): - r"""The type of a variable, e.g., a function argument. Examples: INT64: - {type_kind="INT64"} ARRAY: {type_kind="ARRAY", - array_element_type="STRING"} STRUCT: - {type_kind="STRUCT", struct_type={fields=[ {name="x", - type={type_kind="STRING"}}, {name="y", type={type_kind="ARRAY", - array_element_type="DATE"}} ]}} - - Attributes: - type_kind (google.cloud.bigquery_v2.types.StandardSqlDataType.TypeKind): - Required. The top level type of this field. - Can be any standard SQL data type (e.g., - "INT64", "DATE", "ARRAY"). - array_element_type (google.cloud.bigquery_v2.types.StandardSqlDataType): - The type of the array's elements, if type_kind = "ARRAY". - struct_type (google.cloud.bigquery_v2.types.StandardSqlStructType): - The fields of this struct, in order, if type_kind = - "STRUCT". - """ - - class TypeKind(proto.Enum): - r"""""" - TYPE_KIND_UNSPECIFIED = 0 - INT64 = 2 - BOOL = 5 - FLOAT64 = 7 - STRING = 8 - BYTES = 9 - TIMESTAMP = 19 - DATE = 10 - TIME = 20 - DATETIME = 21 - INTERVAL = 26 - GEOGRAPHY = 22 - NUMERIC = 23 - BIGNUMERIC = 24 - JSON = 25 - ARRAY = 16 - STRUCT = 17 - - type_kind = proto.Field(proto.ENUM, number=1, enum=TypeKind,) - array_element_type = proto.Field( - proto.MESSAGE, number=2, oneof="sub_type", message="StandardSqlDataType", - ) - struct_type = proto.Field( - proto.MESSAGE, number=3, oneof="sub_type", message="StandardSqlStructType", - ) - - -class StandardSqlField(proto.Message): - r"""A field or a column. - Attributes: - name (str): - Optional. The name of this field. Can be - absent for struct fields. - type (google.cloud.bigquery_v2.types.StandardSqlDataType): - Optional. The type of this parameter. Absent - if not explicitly specified (e.g., CREATE - FUNCTION statement can omit the return type; in - this case the output parameter does not have - this "type" field). - """ - - name = proto.Field(proto.STRING, number=1,) - type = proto.Field(proto.MESSAGE, number=2, message="StandardSqlDataType",) - - -class StandardSqlStructType(proto.Message): - r""" - Attributes: - fields (Sequence[google.cloud.bigquery_v2.types.StandardSqlField]): - - """ - - fields = proto.RepeatedField(proto.MESSAGE, number=1, message="StandardSqlField",) - - -class StandardSqlTableType(proto.Message): - r"""A table type - Attributes: - columns (Sequence[google.cloud.bigquery_v2.types.StandardSqlField]): - The columns in this table type - """ - - columns = proto.RepeatedField(proto.MESSAGE, number=1, message="StandardSqlField",) - - -__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/bigquery_v2/types/table_reference.py b/google/cloud/bigquery_v2/types/table_reference.py deleted file mode 100644 index d56e5b09f..000000000 --- a/google/cloud/bigquery_v2/types/table_reference.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import proto # type: ignore - - -__protobuf__ = proto.module( - package="google.cloud.bigquery.v2", manifest={"TableReference",}, -) - - -class TableReference(proto.Message): - r""" - Attributes: - project_id (str): - Required. The ID of the project containing - this table. - dataset_id (str): - Required. The ID of the dataset containing - this table. - table_id (str): - Required. The ID of the table. The ID must contain only - letters (a-z, A-Z), numbers (0-9), or underscores (_). The - maximum length is 1,024 characters. Certain operations allow - suffixing of the table ID with a partition decorator, such - as ``sample_table$20190123``. - project_id_alternative (Sequence[str]): - The alternative field that will be used when ESF is not able - to translate the received data to the project_id field. - dataset_id_alternative (Sequence[str]): - The alternative field that will be used when ESF is not able - to translate the received data to the project_id field. - table_id_alternative (Sequence[str]): - The alternative field that will be used when ESF is not able - to translate the received data to the project_id field. - """ - - project_id = proto.Field(proto.STRING, number=1,) - dataset_id = proto.Field(proto.STRING, number=2,) - table_id = proto.Field(proto.STRING, number=3,) - project_id_alternative = proto.RepeatedField(proto.STRING, number=4,) - dataset_id_alternative = proto.RepeatedField(proto.STRING, number=5,) - table_id_alternative = proto.RepeatedField(proto.STRING, number=6,) - - -__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owlbot.py b/owlbot.py index 09aa8ca6f..be493957e 100644 --- a/owlbot.py +++ b/owlbot.py @@ -21,74 +21,6 @@ common = gcp.CommonTemplates() -default_version = "v2" - -for library in s.get_staging_dirs(default_version): - # Do not expose ModelServiceClient and ModelServiceAsyncClient, as there - # is no public API endpoint for the models service. - s.replace( - library / f"google/cloud/bigquery_{library.name}/__init__.py", - r"from \.services\.model_service import ModelServiceClient", - "", - ) - - s.replace( - library / f"google/cloud/bigquery_{library.name}/__init__.py", - r"from \.services\.model_service import ModelServiceAsyncClient", - "", - ) - - s.replace( - library / f"google/cloud/bigquery_{library.name}/__init__.py", - r"""["']ModelServiceClient["'],""", - "", - ) - - s.replace( - library / f"google/cloud/bigquery_{library.name}/__init__.py", - r"""["']ModelServiceAsyncClient["'],""", - "", - ) - - # Adjust Model docstring so that Sphinx does not think that "predicted_" is - # a reference to something, issuing a false warning. - s.replace( - library / f"google/cloud/bigquery_{library.name}/types/model.py", - r'will have a "predicted_"', - "will have a `predicted_`", - ) - - # Avoid breaking change due to change in field renames. - # https://github.com/googleapis/python-bigquery/issues/319 - s.replace( - library / f"google/cloud/bigquery_{library.name}/types/standard_sql.py", - r"type_ ", - "type ", - ) - - s.move( - library, - excludes=[ - "*.tar.gz", - ".coveragerc", - "docs/index.rst", - f"docs/bigquery_{library.name}/*_service.rst", - f"docs/bigquery_{library.name}/services.rst", - "README.rst", - "noxfile.py", - "setup.py", - f"scripts/fixup_bigquery_{library.name}_keywords.py", - "google/cloud/bigquery/__init__.py", - "google/cloud/bigquery/py.typed", - # There are no public API endpoints for the generated ModelServiceClient, - # thus there's no point in generating it and its tests. - f"google/cloud/bigquery_{library.name}/services/**", - f"tests/unit/gapic/bigquery_{library.name}/**", - ], - ) - -s.remove_staging_dirs() - # ---------------------------------------------------------------------------- # Add templated files # ---------------------------------------------------------------------------- @@ -120,6 +52,10 @@ ], ) +# Remove unneeded intersphinx links, the library does not use any proto-generated code. +s.replace("docs/conf.py", r'\s+"(proto-plus|protobuf)":.*$', "") + + # ---------------------------------------------------------------------------- # Samples templates # ---------------------------------------------------------------------------- @@ -132,13 +68,6 @@ '{"members": True, "inherited-members": True}', ) -# Tell Sphinx to ingore autogenerated docs files. -s.replace( - "docs/conf.py", - r'"samples/snippets/README\.rst",', - '\\g<0>\n "bigquery_v2/services.rst", # generated by the code generator', -) - # ---------------------------------------------------------------------------- # pytype-related changes # ---------------------------------------------------------------------------- @@ -159,7 +88,6 @@ google/cloud/ exclude = tests/ - google/cloud/bigquery_v2/ output = .pytype/ disable = # There's some issue with finding some pyi files, thus disabling. @@ -169,52 +97,4 @@ ), ) - -# Remove the replacements below once -# https://github.com/googleapis/synthtool/pull/1188 is merged - -# Update googleapis/repo-automation-bots repo to main in .kokoro/*.sh files -s.replace( - ".kokoro/*.sh", - "repo-automation-bots/tree/master", - "repo-automation-bots/tree/main", -) - -# Customize CONTRIBUTING.rst to replace master with main -s.replace( - "CONTRIBUTING.rst", - "fetch and merge changes from upstream into master", - "fetch and merge changes from upstream into main", -) - -s.replace( - "CONTRIBUTING.rst", "git merge upstream/master", "git merge upstream/main", -) - -s.replace( - "CONTRIBUTING.rst", - """export GOOGLE_CLOUD_TESTING_BRANCH=\"master\"""", - """export GOOGLE_CLOUD_TESTING_BRANCH=\"main\"""", -) - -s.replace( - "CONTRIBUTING.rst", r"remote \(``master``\)", "remote (``main``)", -) - -s.replace( - "CONTRIBUTING.rst", "blob/master/CONTRIBUTING.rst", "blob/main/CONTRIBUTING.rst", -) - -s.replace( - "CONTRIBUTING.rst", "blob/master/noxfile.py", "blob/main/noxfile.py", -) - -s.replace( - "docs/conf.py", "master_doc", "root_doc", -) - -s.replace( - "docs/conf.py", "# The master toctree document.", "# The root toctree document.", -) - s.shell.run(["nox", "-s", "blacken"], hide_output=False) diff --git a/samples/create_routine.py b/samples/create_routine.py index 1cb4a80b4..b8746905d 100644 --- a/samples/create_routine.py +++ b/samples/create_routine.py @@ -17,7 +17,6 @@ def create_routine(routine_id): # [START bigquery_create_routine] from google.cloud import bigquery - from google.cloud import bigquery_v2 # Construct a BigQuery client object. client = bigquery.Client() @@ -33,8 +32,8 @@ def create_routine(routine_id): arguments=[ bigquery.RoutineArgument( name="x", - data_type=bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + data_type=bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.INT64 ), ) ], diff --git a/samples/tests/conftest.py b/samples/tests/conftest.py index 0fdacaaec..4764a571f 100644 --- a/samples/tests/conftest.py +++ b/samples/tests/conftest.py @@ -20,7 +20,6 @@ import pytest from google.cloud import bigquery -from google.cloud import bigquery_v2 @pytest.fixture(scope="session", autouse=True) @@ -125,8 +124,8 @@ def routine_id(client, dataset_id): routine.arguments = [ bigquery.RoutineArgument( name="x", - data_type=bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + data_type=bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.INT64 ), ) ] diff --git a/samples/tests/test_routine_samples.py b/samples/tests/test_routine_samples.py index c1b0bb5a7..b457c464a 100644 --- a/samples/tests/test_routine_samples.py +++ b/samples/tests/test_routine_samples.py @@ -13,7 +13,6 @@ # limitations under the License. from google.cloud import bigquery -from google.cloud import bigquery_v2 def test_create_routine(capsys, random_routine_id): @@ -37,22 +36,22 @@ def test_create_routine_ddl(capsys, random_routine_id, client): expected_arguments = [ bigquery.RoutineArgument( name="arr", - data_type=bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.ARRAY, - array_element_type=bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.STRUCT, - struct_type=bigquery_v2.types.StandardSqlStructType( + data_type=bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.ARRAY, + array_element_type=bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.STRUCT, + struct_type=bigquery.StandardSqlStructType( fields=[ - bigquery_v2.types.StandardSqlField( + bigquery.StandardSqlField( name="name", - type=bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.STRING + type=bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.STRING ), ), - bigquery_v2.types.StandardSqlField( + bigquery.StandardSqlField( name="val", - type=bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + type=bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.INT64 ), ), ] @@ -82,7 +81,7 @@ def test_get_routine(capsys, routine_id): assert "Type: 'SCALAR_FUNCTION'" in out assert "Language: 'SQL'" in out assert "Name: 'x'" in out - assert "Type: 'type_kind: INT64\n'" in out + assert "type_kind=" in out def test_delete_routine(capsys, routine_id): diff --git a/setup.cfg b/setup.cfg index 8eefc4435..28b7b0f26 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,7 +24,6 @@ inputs = google/cloud/ exclude = tests/ - google/cloud/bigquery_v2/ output = .pytype/ disable = # There's some issue with finding some pyi files, thus disabling. diff --git a/setup.py b/setup.py index f1464e77a..130d8f49c 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,6 @@ # Until this issue is closed # https://github.com/googleapis/google-cloud-python/issues/10566 "google-api-core[grpc] >= 1.29.0, <3.0.0dev", - "proto-plus >= 1.10.0", "google-cloud-bigquery-storage >= 2.0.0, <3.0.0dev", # NOTE: Maintainers, please do not require google-cloud-core>=2.x.x # Until this issue is closed @@ -42,7 +41,6 @@ "google-cloud-core >= 1.4.1, <3.0.0dev", "google-resumable-media >= 0.6.0, < 3.0dev", "packaging >= 14.3", - "protobuf >= 3.12.0", "pyarrow >= 3.0.0, < 6.0dev", "requests >= 2.18.0, < 3.0.0dev", ] diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index 6e27172b2..620f23ca2 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -15,8 +15,6 @@ opentelemetry-api==0.11b0 opentelemetry-instrumentation==0.11b0 opentelemetry-sdk==0.11b0 pandas==1.0.0 -proto-plus==1.10.0 -protobuf==3.12.0 pyarrow==3.0.0 requests==2.18.0 Shapely==1.6.0 diff --git a/tests/system/test_client.py b/tests/system/test_client.py index 4b9868f10..120e10f45 100644 --- a/tests/system/test_client.py +++ b/tests/system/test_client.py @@ -51,7 +51,6 @@ from google.api_core.exceptions import TooManyRequests from google.api_core.iam import Policy from google.cloud import bigquery -from google.cloud import bigquery_v2 from google.cloud.bigquery.dataset import Dataset from google.cloud.bigquery.dataset import DatasetReference from google.cloud.bigquery.table import Table @@ -2181,8 +2180,8 @@ def test_insert_rows_nested_nested_dictionary(self): def test_create_routine(self): routine_name = "test_routine" dataset = self.temp_dataset(_make_dataset_id("create_routine")) - float64_type = bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.FLOAT64 + float64_type = bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.FLOAT64 ) routine = bigquery.Routine( dataset.routine(routine_name), @@ -2196,8 +2195,8 @@ def test_create_routine(self): routine.arguments = [ bigquery.RoutineArgument( name="arr", - data_type=bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.ARRAY, + data_type=bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.ARRAY, array_element_type=float64_type, ), ) @@ -2216,14 +2215,19 @@ def test_create_routine(self): assert rows[0].max_value == 100.0 def test_create_tvf_routine(self): - from google.cloud.bigquery import Routine, RoutineArgument, RoutineType + from google.cloud.bigquery import ( + Routine, + RoutineArgument, + RoutineType, + StandardSqlTypeNames, + ) - StandardSqlDataType = bigquery_v2.types.StandardSqlDataType - StandardSqlField = bigquery_v2.types.StandardSqlField - StandardSqlTableType = bigquery_v2.types.StandardSqlTableType + StandardSqlDataType = bigquery.StandardSqlDataType + StandardSqlField = bigquery.StandardSqlField + StandardSqlTableType = bigquery.StandardSqlTableType - INT64 = StandardSqlDataType.TypeKind.INT64 - STRING = StandardSqlDataType.TypeKind.STRING + INT64 = StandardSqlTypeNames.INT64 + STRING = StandardSqlTypeNames.STRING client = Config.CLIENT diff --git a/tests/unit/enums/__init__.py b/tests/unit/enums/__init__.py deleted file mode 100644 index c5cce0430..000000000 --- a/tests/unit/enums/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2019, Google LLC All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/unit/enums/test_standard_sql_data_types.py b/tests/unit/enums/test_standard_sql_data_types.py deleted file mode 100644 index 7f62c46fd..000000000 --- a/tests/unit/enums/test_standard_sql_data_types.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest - - -@pytest.fixture -def module_under_test(): - from google.cloud.bigquery import enums - - return enums - - -@pytest.fixture -def enum_under_test(): - from google.cloud.bigquery.enums import StandardSqlDataTypes - - return StandardSqlDataTypes - - -@pytest.fixture -def gapic_enum(): - """The referential autogenerated enum the enum under test is based on.""" - from google.cloud.bigquery_v2.types import StandardSqlDataType - - return StandardSqlDataType.TypeKind - - -def test_all_gapic_enum_members_are_known(module_under_test, gapic_enum): - gapic_names = set(type_.name for type_ in gapic_enum) - anticipated_names = ( - module_under_test._SQL_SCALAR_TYPES | module_under_test._SQL_NONSCALAR_TYPES - ) - assert not (gapic_names - anticipated_names) # no unhandled names - - -def test_standard_sql_types_enum_members(enum_under_test, gapic_enum): - # check the presence of a few typical SQL types - for name in ("INT64", "FLOAT64", "DATE", "BOOL", "GEOGRAPHY"): - assert name in enum_under_test.__members__ - - # the enum members must match those in the original gapic enum - for member in enum_under_test: - assert member.name in gapic_enum.__members__ - assert member.value == gapic_enum[member.name].value - - # check a few members that should *not* be copied over from the gapic enum - for name in ("STRUCT", "ARRAY"): - assert name in gapic_enum.__members__ - assert name not in enum_under_test.__members__ - - -@pytest.mark.skip(reason="Code generator issue, the docstring is not generated.") -def test_standard_sql_types_enum_docstring( - enum_under_test, gapic_enum -): # pragma: NO COVER - assert "STRUCT (int):" not in enum_under_test.__doc__ - assert "BOOL (int):" in enum_under_test.__doc__ - assert "TIME (int):" in enum_under_test.__doc__ - - # All lines in the docstring should actually come from the original docstring, - # except for the header. - assert "An Enum of scalar SQL types." in enum_under_test.__doc__ - doc_lines = enum_under_test.__doc__.splitlines() - assert set(doc_lines[1:]) <= set(gapic_enum.__doc__.splitlines()) diff --git a/tests/unit/gapic/__init__.py b/tests/unit/gapic/__init__.py deleted file mode 100644 index 4de65971c..000000000 --- a/tests/unit/gapic/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/tests/unit/model/test_model.py b/tests/unit/model/test_model.py index 8f0bf58d5..c5f9b77c1 100644 --- a/tests/unit/model/test_model.py +++ b/tests/unit/model/test_model.py @@ -19,7 +19,6 @@ import pytest import google.cloud._helpers -from google.cloud.bigquery_v2 import types KMS_KEY_NAME = "projects/1/locations/us/keyRings/1/cryptoKeys/1" @@ -79,7 +78,7 @@ def test_from_api_repr(target_class): "description": "A friendly description.", "friendlyName": "A friendly name.", "modelType": "LOGISTIC_REGRESSION", - "labels": {"greeting": u"こんにちは"}, + "labels": {"greeting": "こんにちは"}, "trainingRuns": [ { "trainingOptions": {"initialLearnRate": 1.0}, @@ -95,9 +94,9 @@ def test_from_api_repr(target_class): }, { "trainingOptions": {"initialLearnRate": 0.25}, - # Allow milliseconds since epoch format. - # TODO: Remove this hack once CL 238585470 hits prod. - "startTime": str(google.cloud._helpers._millis(expiration_time)), + "startTime": str( + google.cloud._helpers._datetime_to_rfc3339(expiration_time) + ), }, ], "featureColumns": [], @@ -115,30 +114,24 @@ def test_from_api_repr(target_class): assert got.created == creation_time assert got.modified == modified_time assert got.expires == expiration_time - assert got.description == u"A friendly description." - assert got.friendly_name == u"A friendly name." - assert got.model_type == types.Model.ModelType.LOGISTIC_REGRESSION - assert got.labels == {"greeting": u"こんにちは"} + assert got.description == "A friendly description." + assert got.friendly_name == "A friendly name." + assert got.model_type == "LOGISTIC_REGRESSION" + assert got.labels == {"greeting": "こんにちは"} assert got.encryption_configuration.kms_key_name == KMS_KEY_NAME - assert got.training_runs[0].training_options.initial_learn_rate == 1.0 + assert got.training_runs[0]["trainingOptions"]["initialLearnRate"] == 1.0 assert ( - got.training_runs[0] - .start_time.ToDatetime() - .replace(tzinfo=google.cloud._helpers.UTC) + google.cloud._helpers._rfc3339_to_datetime(got.training_runs[0]["startTime"]) == creation_time ) - assert got.training_runs[1].training_options.initial_learn_rate == 0.5 + assert got.training_runs[1]["trainingOptions"]["initialLearnRate"] == 0.5 assert ( - got.training_runs[1] - .start_time.ToDatetime() - .replace(tzinfo=google.cloud._helpers.UTC) + google.cloud._helpers._rfc3339_to_datetime(got.training_runs[1]["startTime"]) == modified_time ) - assert got.training_runs[2].training_options.initial_learn_rate == 0.25 + assert got.training_runs[2]["trainingOptions"]["initialLearnRate"] == 0.25 assert ( - got.training_runs[2] - .start_time.ToDatetime() - .replace(tzinfo=google.cloud._helpers.UTC) + google.cloud._helpers._rfc3339_to_datetime(got.training_runs[2]["startTime"]) == expiration_time ) @@ -155,14 +148,14 @@ def test_from_api_repr_w_minimal_resource(target_class): } got = target_class.from_api_repr(resource) assert got.reference == ModelReference.from_string("my-project.my_dataset.my_model") - assert got.location == "" - assert got.etag == "" + assert got.location is None + assert got.etag is None assert got.created is None assert got.modified is None assert got.expires is None assert got.description is None assert got.friendly_name is None - assert got.model_type == types.Model.ModelType.MODEL_TYPE_UNSPECIFIED + assert got.model_type == "MODEL_TYPE_UNSPECIFIED" assert got.labels == {} assert got.encryption_configuration is None assert len(got.training_runs) == 0 @@ -183,7 +176,7 @@ def test_from_api_repr_w_unknown_fields(target_class): } got = target_class.from_api_repr(resource) assert got.reference == ModelReference.from_string("my-project.my_dataset.my_model") - assert got._properties is resource + assert got._properties == resource def test_from_api_repr_w_unknown_type(target_class): @@ -195,12 +188,19 @@ def test_from_api_repr_w_unknown_type(target_class): "datasetId": "my_dataset", "modelId": "my_model", }, - "modelType": "BE_A_GOOD_ROLE_MODEL", + "modelType": "BE_A_GOOD_ROLE_MODEL", # This model type does not exist. } got = target_class.from_api_repr(resource) assert got.reference == ModelReference.from_string("my-project.my_dataset.my_model") - assert got.model_type == 0 - assert got._properties is resource + assert got.model_type == "BE_A_GOOD_ROLE_MODEL" # No checks for invalid types. + assert got._properties == resource + + +def test_from_api_repr_w_missing_reference(target_class): + resource = {} + got = target_class.from_api_repr(resource) + assert got.reference is None + assert got._properties == resource @pytest.mark.parametrize( @@ -338,8 +338,6 @@ def test_repr(target_class): def test_to_api_repr(target_class): - from google.protobuf import json_format - model = target_class("my-proj.my_dset.my_model") resource = { "etag": "abcdefg", @@ -374,8 +372,6 @@ def test_to_api_repr(target_class): "kmsKeyName": "projects/1/locations/us/keyRings/1/cryptoKeys/1" }, } - model._proto = json_format.ParseDict( - resource, types.Model()._pb, ignore_unknown_fields=True - ) + model._properties = resource got = model.to_api_repr() assert got == resource diff --git a/tests/unit/routine/test_routine.py b/tests/unit/routine/test_routine.py index fdaf13324..80a3def73 100644 --- a/tests/unit/routine/test_routine.py +++ b/tests/unit/routine/test_routine.py @@ -19,7 +19,6 @@ import google.cloud._helpers from google.cloud import bigquery -from google.cloud import bigquery_v2 @pytest.fixture @@ -62,15 +61,15 @@ def test_ctor_w_properties(target_class): arguments = [ RoutineArgument( name="x", - data_type=bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + data_type=bigquery.standard_sql.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.INT64 ), ) ] body = "x * 3" language = "SQL" - return_type = bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + return_type = bigquery.standard_sql.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.INT64 ) type_ = "SCALAR_FUNCTION" description = "A routine description." @@ -146,15 +145,15 @@ def test_from_api_repr(target_class): assert actual_routine.arguments == [ RoutineArgument( name="x", - data_type=bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + data_type=bigquery.standard_sql.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.INT64 ), ) ] assert actual_routine.body == "42" assert actual_routine.language == "SQL" - assert actual_routine.return_type == bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + assert actual_routine.return_type == bigquery.standard_sql.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.INT64 ) assert actual_routine.return_table_type is None assert actual_routine.type_ == "SCALAR_FUNCTION" @@ -168,9 +167,9 @@ def test_from_api_repr_tvf_function(target_class): from google.cloud.bigquery.routine import RoutineReference from google.cloud.bigquery.routine import RoutineType - StandardSqlDataType = bigquery_v2.types.StandardSqlDataType - StandardSqlField = bigquery_v2.types.StandardSqlField - StandardSqlTableType = bigquery_v2.types.StandardSqlTableType + StandardSqlDataType = bigquery.standard_sql.StandardSqlDataType + StandardSqlField = bigquery.standard_sql.StandardSqlField + StandardSqlTableType = bigquery.standard_sql.StandardSqlTableType creation_time = datetime.datetime( 2010, 5, 19, 16, 0, 0, tzinfo=google.cloud._helpers.UTC @@ -216,7 +215,9 @@ def test_from_api_repr_tvf_function(target_class): assert actual_routine.arguments == [ RoutineArgument( name="a", - data_type=StandardSqlDataType(type_kind=StandardSqlDataType.TypeKind.INT64), + data_type=StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.INT64 + ), ) ] assert actual_routine.body == "SELECT x FROM UNNEST([1,2,3]) x WHERE x > a" @@ -226,7 +227,7 @@ def test_from_api_repr_tvf_function(target_class): columns=[ StandardSqlField( name="int_col", - type=StandardSqlDataType(type_kind=StandardSqlDataType.TypeKind.INT64), + type=StandardSqlDataType(type_kind=bigquery.StandardSqlTypeNames.INT64), ) ] ) @@ -460,19 +461,21 @@ def test_set_return_table_type_w_none(object_under_test): def test_set_return_table_type_w_not_none(object_under_test): - StandardSqlDataType = bigquery_v2.types.StandardSqlDataType - StandardSqlField = bigquery_v2.types.StandardSqlField - StandardSqlTableType = bigquery_v2.types.StandardSqlTableType + StandardSqlDataType = bigquery.standard_sql.StandardSqlDataType + StandardSqlField = bigquery.standard_sql.StandardSqlField + StandardSqlTableType = bigquery.standard_sql.StandardSqlTableType table_type = StandardSqlTableType( columns=[ StandardSqlField( name="int_col", - type=StandardSqlDataType(type_kind=StandardSqlDataType.TypeKind.INT64), + type=StandardSqlDataType(type_kind=bigquery.StandardSqlTypeNames.INT64), ), StandardSqlField( name="str_col", - type=StandardSqlDataType(type_kind=StandardSqlDataType.TypeKind.STRING), + type=StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.STRING + ), ), ] ) diff --git a/tests/unit/routine/test_routine_argument.py b/tests/unit/routine/test_routine_argument.py index e3bda9539..b7f168a30 100644 --- a/tests/unit/routine/test_routine_argument.py +++ b/tests/unit/routine/test_routine_argument.py @@ -16,7 +16,7 @@ import pytest -from google.cloud import bigquery_v2 +from google.cloud import bigquery @pytest.fixture @@ -27,8 +27,8 @@ def target_class(): def test_ctor(target_class): - data_type = bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + data_type = bigquery.standard_sql.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.INT64 ) actual_arg = target_class( name="field_name", kind="FIXED_TYPE", mode="IN", data_type=data_type @@ -50,8 +50,8 @@ def test_from_api_repr(target_class): assert actual_arg.name == "field_name" assert actual_arg.kind == "FIXED_TYPE" assert actual_arg.mode == "IN" - assert actual_arg.data_type == bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + assert actual_arg.data_type == bigquery.standard_sql.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.INT64 ) @@ -71,8 +71,8 @@ def test_from_api_repr_w_unknown_fields(target_class): def test_eq(target_class): - data_type = bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + data_type = bigquery.standard_sql.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.INT64 ) arg = target_class( name="field_name", kind="FIXED_TYPE", mode="IN", data_type=data_type diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 4c6ec5b4f..9b12128c6 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -49,8 +49,8 @@ import google.api_core.exceptions from google.api_core import client_info import google.cloud._helpers +from google.cloud import bigquery from google.cloud import bigquery_storage -from google.cloud import bigquery_v2 from google.cloud.bigquery.dataset import DatasetReference from google.cloud.bigquery.retry import DEFAULT_TIMEOUT @@ -1832,7 +1832,7 @@ def test_update_model(self): self.assertEqual(updated_model.expires, model.expires) # ETag becomes If-Match header. - model._proto.etag = "etag" + model._properties["etag"] = "etag" client.update_model(model, []) req = conn.api_request.call_args self.assertEqual(req[1]["headers"]["If-Match"], "etag") @@ -1862,8 +1862,8 @@ def test_update_routine(self): routine.arguments = [ RoutineArgument( name="x", - data_type=bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + data_type=bigquery.standard_sql.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.INT64 ), ) ] diff --git a/tests/unit/test_dbapi__helpers.py b/tests/unit/test_dbapi__helpers.py index 84c74eeec..b4bb9365f 100644 --- a/tests/unit/test_dbapi__helpers.py +++ b/tests/unit/test_dbapi__helpers.py @@ -22,7 +22,7 @@ import pytest import google.cloud._helpers -from google.cloud.bigquery import table, enums +from google.cloud.bigquery import query, table from google.cloud.bigquery.dbapi import _helpers from google.cloud.bigquery.dbapi import exceptions from tests.unit.helpers import _to_pyarrow @@ -338,8 +338,8 @@ def test_custom_on_closed_error_type(self): VALID_BQ_TYPES = [ - (name, getattr(enums.SqlParameterScalarTypes, name)._type) - for name in dir(enums.SqlParameterScalarTypes) + (name, getattr(query.SqlParameterScalarTypes, name)._type) + for name in dir(query.SqlParameterScalarTypes) if not name.startswith("_") ] diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py index 69a6772e5..71ca67616 100644 --- a/tests/unit/test_query.py +++ b/tests/unit/test_query.py @@ -432,11 +432,11 @@ def test_positional(self): self.assertEqual(param.value, 123) def test_ctor_w_scalar_query_parameter_type(self): - from google.cloud.bigquery import enums + from google.cloud.bigquery import query param = self._make_one( name="foo", - type_=enums.SqlParameterScalarTypes.BIGNUMERIC, + type_=query.SqlParameterScalarTypes.BIGNUMERIC, value=decimal.Decimal("123.456"), ) self.assertEqual(param.name, "foo") diff --git a/tests/unit/test_schema.py b/tests/unit/test_schema.py index d0b5ca54c..edc05494c 100644 --- a/tests/unit/test_schema.py +++ b/tests/unit/test_schema.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from google.cloud import bigquery +from google.cloud.bigquery.standard_sql import StandardSqlStructType from google.cloud.bigquery.schema import PolicyTagList import unittest @@ -28,9 +30,9 @@ def _get_target_class(): @staticmethod def _get_standard_sql_data_type_class(): - from google.cloud.bigquery_v2 import types + from google.cloud.bigquery import standard_sql - return types.StandardSqlDataType + return standard_sql.StandardSqlDataType def _make_one(self, *args, **kw): return self._get_target_class()(*args, **kw) @@ -204,18 +206,17 @@ def test_fields_property(self): self.assertEqual(schema_field.fields, fields) def test_to_standard_sql_simple_type(self): - sql_type = self._get_standard_sql_data_type_class() examples = ( # a few legacy types - ("INTEGER", sql_type.TypeKind.INT64), - ("FLOAT", sql_type.TypeKind.FLOAT64), - ("BOOLEAN", sql_type.TypeKind.BOOL), - ("DATETIME", sql_type.TypeKind.DATETIME), + ("INTEGER", bigquery.StandardSqlTypeNames.INT64), + ("FLOAT", bigquery.StandardSqlTypeNames.FLOAT64), + ("BOOLEAN", bigquery.StandardSqlTypeNames.BOOL), + ("DATETIME", bigquery.StandardSqlTypeNames.DATETIME), # a few standard types - ("INT64", sql_type.TypeKind.INT64), - ("FLOAT64", sql_type.TypeKind.FLOAT64), - ("BOOL", sql_type.TypeKind.BOOL), - ("GEOGRAPHY", sql_type.TypeKind.GEOGRAPHY), + ("INT64", bigquery.StandardSqlTypeNames.INT64), + ("FLOAT64", bigquery.StandardSqlTypeNames.FLOAT64), + ("BOOL", bigquery.StandardSqlTypeNames.BOOL), + ("GEOGRAPHY", bigquery.StandardSqlTypeNames.GEOGRAPHY), ) for legacy_type, standard_type in examples: field = self._make_one("some_field", legacy_type) @@ -224,7 +225,7 @@ def test_to_standard_sql_simple_type(self): self.assertEqual(standard_field.type.type_kind, standard_type) def test_to_standard_sql_struct_type(self): - from google.cloud.bigquery_v2 import types + from google.cloud.bigquery import standard_sql # Expected result object: # @@ -258,30 +259,39 @@ def test_to_standard_sql_struct_type(self): sql_type = self._get_standard_sql_data_type_class() # level 2 fields - sub_sub_field_date = types.StandardSqlField( - name="date_field", type=sql_type(type_kind=sql_type.TypeKind.DATE) + sub_sub_field_date = standard_sql.StandardSqlField( + name="date_field", + type=sql_type(type_kind=bigquery.StandardSqlTypeNames.DATE), ) - sub_sub_field_time = types.StandardSqlField( - name="time_field", type=sql_type(type_kind=sql_type.TypeKind.TIME) + sub_sub_field_time = standard_sql.StandardSqlField( + name="time_field", + type=sql_type(type_kind=bigquery.StandardSqlTypeNames.TIME), ) # level 1 fields - sub_field_struct = types.StandardSqlField( - name="last_used", type=sql_type(type_kind=sql_type.TypeKind.STRUCT) - ) - sub_field_struct.type.struct_type.fields.extend( - [sub_sub_field_date, sub_sub_field_time] + sub_field_struct = standard_sql.StandardSqlField( + name="last_used", + type=sql_type( + type_kind=bigquery.StandardSqlTypeNames.STRUCT, + struct_type=standard_sql.StandardSqlStructType( + fields=[sub_sub_field_date, sub_sub_field_time] + ), + ), ) - sub_field_bytes = types.StandardSqlField( - name="image_content", type=sql_type(type_kind=sql_type.TypeKind.BYTES) + sub_field_bytes = standard_sql.StandardSqlField( + name="image_content", + type=sql_type(type_kind=bigquery.StandardSqlTypeNames.BYTES), ) # level 0 (top level) - expected_result = types.StandardSqlField( - name="image_usage", type=sql_type(type_kind=sql_type.TypeKind.STRUCT) - ) - expected_result.type.struct_type.fields.extend( - [sub_field_bytes, sub_field_struct] + expected_result = standard_sql.StandardSqlField( + name="image_usage", + type=sql_type( + type_kind=bigquery.StandardSqlTypeNames.STRUCT, + struct_type=standard_sql.StandardSqlStructType( + fields=[sub_field_bytes, sub_field_struct] + ), + ), ) # construct legacy SchemaField object @@ -300,14 +310,16 @@ def test_to_standard_sql_struct_type(self): self.assertEqual(standard_field, expected_result) def test_to_standard_sql_array_type_simple(self): - from google.cloud.bigquery_v2 import types + from google.cloud.bigquery import standard_sql sql_type = self._get_standard_sql_data_type_class() # construct expected result object - expected_sql_type = sql_type(type_kind=sql_type.TypeKind.ARRAY) - expected_sql_type.array_element_type.type_kind = sql_type.TypeKind.INT64 - expected_result = types.StandardSqlField( + expected_sql_type = sql_type( + type_kind=bigquery.StandardSqlTypeNames.ARRAY, + array_element_type=sql_type(type_kind=bigquery.StandardSqlTypeNames.INT64), + ) + expected_result = standard_sql.StandardSqlField( name="valid_numbers", type=expected_sql_type ) @@ -318,27 +330,31 @@ def test_to_standard_sql_array_type_simple(self): self.assertEqual(standard_field, expected_result) def test_to_standard_sql_array_type_struct(self): - from google.cloud.bigquery_v2 import types + from google.cloud.bigquery import standard_sql sql_type = self._get_standard_sql_data_type_class() # define person STRUCT - name_field = types.StandardSqlField( - name="name", type=sql_type(type_kind=sql_type.TypeKind.STRING) + name_field = standard_sql.StandardSqlField( + name="name", type=sql_type(type_kind=bigquery.StandardSqlTypeNames.STRING) ) - age_field = types.StandardSqlField( - name="age", type=sql_type(type_kind=sql_type.TypeKind.INT64) + age_field = standard_sql.StandardSqlField( + name="age", type=sql_type(type_kind=bigquery.StandardSqlTypeNames.INT64) ) - person_struct = types.StandardSqlField( - name="person_info", type=sql_type(type_kind=sql_type.TypeKind.STRUCT) + person_struct = standard_sql.StandardSqlField( + name="person_info", + type=sql_type( + type_kind=bigquery.StandardSqlTypeNames.STRUCT, + struct_type=StandardSqlStructType(fields=[name_field, age_field]), + ), ) - person_struct.type.struct_type.fields.extend([name_field, age_field]) # define expected result - an ARRAY of person structs expected_sql_type = sql_type( - type_kind=sql_type.TypeKind.ARRAY, array_element_type=person_struct.type + type_kind=bigquery.StandardSqlTypeNames.ARRAY, + array_element_type=person_struct.type, ) - expected_result = types.StandardSqlField( + expected_result = standard_sql.StandardSqlField( name="known_people", type=expected_sql_type ) @@ -353,14 +369,14 @@ def test_to_standard_sql_array_type_struct(self): self.assertEqual(standard_field, expected_result) def test_to_standard_sql_unknown_type(self): - sql_type = self._get_standard_sql_data_type_class() field = self._make_one("weird_field", "TROOLEAN") standard_field = field.to_standard_sql() self.assertEqual(standard_field.name, "weird_field") self.assertEqual( - standard_field.type.type_kind, sql_type.TypeKind.TYPE_KIND_UNSPECIFIED + standard_field.type.type_kind, + bigquery.StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED, ) def test___eq___wrong_type(self): diff --git a/tests/unit/test_standard_sql_types.py b/tests/unit/test_standard_sql_types.py new file mode 100644 index 000000000..b91f877cc --- /dev/null +++ b/tests/unit/test_standard_sql_types.py @@ -0,0 +1,588 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +import pytest + +from google.cloud import bigquery as bq + + +class TestStandardSqlDataType: + @staticmethod + def _get_target_class(): + from google.cloud.bigquery.standard_sql import StandardSqlDataType + + return StandardSqlDataType + + def _make_one(self, *args, **kw): + return self._get_target_class()(*args, **kw) + + def test_ctor_default_type_kind(self): + instance = self._make_one() + assert instance.type_kind == bq.StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED + + def test_to_api_repr_no_type_set(self): + instance = self._make_one() + instance.type_kind = None + + result = instance.to_api_repr() + + assert result == {"typeKind": "TYPE_KIND_UNSPECIFIED"} + + def test_to_api_repr_scalar_type(self): + instance = self._make_one(bq.StandardSqlTypeNames.FLOAT64) + + result = instance.to_api_repr() + + assert result == {"typeKind": "FLOAT64"} + + def test_to_api_repr_array_type_element_type_missing(self): + instance = self._make_one( + bq.StandardSqlTypeNames.ARRAY, array_element_type=None + ) + + result = instance.to_api_repr() + + expected = {"typeKind": "ARRAY"} + assert result == expected + + def test_to_api_repr_array_type_w_element_type(self): + array_element_type = self._make_one(type_kind=bq.StandardSqlTypeNames.BOOL) + instance = self._make_one( + bq.StandardSqlTypeNames.ARRAY, array_element_type=array_element_type + ) + + result = instance.to_api_repr() + + expected = {"typeKind": "ARRAY", "arrayElementType": {"typeKind": "BOOL"}} + assert result == expected + + def test_to_api_repr_struct_type_field_types_missing(self): + instance = self._make_one(bq.StandardSqlTypeNames.STRUCT, struct_type=None) + + result = instance.to_api_repr() + + assert result == {"typeKind": "STRUCT"} + + def test_to_api_repr_struct_type_w_field_types(self): + from google.cloud.bigquery.standard_sql import StandardSqlField + from google.cloud.bigquery.standard_sql import StandardSqlStructType + + StandardSqlDataType = self._get_target_class() + TypeNames = bq.StandardSqlTypeNames + + person_type = StandardSqlStructType( + fields=[ + StandardSqlField("name", StandardSqlDataType(TypeNames.STRING)), + StandardSqlField("age", StandardSqlDataType(TypeNames.INT64)), + ] + ) + employee_type = StandardSqlStructType( + fields=[ + StandardSqlField("job_title", StandardSqlDataType(TypeNames.STRING)), + StandardSqlField("salary", StandardSqlDataType(TypeNames.FLOAT64)), + StandardSqlField( + "employee_info", + StandardSqlDataType( + type_kind=TypeNames.STRUCT, struct_type=person_type, + ), + ), + ] + ) + + instance = self._make_one(TypeNames.STRUCT, struct_type=employee_type) + result = instance.to_api_repr() + + expected = { + "typeKind": "STRUCT", + "structType": { + "fields": [ + {"name": "job_title", "type": {"typeKind": "STRING"}}, + {"name": "salary", "type": {"typeKind": "FLOAT64"}}, + { + "name": "employee_info", + "type": { + "typeKind": "STRUCT", + "structType": { + "fields": [ + {"name": "name", "type": {"typeKind": "STRING"}}, + {"name": "age", "type": {"typeKind": "INT64"}}, + ], + }, + }, + }, + ], + }, + } + assert result == expected + + def test_from_api_repr_empty_resource(self): + klass = self._get_target_class() + result = klass.from_api_repr(resource={}) + + expected = klass( + type_kind=bq.StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED, + array_element_type=None, + struct_type=None, + ) + assert result == expected + + def test_from_api_repr_scalar_type(self): + klass = self._get_target_class() + resource = {"typeKind": "DATE"} + + result = klass.from_api_repr(resource=resource) + + expected = klass( + type_kind=bq.StandardSqlTypeNames.DATE, + array_element_type=None, + struct_type=None, + ) + assert result == expected + + def test_from_api_repr_array_type_full(self): + klass = self._get_target_class() + resource = {"typeKind": "ARRAY", "arrayElementType": {"typeKind": "BYTES"}} + + result = klass.from_api_repr(resource=resource) + + expected = klass( + type_kind=bq.StandardSqlTypeNames.ARRAY, + array_element_type=klass(type_kind=bq.StandardSqlTypeNames.BYTES), + struct_type=None, + ) + assert result == expected + + def test_from_api_repr_array_type_missing_element_type(self): + klass = self._get_target_class() + resource = {"typeKind": "ARRAY"} + + result = klass.from_api_repr(resource=resource) + + expected = klass( + type_kind=bq.StandardSqlTypeNames.ARRAY, + array_element_type=None, + struct_type=None, + ) + assert result == expected + + def test_from_api_repr_struct_type_nested(self): + from google.cloud.bigquery.standard_sql import StandardSqlField + from google.cloud.bigquery.standard_sql import StandardSqlStructType + + klass = self._get_target_class() + TypeNames = bq.StandardSqlTypeNames + + resource = { + "typeKind": "STRUCT", + "structType": { + "fields": [ + {"name": "job_title", "type": {"typeKind": "STRING"}}, + {"name": "salary", "type": {"typeKind": "FLOAT64"}}, + { + "name": "employee_info", + "type": { + "typeKind": "STRUCT", + "structType": { + "fields": [ + {"name": "name", "type": {"typeKind": "STRING"}}, + {"name": "age", "type": {"typeKind": "INT64"}}, + ], + }, + }, + }, + ], + }, + } + + result = klass.from_api_repr(resource=resource) + + expected = klass( + type_kind=TypeNames.STRUCT, + struct_type=StandardSqlStructType( + fields=[ + StandardSqlField("job_title", klass(TypeNames.STRING)), + StandardSqlField("salary", klass(TypeNames.FLOAT64)), + StandardSqlField( + "employee_info", + klass( + type_kind=TypeNames.STRUCT, + struct_type=StandardSqlStructType( + fields=[ + StandardSqlField("name", klass(TypeNames.STRING)), + StandardSqlField("age", klass(TypeNames.INT64)), + ] + ), + ), + ), + ] + ), + ) + assert result == expected + + def test_from_api_repr_struct_type_missing_struct_info(self): + klass = self._get_target_class() + resource = {"typeKind": "STRUCT"} + + result = klass.from_api_repr(resource=resource) + + expected = klass( + type_kind=bq.StandardSqlTypeNames.STRUCT, + array_element_type=None, + struct_type=None, + ) + assert result == expected + + def test_from_api_repr_struct_type_incomplete_field_info(self): + from google.cloud.bigquery.standard_sql import StandardSqlField + from google.cloud.bigquery.standard_sql import StandardSqlStructType + + klass = self._get_target_class() + TypeNames = bq.StandardSqlTypeNames + + resource = { + "typeKind": "STRUCT", + "structType": { + "fields": [ + {"type": {"typeKind": "STRING"}}, # missing name + {"name": "salary"}, # missing type + ], + }, + } + + result = klass.from_api_repr(resource=resource) + + expected = klass( + type_kind=TypeNames.STRUCT, + struct_type=StandardSqlStructType( + fields=[ + StandardSqlField(None, klass(TypeNames.STRING)), + StandardSqlField("salary", klass(TypeNames.TYPE_KIND_UNSPECIFIED)), + ] + ), + ) + assert result == expected + + def test__eq__another_type(self): + instance = self._make_one() + + class SqlTypeWannabe: + pass + + not_a_type = SqlTypeWannabe() + not_a_type._properties = instance._properties + + assert instance != not_a_type # Can't fake it. + + def test__eq__delegates_comparison_to_another_type(self): + instance = self._make_one() + assert instance == mock.ANY + + def test__eq__similar_instance(self): + kwargs = { + "type_kind": bq.StandardSqlTypeNames.GEOGRAPHY, + "array_element_type": bq.StandardSqlDataType( + type_kind=bq.StandardSqlTypeNames.INT64 + ), + "struct_type": bq.StandardSqlStructType(fields=[]), + } + instance = self._make_one(**kwargs) + instance2 = self._make_one(**kwargs) + assert instance == instance2 + + @pytest.mark.parametrize( + ("attr_name", "value", "value2"), + ( + ( + "type_kind", + bq.StandardSqlTypeNames.INT64, + bq.StandardSqlTypeNames.FLOAT64, + ), + ( + "array_element_type", + bq.StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.STRING), + bq.StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.BOOL), + ), + ( + "struct_type", + bq.StandardSqlStructType(fields=[bq.StandardSqlField(name="foo")]), + bq.StandardSqlStructType(fields=[bq.StandardSqlField(name="bar")]), + ), + ), + ) + def test__eq__attribute_differs(self, attr_name, value, value2): + instance = self._make_one(**{attr_name: value}) + instance2 = self._make_one(**{attr_name: value2}) + assert instance != instance2 + + def test_str(self): + instance = self._make_one(type_kind=bq.StandardSqlTypeNames.BOOL) + bool_type_repr = repr(bq.StandardSqlTypeNames.BOOL) + assert str(instance) == f"StandardSqlDataType(type_kind={bool_type_repr}, ...)" + + +class TestStandardSqlField: + # This class only contains minimum tests to cover what other tests don't + + @staticmethod + def _get_target_class(): + from google.cloud.bigquery.standard_sql import StandardSqlField + + return StandardSqlField + + def _make_one(self, *args, **kw): + return self._get_target_class()(*args, **kw) + + def test_name(self): + instance = self._make_one(name="foo") + assert instance.name == "foo" + instance.name = "bar" + assert instance.name == "bar" + + def test_type_missing(self): + instance = self._make_one(type=None) + assert instance.type is None + + def test_type_set_none(self): + instance = self._make_one( + type=bq.StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.BOOL) + ) + instance.type = None + assert instance.type is None + + def test_type_set_not_none(self): + instance = self._make_one(type=bq.StandardSqlDataType(type_kind=None)) + instance.type = bq.StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.INT64) + assert instance.type == bq.StandardSqlDataType( + type_kind=bq.StandardSqlTypeNames.INT64 + ) + + def test__eq__another_type(self): + instance = self._make_one( + name="foo", + type=bq.StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.BOOL), + ) + + class FieldWannabe: + pass + + not_a_field = FieldWannabe() + not_a_field._properties = instance._properties + + assert instance != not_a_field # Can't fake it. + + def test__eq__delegates_comparison_to_another_type(self): + instance = self._make_one( + name="foo", + type=bq.StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.BOOL), + ) + assert instance == mock.ANY + + def test__eq__similar_instance(self): + kwargs = { + "name": "foo", + "type": bq.StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.INT64), + } + instance = self._make_one(**kwargs) + instance2 = self._make_one(**kwargs) + assert instance == instance2 + + @pytest.mark.parametrize( + ("attr_name", "value", "value2"), + ( + ("name", "foo", "bar",), + ( + "type", + bq.StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.INTERVAL), + bq.StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.TIME), + ), + ), + ) + def test__eq__attribute_differs(self, attr_name, value, value2): + instance = self._make_one(**{attr_name: value}) + instance2 = self._make_one(**{attr_name: value2}) + assert instance != instance2 + + +class TestStandardSqlStructType: + # This class only contains minimum tests to cover what other tests don't + + @staticmethod + def _get_target_class(): + from google.cloud.bigquery.standard_sql import StandardSqlStructType + + return StandardSqlStructType + + def _make_one(self, *args, **kw): + return self._get_target_class()(*args, **kw) + + def test_fields(self): + instance = self._make_one(fields=[]) + assert instance.fields == [] + + new_fields = [bq.StandardSqlField(name="foo"), bq.StandardSqlField(name="bar")] + instance.fields = new_fields + assert instance.fields == new_fields + + def test__eq__another_type(self): + instance = self._make_one(fields=[bq.StandardSqlField(name="foo")]) + + class StructTypeWannabe: + pass + + not_a_type = StructTypeWannabe() + not_a_type._properties = instance._properties + + assert instance != not_a_type # Can't fake it. + + def test__eq__delegates_comparison_to_another_type(self): + instance = self._make_one(fields=[bq.StandardSqlField(name="foo")]) + assert instance == mock.ANY + + def test__eq__similar_instance(self): + kwargs = { + "fields": [bq.StandardSqlField(name="foo"), bq.StandardSqlField(name="bar")] + } + instance = self._make_one(**kwargs) + instance2 = self._make_one(**kwargs) + assert instance == instance2 + + def test__eq__attribute_differs(self): + instance = self._make_one(fields=[bq.StandardSqlField(name="foo")]) + instance2 = self._make_one( + fields=[bq.StandardSqlField(name="foo"), bq.StandardSqlField(name="bar")] + ) + assert instance != instance2 + + +class TestStandardSqlTableType: + @staticmethod + def _get_target_class(): + from google.cloud.bigquery.standard_sql import StandardSqlTableType + + return StandardSqlTableType + + def _make_one(self, *args, **kw): + return self._get_target_class()(*args, **kw) + + def test_columns_shallow_copy(self): + from google.cloud.bigquery.standard_sql import StandardSqlField + + columns = [ + StandardSqlField("foo"), + StandardSqlField("bar"), + StandardSqlField("baz"), + ] + + instance = self._make_one(columns=columns) + + assert len(instance.columns) == 3 + columns.pop() + assert len(instance.columns) == 3 # Still the same. + + def test_columns_setter(self): + from google.cloud.bigquery.standard_sql import StandardSqlField + + columns = [StandardSqlField("foo")] + instance = self._make_one(columns=columns) + assert instance.columns == columns + + new_columns = [StandardSqlField(name="bar")] + instance.columns = new_columns + assert instance.columns == new_columns + + def test_to_api_repr_no_columns(self): + instance = self._make_one(columns=[]) + result = instance.to_api_repr() + assert result == {"columns": []} + + def test_to_api_repr_with_columns(self): + from google.cloud.bigquery.standard_sql import StandardSqlField + + columns = [StandardSqlField("foo"), StandardSqlField("bar")] + instance = self._make_one(columns=columns) + + result = instance.to_api_repr() + + expected = { + "columns": [{"name": "foo", "type": None}, {"name": "bar", "type": None}] + } + assert result == expected + + def test_from_api_repr_missing_columns(self): + resource = {} + result = self._get_target_class().from_api_repr(resource) + assert result.columns == [] + + def test_from_api_repr_with_incomplete_columns(self): + from google.cloud.bigquery.standard_sql import StandardSqlDataType + from google.cloud.bigquery.standard_sql import StandardSqlField + + resource = { + "columns": [ + {"type": {"typeKind": "BOOL"}}, # missing name + {"name": "bar"}, # missing type + ] + } + + result = self._get_target_class().from_api_repr(resource) + + assert len(result.columns) == 2 + + expected = StandardSqlField( + name=None, type=StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.BOOL), + ) + assert result.columns[0] == expected + + expected = StandardSqlField( + name="bar", + type=StandardSqlDataType( + type_kind=bq.StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED + ), + ) + assert result.columns[1] == expected + + def test__eq__another_type(self): + instance = self._make_one(columns=[bq.StandardSqlField(name="foo")]) + + class TableTypeWannabe: + pass + + not_a_type = TableTypeWannabe() + not_a_type._properties = instance._properties + + assert instance != not_a_type # Can't fake it. + + def test__eq__delegates_comparison_to_another_type(self): + instance = self._make_one(columns=[bq.StandardSqlField(name="foo")]) + assert instance == mock.ANY + + def test__eq__similar_instance(self): + kwargs = { + "columns": [ + bq.StandardSqlField(name="foo"), + bq.StandardSqlField(name="bar"), + ] + } + instance = self._make_one(**kwargs) + instance2 = self._make_one(**kwargs) + assert instance == instance2 + + def test__eq__attribute_differs(self): + instance = self._make_one(columns=[bq.StandardSqlField(name="foo")]) + instance2 = self._make_one( + columns=[bq.StandardSqlField(name="foo"), bq.StandardSqlField(name="bar")] + ) + assert instance != instance2