diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/flex_templates_service/transports/grpc.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/flex_templates_service/transports/grpc.py index 3f9bfe7f8a88..571f6bf71710 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/flex_templates_service/transports/grpc.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/flex_templates_service/transports/grpc.py @@ -159,8 +159,11 @@ def __init__( if not self._grpc_channel: self._grpc_channel = type(self).create_channel( self._host, + # use the credentials which are saved credentials=self._credentials, - credentials_file=credentials_file, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, scopes=self._scopes, ssl_credentials=self._ssl_channel_credentials, quota_project_id=quota_project_id, diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/flex_templates_service/transports/grpc_asyncio.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/flex_templates_service/transports/grpc_asyncio.py index f60d014512ff..9b521626e039 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/flex_templates_service/transports/grpc_asyncio.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/flex_templates_service/transports/grpc_asyncio.py @@ -204,8 +204,11 @@ def __init__( if not self._grpc_channel: self._grpc_channel = type(self).create_channel( self._host, + # use the credentials which are saved credentials=self._credentials, - credentials_file=credentials_file, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, scopes=self._scopes, ssl_credentials=self._ssl_channel_credentials, quota_project_id=quota_project_id, diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/jobs_v1_beta3/transports/grpc.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/jobs_v1_beta3/transports/grpc.py index a90a904b3287..fc2bf53eba88 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/jobs_v1_beta3/transports/grpc.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/jobs_v1_beta3/transports/grpc.py @@ -161,8 +161,11 @@ def __init__( if not self._grpc_channel: self._grpc_channel = type(self).create_channel( self._host, + # use the credentials which are saved credentials=self._credentials, - credentials_file=credentials_file, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, scopes=self._scopes, ssl_credentials=self._ssl_channel_credentials, quota_project_id=quota_project_id, diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/jobs_v1_beta3/transports/grpc_asyncio.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/jobs_v1_beta3/transports/grpc_asyncio.py index f76d9cb768bc..f94a1004b46e 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/jobs_v1_beta3/transports/grpc_asyncio.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/jobs_v1_beta3/transports/grpc_asyncio.py @@ -206,8 +206,11 @@ def __init__( if not self._grpc_channel: self._grpc_channel = type(self).create_channel( self._host, + # use the credentials which are saved credentials=self._credentials, - credentials_file=credentials_file, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, scopes=self._scopes, ssl_credentials=self._ssl_channel_credentials, quota_project_id=quota_project_id, diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/messages_v1_beta3/transports/grpc.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/messages_v1_beta3/transports/grpc.py index 33385d4cca87..4013dc9284a1 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/messages_v1_beta3/transports/grpc.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/messages_v1_beta3/transports/grpc.py @@ -159,8 +159,11 @@ def __init__( if not self._grpc_channel: self._grpc_channel = type(self).create_channel( self._host, + # use the credentials which are saved credentials=self._credentials, - credentials_file=credentials_file, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, scopes=self._scopes, ssl_credentials=self._ssl_channel_credentials, quota_project_id=quota_project_id, diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/messages_v1_beta3/transports/grpc_asyncio.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/messages_v1_beta3/transports/grpc_asyncio.py index c9706aac9495..b26c4693069a 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/messages_v1_beta3/transports/grpc_asyncio.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/messages_v1_beta3/transports/grpc_asyncio.py @@ -204,8 +204,11 @@ def __init__( if not self._grpc_channel: self._grpc_channel = type(self).create_channel( self._host, + # use the credentials which are saved credentials=self._credentials, - credentials_file=credentials_file, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, scopes=self._scopes, ssl_credentials=self._ssl_channel_credentials, quota_project_id=quota_project_id, diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/metrics_v1_beta3/async_client.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/metrics_v1_beta3/async_client.py index 75da40948eab..74525effc986 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/metrics_v1_beta3/async_client.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/metrics_v1_beta3/async_client.py @@ -239,8 +239,8 @@ async def get_job_metrics( in the job. This resource captures only the most - recent values of each metric; time- - series data can be queried for them + recent values of each metric; + time-series data can be queried for them (under the same metric names) from Cloud Monitoring. diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/metrics_v1_beta3/client.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/metrics_v1_beta3/client.py index 33c2ec932b89..64049ff54f10 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/metrics_v1_beta3/client.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/metrics_v1_beta3/client.py @@ -422,8 +422,8 @@ def get_job_metrics( in the job. This resource captures only the most - recent values of each metric; time- - series data can be queried for them + recent values of each metric; + time-series data can be queried for them (under the same metric names) from Cloud Monitoring. diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/metrics_v1_beta3/transports/grpc.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/metrics_v1_beta3/transports/grpc.py index 476f619215ff..6f89df87ba66 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/metrics_v1_beta3/transports/grpc.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/metrics_v1_beta3/transports/grpc.py @@ -159,8 +159,11 @@ def __init__( if not self._grpc_channel: self._grpc_channel = type(self).create_channel( self._host, + # use the credentials which are saved credentials=self._credentials, - credentials_file=credentials_file, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, scopes=self._scopes, ssl_credentials=self._ssl_channel_credentials, quota_project_id=quota_project_id, diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/metrics_v1_beta3/transports/grpc_asyncio.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/metrics_v1_beta3/transports/grpc_asyncio.py index 463d4c16f23d..11862191221c 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/metrics_v1_beta3/transports/grpc_asyncio.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/metrics_v1_beta3/transports/grpc_asyncio.py @@ -204,8 +204,11 @@ def __init__( if not self._grpc_channel: self._grpc_channel = type(self).create_channel( self._host, + # use the credentials which are saved credentials=self._credentials, - credentials_file=credentials_file, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, scopes=self._scopes, ssl_credentials=self._ssl_channel_credentials, quota_project_id=quota_project_id, diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/snapshots_v1_beta3/transports/grpc.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/snapshots_v1_beta3/transports/grpc.py index d7cea869ba6b..f0089dc6ea3e 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/snapshots_v1_beta3/transports/grpc.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/snapshots_v1_beta3/transports/grpc.py @@ -159,8 +159,11 @@ def __init__( if not self._grpc_channel: self._grpc_channel = type(self).create_channel( self._host, + # use the credentials which are saved credentials=self._credentials, - credentials_file=credentials_file, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, scopes=self._scopes, ssl_credentials=self._ssl_channel_credentials, quota_project_id=quota_project_id, diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/snapshots_v1_beta3/transports/grpc_asyncio.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/snapshots_v1_beta3/transports/grpc_asyncio.py index ad12d05838d3..8d88dac4d910 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/snapshots_v1_beta3/transports/grpc_asyncio.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/snapshots_v1_beta3/transports/grpc_asyncio.py @@ -204,8 +204,11 @@ def __init__( if not self._grpc_channel: self._grpc_channel = type(self).create_channel( self._host, + # use the credentials which are saved credentials=self._credentials, - credentials_file=credentials_file, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, scopes=self._scopes, ssl_credentials=self._ssl_channel_credentials, quota_project_id=quota_project_id, diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/templates_service/transports/grpc.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/templates_service/transports/grpc.py index cedc7be16118..e9094aaecbe4 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/templates_service/transports/grpc.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/templates_service/transports/grpc.py @@ -160,8 +160,11 @@ def __init__( if not self._grpc_channel: self._grpc_channel = type(self).create_channel( self._host, + # use the credentials which are saved credentials=self._credentials, - credentials_file=credentials_file, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, scopes=self._scopes, ssl_credentials=self._ssl_channel_credentials, quota_project_id=quota_project_id, diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/templates_service/transports/grpc_asyncio.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/templates_service/transports/grpc_asyncio.py index 9e0d75b246fe..c6dba8c04752 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/templates_service/transports/grpc_asyncio.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/services/templates_service/transports/grpc_asyncio.py @@ -205,8 +205,11 @@ def __init__( if not self._grpc_channel: self._grpc_channel = type(self).create_channel( self._host, + # use the credentials which are saved credentials=self._credentials, - credentials_file=credentials_file, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, scopes=self._scopes, ssl_credentials=self._ssl_channel_credentials, quota_project_id=quota_project_id, diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/environment.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/environment.py index 27316892d90f..0728c37bcc1d 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/environment.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/environment.py @@ -280,8 +280,8 @@ class Disk(proto.Message): default. For example, the standard persistent disk type - is a resource name typically ending in "pd- - standard". If SSD persistent disks are + is a resource name typically ending in + "pd-standard". If SSD persistent disks are available, the resource name typically ends with "pd-ssd". The actual valid values are defined the Google Compute Engine API, not by the Cloud @@ -294,8 +294,7 @@ class Disk(proto.Message): the resource name will typically look something like this: - compute.googleapis.com/projects/project- - id/zones/zone/diskTypes/pd-standard + compute.googleapis.com/projects/project-id/zones/zone/diskTypes/pd-standard mount_point (str): Directory in a VM where disk is mounted. """ diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/jobs.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/jobs.py index 854184f5fb40..6b012dde6955 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/jobs.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/jobs.py @@ -331,7 +331,7 @@ class FileIODetails(proto.Message): class BigTableIODetails(proto.Message): - r"""Metadata for a Cloud BigTable connector used by the job. + r"""Metadata for a Cloud Bigtable connector used by the job. Attributes: project_id (str): @@ -424,11 +424,11 @@ class JobMetadata(proto.Message): Identification of a BigQuery source used in the Dataflow job. big_table_details (Sequence[google.cloud.dataflow_v1beta3.types.BigTableIODetails]): - Identification of a Cloud BigTable source + Identification of a Cloud Bigtable source used in the Dataflow job. pubsub_details (Sequence[google.cloud.dataflow_v1beta3.types.PubSubIODetails]): - Identification of a PubSub source used in the - Dataflow job. + Identification of a Pub/Sub source used in + the Dataflow job. file_details (Sequence[google.cloud.dataflow_v1beta3.types.FileIODetails]): Identification of a File source used in the Dataflow job. diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/messages.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/messages.py index 355ab7978b69..4b933418e3e9 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/messages.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/messages.py @@ -118,9 +118,8 @@ class AutoscalingEvent(proto.Message): The time this event was emitted to indicate a new target or current num_workers value. worker_pool (str): - A short and friendly name for the worker pool this event - refers to, populated from the value of - PoolStageRelation::user_pool_name. + A short and friendly name for the worker pool + this event refers to. """ class AutoscalingEventType(proto.Enum): diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/metrics.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/metrics.py index 887470fe54df..eb31743f0d1f 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/metrics.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/metrics.py @@ -86,8 +86,8 @@ class MetricUpdate(proto.Message): Metric aggregation kind. The possible metric aggregation kinds are "Sum", "Max", "Min", "Mean", "Set", "And", "Or", and "Distribution". - The specified aggregation kind is case- - insensitive. + The specified aggregation kind is + case-insensitive. If omitted, this is not an aggregated value but instead a single metric sample value. cumulative (bool): @@ -177,8 +177,8 @@ class GetJobMetricsRequest(proto.Message): class JobMetrics(proto.Message): r"""JobMetrics contains a collection of metrics describing the - detailed progress of a Dataflow job. Metrics correspond to user- - defined and system-defined metrics in the job. + detailed progress of a Dataflow job. Metrics correspond to + user-defined and system-defined metrics in the job. This resource captures only the most recent values of each metric; time-series data can be queried for them (under the same diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/snapshots.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/snapshots.py index 9062c58a9f5c..f63e73503449 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/snapshots.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/snapshots.py @@ -79,7 +79,7 @@ class Snapshot(proto.Message): state (google.cloud.dataflow_v1beta3.types.SnapshotState): State of the snapshot. pubsub_metadata (Sequence[google.cloud.dataflow_v1beta3.types.PubsubSnapshotMetadata]): - PubSub snapshot metadata. + Pub/Sub snapshot metadata. description (str): User specified description of the snapshot. Maybe empty. diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/streaming.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/streaming.py index 4ea44793c2a2..9fd8816315d6 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/streaming.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/streaming.py @@ -78,12 +78,10 @@ class PubsubLocation(proto.Message): Attributes: topic (str): A pubsub topic, in the form of - "pubsub.googleapis.com/topics//". + "pubsub.googleapis.com/topics//". subscription (str): A pubsub subscription, in the form of - "pubsub.googleapis.com/subscriptions//". + "pubsub.googleapis.com/subscriptions//". timestamp_label (str): If set, contains a pubsub label from which to extract record timestamps. If left empty, record @@ -93,8 +91,8 @@ class PubsubLocation(proto.Message): extract record ids. If left empty, record deduplication will be strictly best effort. drop_late_data (bool): - Indicates whether the pipeline allows late- - rriving data. + Indicates whether the pipeline allows + late-arriving data. tracking_subscription (str): If set, specifies the pubsub subscription that will be used for tracking custom time diff --git a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/templates.py b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/templates.py index e204ce3e3461..2ba43147bbee 100644 --- a/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/templates.py +++ b/packages/google-cloud-dataflow-client/google/cloud/dataflow_v1beta3/types/templates.py @@ -239,6 +239,24 @@ class FlexTemplateRuntimeEnvironment(proto.Message): to use for the 'worker harness. Default is the container for the version of the SDK. Note this field is only valid for portable pipelines. + disk_size_gb (int): + Worker disk size, in gigabytes. + autoscaling_algorithm (google.cloud.dataflow_v1beta3.types.AutoscalingAlgorithm): + The algorithm to use for autoscaling + dump_heap_on_oom (bool): + If true, save a heap dump before killing a + thread or process which is GC thrashing or out + of memory. The location of the heap file will + either be echoed back to the user, or the user + will be given the opportunity to download the + heap file. + save_heap_dumps_to_gcs_path (str): + Cloud Storage bucket (directory) to upload heap dumps to the + given location. Enabling this implies that heap dumps should + be generated on OOM (dump_heap_on_oom is set to true). + launcher_machine_type (str): + The machine type to use for launching the + job. The default is n1-standard-1. """ num_workers = proto.Field(proto.INT32, number=1,) @@ -263,6 +281,13 @@ class FlexTemplateRuntimeEnvironment(proto.Message): ) staging_location = proto.Field(proto.STRING, number=17,) sdk_container_image = proto.Field(proto.STRING, number=18,) + disk_size_gb = proto.Field(proto.INT32, number=20,) + autoscaling_algorithm = proto.Field( + proto.ENUM, number=21, enum=gd_environment.AutoscalingAlgorithm, + ) + dump_heap_on_oom = proto.Field(proto.BOOL, number=22,) + save_heap_dumps_to_gcs_path = proto.Field(proto.STRING, number=23,) + launcher_machine_type = proto.Field(proto.STRING, number=24,) class LaunchFlexTemplateRequest(proto.Message): @@ -321,7 +346,8 @@ class RuntimeEnvironment(proto.Message): The machine type to use for the job. Defaults to the value from the template if not specified. additional_experiments (Sequence[str]): - Additional experiment flags for the job. + Additional experiment flags for the job, specified with the + ``--experiments`` option. network (str): Network to which VMs will be assigned. If empty or unspecified, the service will use the diff --git a/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_flex_templates_service.py b/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_flex_templates_service.py index 74eb577211be..c9e73abc83a6 100644 --- a/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_flex_templates_service.py +++ b/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_flex_templates_service.py @@ -532,25 +532,28 @@ def test_flex_templates_service_client_client_options_scopes( @pytest.mark.parametrize( - "client_class,transport_class,transport_name", + "client_class,transport_class,transport_name,grpc_helpers", [ ( FlexTemplatesServiceClient, transports.FlexTemplatesServiceGrpcTransport, "grpc", + grpc_helpers, ), ( FlexTemplatesServiceAsyncClient, transports.FlexTemplatesServiceGrpcAsyncIOTransport, "grpc_asyncio", + grpc_helpers_async, ), ], ) def test_flex_templates_service_client_client_options_credentials_file( - client_class, transport_class, transport_name + client_class, transport_class, transport_name, grpc_helpers ): # Check the case credentials file is provided. options = client_options.ClientOptions(credentials_file="credentials.json") + with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None client = client_class(client_options=options, transport=transport_name) @@ -586,6 +589,77 @@ def test_flex_templates_service_client_client_options_from_dict(): ) +@pytest.mark.parametrize( + "client_class,transport_class,transport_name,grpc_helpers", + [ + ( + FlexTemplatesServiceClient, + transports.FlexTemplatesServiceGrpcTransport, + "grpc", + grpc_helpers, + ), + ( + FlexTemplatesServiceAsyncClient, + transports.FlexTemplatesServiceGrpcAsyncIOTransport, + "grpc_asyncio", + grpc_helpers_async, + ), + ], +) +def test_flex_templates_service_client_create_channel_credentials_file( + client_class, transport_class, transport_name, grpc_helpers +): + # Check the case credentials file is provided. + options = client_options.ClientOptions(credentials_file="credentials.json") + + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # test that the credentials from file are saved and used as the credentials. + with mock.patch.object( + google.auth, "load_credentials_from_file", autospec=True + ) as load_creds, mock.patch.object( + google.auth, "default", autospec=True + ) as adc, mock.patch.object( + grpc_helpers, "create_channel" + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + file_creds = ga_credentials.AnonymousCredentials() + load_creds.return_value = (file_creds, None) + adc.return_value = (creds, None) + client = client_class(client_options=options, transport=transport_name) + create_channel.assert_called_with( + "dataflow.googleapis.com:443", + credentials=file_creds, + credentials_file=None, + quota_project_id=None, + default_scopes=( + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/compute", + "https://www.googleapis.com/auth/compute.readonly", + "https://www.googleapis.com/auth/userinfo.email", + ), + scopes=None, + default_host="dataflow.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + @pytest.mark.parametrize("request_type", [templates.LaunchFlexTemplateRequest, dict,]) def test_launch_flex_template(request_type, transport: str = "grpc"): client = FlexTemplatesServiceClient( diff --git a/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_jobs_v1_beta3.py b/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_jobs_v1_beta3.py index e117dff3f17f..d6f50c21d2d7 100644 --- a/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_jobs_v1_beta3.py +++ b/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_jobs_v1_beta3.py @@ -499,21 +499,23 @@ def test_jobs_v1_beta3_client_client_options_scopes( @pytest.mark.parametrize( - "client_class,transport_class,transport_name", + "client_class,transport_class,transport_name,grpc_helpers", [ - (JobsV1Beta3Client, transports.JobsV1Beta3GrpcTransport, "grpc"), + (JobsV1Beta3Client, transports.JobsV1Beta3GrpcTransport, "grpc", grpc_helpers), ( JobsV1Beta3AsyncClient, transports.JobsV1Beta3GrpcAsyncIOTransport, "grpc_asyncio", + grpc_helpers_async, ), ], ) def test_jobs_v1_beta3_client_client_options_credentials_file( - client_class, transport_class, transport_name + client_class, transport_class, transport_name, grpc_helpers ): # Check the case credentials file is provided. options = client_options.ClientOptions(credentials_file="credentials.json") + with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None client = client_class(client_options=options, transport=transport_name) @@ -547,6 +549,72 @@ def test_jobs_v1_beta3_client_client_options_from_dict(): ) +@pytest.mark.parametrize( + "client_class,transport_class,transport_name,grpc_helpers", + [ + (JobsV1Beta3Client, transports.JobsV1Beta3GrpcTransport, "grpc", grpc_helpers), + ( + JobsV1Beta3AsyncClient, + transports.JobsV1Beta3GrpcAsyncIOTransport, + "grpc_asyncio", + grpc_helpers_async, + ), + ], +) +def test_jobs_v1_beta3_client_create_channel_credentials_file( + client_class, transport_class, transport_name, grpc_helpers +): + # Check the case credentials file is provided. + options = client_options.ClientOptions(credentials_file="credentials.json") + + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # test that the credentials from file are saved and used as the credentials. + with mock.patch.object( + google.auth, "load_credentials_from_file", autospec=True + ) as load_creds, mock.patch.object( + google.auth, "default", autospec=True + ) as adc, mock.patch.object( + grpc_helpers, "create_channel" + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + file_creds = ga_credentials.AnonymousCredentials() + load_creds.return_value = (file_creds, None) + adc.return_value = (creds, None) + client = client_class(client_options=options, transport=transport_name) + create_channel.assert_called_with( + "dataflow.googleapis.com:443", + credentials=file_creds, + credentials_file=None, + quota_project_id=None, + default_scopes=( + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/compute", + "https://www.googleapis.com/auth/compute.readonly", + "https://www.googleapis.com/auth/userinfo.email", + ), + scopes=None, + default_host="dataflow.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + @pytest.mark.parametrize("request_type", [jobs.CreateJobRequest, dict,]) def test_create_job(request_type, transport: str = "grpc"): client = JobsV1Beta3Client( diff --git a/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_messages_v1_beta3.py b/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_messages_v1_beta3.py index 22bc6f6fea78..9600b7a40380 100644 --- a/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_messages_v1_beta3.py +++ b/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_messages_v1_beta3.py @@ -523,21 +523,28 @@ def test_messages_v1_beta3_client_client_options_scopes( @pytest.mark.parametrize( - "client_class,transport_class,transport_name", + "client_class,transport_class,transport_name,grpc_helpers", [ - (MessagesV1Beta3Client, transports.MessagesV1Beta3GrpcTransport, "grpc"), + ( + MessagesV1Beta3Client, + transports.MessagesV1Beta3GrpcTransport, + "grpc", + grpc_helpers, + ), ( MessagesV1Beta3AsyncClient, transports.MessagesV1Beta3GrpcAsyncIOTransport, "grpc_asyncio", + grpc_helpers_async, ), ], ) def test_messages_v1_beta3_client_client_options_credentials_file( - client_class, transport_class, transport_name + client_class, transport_class, transport_name, grpc_helpers ): # Check the case credentials file is provided. options = client_options.ClientOptions(credentials_file="credentials.json") + with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None client = client_class(client_options=options, transport=transport_name) @@ -573,6 +580,77 @@ def test_messages_v1_beta3_client_client_options_from_dict(): ) +@pytest.mark.parametrize( + "client_class,transport_class,transport_name,grpc_helpers", + [ + ( + MessagesV1Beta3Client, + transports.MessagesV1Beta3GrpcTransport, + "grpc", + grpc_helpers, + ), + ( + MessagesV1Beta3AsyncClient, + transports.MessagesV1Beta3GrpcAsyncIOTransport, + "grpc_asyncio", + grpc_helpers_async, + ), + ], +) +def test_messages_v1_beta3_client_create_channel_credentials_file( + client_class, transport_class, transport_name, grpc_helpers +): + # Check the case credentials file is provided. + options = client_options.ClientOptions(credentials_file="credentials.json") + + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # test that the credentials from file are saved and used as the credentials. + with mock.patch.object( + google.auth, "load_credentials_from_file", autospec=True + ) as load_creds, mock.patch.object( + google.auth, "default", autospec=True + ) as adc, mock.patch.object( + grpc_helpers, "create_channel" + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + file_creds = ga_credentials.AnonymousCredentials() + load_creds.return_value = (file_creds, None) + adc.return_value = (creds, None) + client = client_class(client_options=options, transport=transport_name) + create_channel.assert_called_with( + "dataflow.googleapis.com:443", + credentials=file_creds, + credentials_file=None, + quota_project_id=None, + default_scopes=( + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/compute", + "https://www.googleapis.com/auth/compute.readonly", + "https://www.googleapis.com/auth/userinfo.email", + ), + scopes=None, + default_host="dataflow.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + @pytest.mark.parametrize("request_type", [messages.ListJobMessagesRequest, dict,]) def test_list_job_messages(request_type, transport: str = "grpc"): client = MessagesV1Beta3Client( diff --git a/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_metrics_v1_beta3.py b/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_metrics_v1_beta3.py index 39dd614bdf09..9ed64434fa94 100644 --- a/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_metrics_v1_beta3.py +++ b/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_metrics_v1_beta3.py @@ -511,21 +511,28 @@ def test_metrics_v1_beta3_client_client_options_scopes( @pytest.mark.parametrize( - "client_class,transport_class,transport_name", + "client_class,transport_class,transport_name,grpc_helpers", [ - (MetricsV1Beta3Client, transports.MetricsV1Beta3GrpcTransport, "grpc"), + ( + MetricsV1Beta3Client, + transports.MetricsV1Beta3GrpcTransport, + "grpc", + grpc_helpers, + ), ( MetricsV1Beta3AsyncClient, transports.MetricsV1Beta3GrpcAsyncIOTransport, "grpc_asyncio", + grpc_helpers_async, ), ], ) def test_metrics_v1_beta3_client_client_options_credentials_file( - client_class, transport_class, transport_name + client_class, transport_class, transport_name, grpc_helpers ): # Check the case credentials file is provided. options = client_options.ClientOptions(credentials_file="credentials.json") + with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None client = client_class(client_options=options, transport=transport_name) @@ -561,6 +568,77 @@ def test_metrics_v1_beta3_client_client_options_from_dict(): ) +@pytest.mark.parametrize( + "client_class,transport_class,transport_name,grpc_helpers", + [ + ( + MetricsV1Beta3Client, + transports.MetricsV1Beta3GrpcTransport, + "grpc", + grpc_helpers, + ), + ( + MetricsV1Beta3AsyncClient, + transports.MetricsV1Beta3GrpcAsyncIOTransport, + "grpc_asyncio", + grpc_helpers_async, + ), + ], +) +def test_metrics_v1_beta3_client_create_channel_credentials_file( + client_class, transport_class, transport_name, grpc_helpers +): + # Check the case credentials file is provided. + options = client_options.ClientOptions(credentials_file="credentials.json") + + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # test that the credentials from file are saved and used as the credentials. + with mock.patch.object( + google.auth, "load_credentials_from_file", autospec=True + ) as load_creds, mock.patch.object( + google.auth, "default", autospec=True + ) as adc, mock.patch.object( + grpc_helpers, "create_channel" + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + file_creds = ga_credentials.AnonymousCredentials() + load_creds.return_value = (file_creds, None) + adc.return_value = (creds, None) + client = client_class(client_options=options, transport=transport_name) + create_channel.assert_called_with( + "dataflow.googleapis.com:443", + credentials=file_creds, + credentials_file=None, + quota_project_id=None, + default_scopes=( + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/compute", + "https://www.googleapis.com/auth/compute.readonly", + "https://www.googleapis.com/auth/userinfo.email", + ), + scopes=None, + default_host="dataflow.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + @pytest.mark.parametrize("request_type", [metrics.GetJobMetricsRequest, dict,]) def test_get_job_metrics(request_type, transport: str = "grpc"): client = MetricsV1Beta3Client( diff --git a/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_snapshots_v1_beta3.py b/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_snapshots_v1_beta3.py index 0384e58ec2b3..43478d56d43f 100644 --- a/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_snapshots_v1_beta3.py +++ b/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_snapshots_v1_beta3.py @@ -524,21 +524,28 @@ def test_snapshots_v1_beta3_client_client_options_scopes( @pytest.mark.parametrize( - "client_class,transport_class,transport_name", + "client_class,transport_class,transport_name,grpc_helpers", [ - (SnapshotsV1Beta3Client, transports.SnapshotsV1Beta3GrpcTransport, "grpc"), + ( + SnapshotsV1Beta3Client, + transports.SnapshotsV1Beta3GrpcTransport, + "grpc", + grpc_helpers, + ), ( SnapshotsV1Beta3AsyncClient, transports.SnapshotsV1Beta3GrpcAsyncIOTransport, "grpc_asyncio", + grpc_helpers_async, ), ], ) def test_snapshots_v1_beta3_client_client_options_credentials_file( - client_class, transport_class, transport_name + client_class, transport_class, transport_name, grpc_helpers ): # Check the case credentials file is provided. options = client_options.ClientOptions(credentials_file="credentials.json") + with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None client = client_class(client_options=options, transport=transport_name) @@ -574,6 +581,77 @@ def test_snapshots_v1_beta3_client_client_options_from_dict(): ) +@pytest.mark.parametrize( + "client_class,transport_class,transport_name,grpc_helpers", + [ + ( + SnapshotsV1Beta3Client, + transports.SnapshotsV1Beta3GrpcTransport, + "grpc", + grpc_helpers, + ), + ( + SnapshotsV1Beta3AsyncClient, + transports.SnapshotsV1Beta3GrpcAsyncIOTransport, + "grpc_asyncio", + grpc_helpers_async, + ), + ], +) +def test_snapshots_v1_beta3_client_create_channel_credentials_file( + client_class, transport_class, transport_name, grpc_helpers +): + # Check the case credentials file is provided. + options = client_options.ClientOptions(credentials_file="credentials.json") + + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # test that the credentials from file are saved and used as the credentials. + with mock.patch.object( + google.auth, "load_credentials_from_file", autospec=True + ) as load_creds, mock.patch.object( + google.auth, "default", autospec=True + ) as adc, mock.patch.object( + grpc_helpers, "create_channel" + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + file_creds = ga_credentials.AnonymousCredentials() + load_creds.return_value = (file_creds, None) + adc.return_value = (creds, None) + client = client_class(client_options=options, transport=transport_name) + create_channel.assert_called_with( + "dataflow.googleapis.com:443", + credentials=file_creds, + credentials_file=None, + quota_project_id=None, + default_scopes=( + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/compute", + "https://www.googleapis.com/auth/compute.readonly", + "https://www.googleapis.com/auth/userinfo.email", + ), + scopes=None, + default_host="dataflow.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + @pytest.mark.parametrize("request_type", [snapshots.GetSnapshotRequest, dict,]) def test_get_snapshot(request_type, transport: str = "grpc"): client = SnapshotsV1Beta3Client( diff --git a/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_templates_service.py b/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_templates_service.py index bbfa95199ea4..90ac49f50468 100644 --- a/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_templates_service.py +++ b/packages/google-cloud-dataflow-client/tests/unit/gapic/dataflow_v1beta3/test_templates_service.py @@ -526,21 +526,28 @@ def test_templates_service_client_client_options_scopes( @pytest.mark.parametrize( - "client_class,transport_class,transport_name", + "client_class,transport_class,transport_name,grpc_helpers", [ - (TemplatesServiceClient, transports.TemplatesServiceGrpcTransport, "grpc"), + ( + TemplatesServiceClient, + transports.TemplatesServiceGrpcTransport, + "grpc", + grpc_helpers, + ), ( TemplatesServiceAsyncClient, transports.TemplatesServiceGrpcAsyncIOTransport, "grpc_asyncio", + grpc_helpers_async, ), ], ) def test_templates_service_client_client_options_credentials_file( - client_class, transport_class, transport_name + client_class, transport_class, transport_name, grpc_helpers ): # Check the case credentials file is provided. options = client_options.ClientOptions(credentials_file="credentials.json") + with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None client = client_class(client_options=options, transport=transport_name) @@ -576,6 +583,77 @@ def test_templates_service_client_client_options_from_dict(): ) +@pytest.mark.parametrize( + "client_class,transport_class,transport_name,grpc_helpers", + [ + ( + TemplatesServiceClient, + transports.TemplatesServiceGrpcTransport, + "grpc", + grpc_helpers, + ), + ( + TemplatesServiceAsyncClient, + transports.TemplatesServiceGrpcAsyncIOTransport, + "grpc_asyncio", + grpc_helpers_async, + ), + ], +) +def test_templates_service_client_create_channel_credentials_file( + client_class, transport_class, transport_name, grpc_helpers +): + # Check the case credentials file is provided. + options = client_options.ClientOptions(credentials_file="credentials.json") + + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # test that the credentials from file are saved and used as the credentials. + with mock.patch.object( + google.auth, "load_credentials_from_file", autospec=True + ) as load_creds, mock.patch.object( + google.auth, "default", autospec=True + ) as adc, mock.patch.object( + grpc_helpers, "create_channel" + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + file_creds = ga_credentials.AnonymousCredentials() + load_creds.return_value = (file_creds, None) + adc.return_value = (creds, None) + client = client_class(client_options=options, transport=transport_name) + create_channel.assert_called_with( + "dataflow.googleapis.com:443", + credentials=file_creds, + credentials_file=None, + quota_project_id=None, + default_scopes=( + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/compute", + "https://www.googleapis.com/auth/compute.readonly", + "https://www.googleapis.com/auth/userinfo.email", + ), + scopes=None, + default_host="dataflow.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + @pytest.mark.parametrize( "request_type", [templates.CreateJobFromTemplateRequest, dict,] )