diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 12f0d23f17dc..d2626c7b19a8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -51,7 +51,7 @@ /pubsub/ @anguillanneuf @hongalex @GoogleCloudPlatform/python-samples-owners /run/ @averikitsch @GoogleCloudPlatform/python-samples-owners /scheduler/ @averikitsch @GoogleCloudPlatform/python-samples-owners -/spanner/ @mmmarklu @GoogleCloudPlatform/python-samples-owners +/spanner/ @larkee @GoogleCloudPlatform/python-samples-owners /speech/ @telpirion @sirtorry @GoogleCloudPlatform/python-samples-owners /storage/ @GoogleCloudPlatform/storage-dpe @GoogleCloudPlatform/python-samples-owners /tables/automl/ @telpirion @sirtorry @GoogleCloudPlatform/python-samples-owners diff --git a/appengine/flexible/numpy/requirements.txt b/appengine/flexible/numpy/requirements.txt index 3ef68dd38fe4..c6eb5263b7cf 100644 --- a/appengine/flexible/numpy/requirements.txt +++ b/appengine/flexible/numpy/requirements.txt @@ -1,3 +1,3 @@ Flask==1.1.2 gunicorn==20.0.4 -numpy==1.18.4 +numpy==1.18.5 diff --git a/appengine/flexible/scipy/requirements.txt b/appengine/flexible/scipy/requirements.txt index c15f73756036..2b958ff59d1d 100644 --- a/appengine/flexible/scipy/requirements.txt +++ b/appengine/flexible/scipy/requirements.txt @@ -1,6 +1,6 @@ Flask==1.1.2 gunicorn==20.0.4 imageio==2.8.0 -numpy==1.18.4 +numpy==1.18.5 pillow==7.1.2 scipy==1.4.1 diff --git a/appengine/flexible/tasks/requirements.txt b/appengine/flexible/tasks/requirements.txt index cc593bafbfe1..3b8f608fdfc8 100644 --- a/appengine/flexible/tasks/requirements.txt +++ b/appengine/flexible/tasks/requirements.txt @@ -1,4 +1,3 @@ Flask==1.1.2 gunicorn==20.0.4 google-cloud-tasks==1.5.0 -googleapis-common-protos==1.51.0 diff --git a/appengine/standard/firebase/firenotes/backend/requirements.txt b/appengine/standard/firebase/firenotes/backend/requirements.txt index 7c60c068bfb2..5f4cac55e224 100644 --- a/appengine/standard/firebase/firenotes/backend/requirements.txt +++ b/appengine/standard/firebase/firenotes/backend/requirements.txt @@ -1,6 +1,6 @@ Flask==1.1.2 pyjwt==1.7.1 flask-cors==3.0.8 -google-auth==1.16.0 +google-auth==1.16.1 requests==2.23.0 requests-toolbelt==0.9.1 diff --git a/appengine/standard/firebase/firetactoe/requirements.txt b/appengine/standard/firebase/firetactoe/requirements.txt index 25decdab32d5..1767352488eb 100644 --- a/appengine/standard/firebase/firetactoe/requirements.txt +++ b/appengine/standard/firebase/firetactoe/requirements.txt @@ -1,5 +1,5 @@ flask==1.1.2 requests==2.23.0 requests_toolbelt==0.9.1 -google-auth==1.16.0 +google-auth==1.16.1 functools32==3.2.3.post2; python_version < "3" diff --git a/appengine/standard/storage/api-client/requirements.txt b/appengine/standard/storage/api-client/requirements.txt index 0f61949724cb..130c1828654a 100644 --- a/appengine/standard/storage/api-client/requirements.txt +++ b/appengine/standard/storage/api-client/requirements.txt @@ -1,3 +1,3 @@ google-api-python-client==1.9.1 -google-auth==1.16.0 +google-auth==1.16.1 google-auth-httplib2==0.0.3 diff --git a/appengine/standard_python3/building-an-app/building-an-app-3/requirements.txt b/appengine/standard_python3/building-an-app/building-an-app-3/requirements.txt index f26b234194be..4794e87f05d3 100644 --- a/appengine/standard_python3/building-an-app/building-an-app-3/requirements.txt +++ b/appengine/standard_python3/building-an-app/building-an-app-3/requirements.txt @@ -1,4 +1,4 @@ Flask==1.1.2 google-cloud-datastore==1.12.0 -google-auth==1.16.0 +google-auth==1.16.1 requests==2.23.0 diff --git a/appengine/standard_python3/building-an-app/building-an-app-4/requirements.txt b/appengine/standard_python3/building-an-app/building-an-app-4/requirements.txt index f26b234194be..4794e87f05d3 100644 --- a/appengine/standard_python3/building-an-app/building-an-app-4/requirements.txt +++ b/appengine/standard_python3/building-an-app/building-an-app-4/requirements.txt @@ -1,4 +1,4 @@ Flask==1.1.2 google-cloud-datastore==1.12.0 -google-auth==1.16.0 +google-auth==1.16.1 requests==2.23.0 diff --git a/appengine/standard_python3/pubsub/requirements.txt b/appengine/standard_python3/pubsub/requirements.txt index 319134d413e5..04821fddad20 100644 --- a/appengine/standard_python3/pubsub/requirements.txt +++ b/appengine/standard_python3/pubsub/requirements.txt @@ -1,4 +1,4 @@ Flask==1.1.2 google-api-python-client==1.9.1 -google-auth==1.16.0 +google-auth==1.16.1 google-cloud-pubsub==1.5.0 diff --git a/appengine/standard_python37/building-an-app/building-an-app-3/requirements.txt b/appengine/standard_python37/building-an-app/building-an-app-3/requirements.txt index f26b234194be..4794e87f05d3 100644 --- a/appengine/standard_python37/building-an-app/building-an-app-3/requirements.txt +++ b/appengine/standard_python37/building-an-app/building-an-app-3/requirements.txt @@ -1,4 +1,4 @@ Flask==1.1.2 google-cloud-datastore==1.12.0 -google-auth==1.16.0 +google-auth==1.16.1 requests==2.23.0 diff --git a/appengine/standard_python37/building-an-app/building-an-app-4/requirements.txt b/appengine/standard_python37/building-an-app/building-an-app-4/requirements.txt index f26b234194be..4794e87f05d3 100644 --- a/appengine/standard_python37/building-an-app/building-an-app-4/requirements.txt +++ b/appengine/standard_python37/building-an-app/building-an-app-4/requirements.txt @@ -1,4 +1,4 @@ Flask==1.1.2 google-cloud-datastore==1.12.0 -google-auth==1.16.0 +google-auth==1.16.1 requests==2.23.0 diff --git a/appengine/standard_python37/pubsub/requirements.txt b/appengine/standard_python37/pubsub/requirements.txt index 319134d413e5..04821fddad20 100644 --- a/appengine/standard_python37/pubsub/requirements.txt +++ b/appengine/standard_python37/pubsub/requirements.txt @@ -1,4 +1,4 @@ Flask==1.1.2 google-api-python-client==1.9.1 -google-auth==1.16.0 +google-auth==1.16.1 google-cloud-pubsub==1.5.0 diff --git a/auth/api-client/requirements.txt b/auth/api-client/requirements.txt index ca8c24c579a8..9e7728810ea6 100644 --- a/auth/api-client/requirements.txt +++ b/auth/api-client/requirements.txt @@ -1,3 +1,3 @@ google-api-python-client==1.9.1 google-auth-httplib2==0.0.3 -google-auth==1.16.0 +google-auth==1.16.1 diff --git a/auth/end-user/web/requirements.txt b/auth/end-user/web/requirements.txt index 2d881c4c8639..1ec429671c57 100644 --- a/auth/end-user/web/requirements.txt +++ b/auth/end-user/web/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.16.0 +google-auth==1.16.1 google-auth-oauthlib==0.4.1 google-auth-httplib2==0.0.3 google-api-python-client==1.9.1 diff --git a/bigquery/pandas-gbq-migration/requirements.txt b/bigquery/pandas-gbq-migration/requirements.txt index e3bef02853a0..7b09b45aafbc 100644 --- a/bigquery/pandas-gbq-migration/requirements.txt +++ b/bigquery/pandas-gbq-migration/requirements.txt @@ -1,5 +1,5 @@ google-cloud-bigquery==1.24.0 -google-cloud-bigquery-storage==0.8.0 +google-cloud-bigquery-storage==1.0.0 pandas==0.25.3; python_version > '3.0' pandas==0.24.2; python_version < '3.0' pandas-gbq==0.13.2 diff --git a/bigquery_storage/to_dataframe/requirements.txt b/bigquery_storage/to_dataframe/requirements.txt index bb7f7eb8c3c2..2f8614ad0ca5 100644 --- a/bigquery_storage/to_dataframe/requirements.txt +++ b/bigquery_storage/to_dataframe/requirements.txt @@ -1,5 +1,5 @@ -google-auth==1.16.0 -google-cloud-bigquery-storage==0.8.0 +google-auth==1.16.1 +google-cloud-bigquery-storage==1.0.0 google-cloud-bigquery==1.24.0 pyarrow==0.17.1 ipython==7.10.2; python_version > '3.0' diff --git a/composer/rest/requirements.txt b/composer/rest/requirements.txt index e468d46bbae8..88faa31c220b 100644 --- a/composer/rest/requirements.txt +++ b/composer/rest/requirements.txt @@ -1,3 +1,3 @@ -google-auth==1.16.0 +google-auth==1.16.1 requests==2.23.0 six==1.15.0 diff --git a/compute/api/requirements.txt b/compute/api/requirements.txt index 0f61949724cb..130c1828654a 100644 --- a/compute/api/requirements.txt +++ b/compute/api/requirements.txt @@ -1,3 +1,3 @@ google-api-python-client==1.9.1 -google-auth==1.16.0 +google-auth==1.16.1 google-auth-httplib2==0.0.3 diff --git a/compute/auth/requirements.txt b/compute/auth/requirements.txt index 70a06760da8b..cbbda5db1636 100644 --- a/compute/auth/requirements.txt +++ b/compute/auth/requirements.txt @@ -1,4 +1,4 @@ requests==2.23.0 google-api-python-client==1.9.1 -google-auth==1.16.0 +google-auth==1.16.1 google-auth-httplib2==0.0.3 diff --git a/compute/encryption/requirements.txt b/compute/encryption/requirements.txt index bd7fc056d4ee..929a86a485fc 100644 --- a/compute/encryption/requirements.txt +++ b/compute/encryption/requirements.txt @@ -1,5 +1,5 @@ cryptography==2.9.2 requests==2.23.0 google-api-python-client==1.9.1 -google-auth==1.16.0 +google-auth==1.16.1 google-auth-httplib2==0.0.3 diff --git a/compute/oslogin/requirements.txt b/compute/oslogin/requirements.txt index 28ea0ff5b8ab..375974b66426 100644 --- a/compute/oslogin/requirements.txt +++ b/compute/oslogin/requirements.txt @@ -1,4 +1,4 @@ google-api-python-client==1.9.1 -google-auth==1.16.0 +google-auth==1.16.1 google-auth-httplib2==0.0.3 requests==2.23.0 diff --git a/dataproc/requirements.txt b/dataproc/requirements.txt index 7d1527e66b22..9046478fddff 100644 --- a/dataproc/requirements.txt +++ b/dataproc/requirements.txt @@ -1,5 +1,5 @@ grpcio==1.29.0 -google-auth==1.16.0 +google-auth==1.16.1 google-auth-httplib2==0.0.3 google-cloud==0.34.0 google-cloud-storage==1.28.1 diff --git a/dlp/README.rst b/dlp/README.rst index ce8b85500249..76bd9dd8dfc6 100644 --- a/dlp/README.rst +++ b/dlp/README.rst @@ -14,6 +14,15 @@ This directory contains samples for Google Data Loss Prevention. `Google Data Lo .. _Google Data Loss Prevention: https://cloud.google.com/dlp/docs/ +To run the sample, you need to enable the API at: https://console.cloud.google.com/apis/library/dlp.googleapis.com + + +To run the sample, you need to have the following roles: +* `DLP Administrator` +* `DLP API Service Agent` + + + Setup ------------------------------------------------------------------------------- @@ -58,15 +67,6 @@ Install Dependencies .. _pip: https://pip.pypa.io/ .. _virtualenv: https://virtualenv.pypa.io/ -#. For running *_test.py files, install test dependencies - - .. code-block:: bash - - $ pip install -r requirements-test.txt - $ pytest inspect_content_test.py - -** *_test.py files are demo wrappers and make API calls. You may get rate limited for making high number of requests. ** - Samples ------------------------------------------------------------------------------- @@ -83,7 +83,7 @@ To run this sample: .. code-block:: bash - $ python quickstart.py + $ python quickstart.py Inspect Content @@ -101,15 +101,16 @@ To run this sample: $ python inspect_content.py - usage: inspect_content.py [-h] {string,file,gcs,datastore,bigquery} ... + usage: inspect_content.py [-h] {string,table,file,gcs,datastore,bigquery} ... Sample app that uses the Data Loss Prevention API to inspect a string, a local file or a file on Google Cloud Storage. positional arguments: - {string,file,gcs,datastore,bigquery} + {string,table,file,gcs,datastore,bigquery} Select how to submit content to the API. string Inspect a string. + table Inspect a table. file Inspect a local file. gcs Inspect files on Google Cloud Storage. datastore Inspect files on Google Datastore. @@ -135,13 +136,14 @@ To run this sample: $ python redact.py - usage: redact.py [-h] [--project PROJECT] [--info_types INFO_TYPES] + usage: redact.py [-h] [--project PROJECT] + [--info_types INFO_TYPES [INFO_TYPES ...]] [--min_likelihood {LIKELIHOOD_UNSPECIFIED,VERY_UNLIKELY,UNLIKELY,POSSIBLE,LIKELY,VERY_LIKELY}] [--mime_type MIME_TYPE] filename output_filename - Sample app that uses the Data Loss Prevent API to redact the contents of a - string or an image file. + Sample app that uses the Data Loss Prevent API to redact the contents of an + image file. positional arguments: filename The path to the file to inspect. @@ -151,7 +153,7 @@ To run this sample: -h, --help show this help message and exit --project PROJECT The Google Cloud project id to use as a parent resource. - --info_types INFO_TYPES + --info_types INFO_TYPES [INFO_TYPES ...] Strings representing info types to look for. A full list of info categories and types is available from the API. Examples include "FIRST_NAME", "LAST_NAME", diff --git a/dlp/README.rst.in b/dlp/README.rst.in index 8a143392b17e..708e870fa08a 100644 --- a/dlp/README.rst.in +++ b/dlp/README.rst.in @@ -4,7 +4,7 @@ product: name: Google Data Loss Prevention short_name: Data Loss Prevention url: https://cloud.google.com/dlp/docs/ - description: > + description: > `Google Data Loss Prevention`_ provides programmatic access to a powerful detection engine for personally identifiable information and other privacy-sensitive data in unstructured data streams. @@ -13,6 +13,12 @@ setup: - auth - install_deps +required_api_url: https://console.cloud.google.com/apis/library/dlp.googleapis.com + +required_roles: +- DLP Administrator +- DLP API Service Agent + samples: - name: Quickstart file: quickstart.py diff --git a/dlp/conftest.py b/dlp/conftest.py deleted file mode 100644 index 362e5a2c2711..000000000000 --- a/dlp/conftest.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest - - -# Used in risk_test.py to limit the maximum wait time before the flaky retries. -def pytest_configure(config): - pytest.MAX_FLAKY_WAIT = 3600 # maximum of an hour diff --git a/dlp/inspect_content.py b/dlp/inspect_content.py index 6d6baad4827a..1c5f9c4df958 100644 --- a/dlp/inspect_content.py +++ b/dlp/inspect_content.py @@ -459,11 +459,12 @@ def inspect_gcs_file( url = "gs://{}/{}".format(bucket, filename) storage_config = {"cloud_storage_options": {"file_set": {"url": url}}} - # Convert the project id into a full resource id. - parent = dlp.project_path(project) + # Convert the project id into full resource ids. + topic = google.cloud.pubsub.PublisherClient.topic_path(project, topic_id) + parent = dlp.location_path(project, 'global') # Tell the API where to send a notification when the job is complete. - actions = [{"pub_sub": {"topic": "{}/topics/{}".format(parent, topic_id)}}] + actions = [{"pub_sub": {"topic": topic}}] # Construct the inspect_job, which defines the entire inspect content task. inspect_job = { @@ -623,11 +624,12 @@ def inspect_datastore( } } - # Convert the project id into a full resource id. - parent = dlp.project_path(project) + # Convert the project id into full resource ids. + topic = google.cloud.pubsub.PublisherClient.topic_path(project, topic_id) + parent = dlp.location_path(project, 'global') # Tell the API where to send a notification when the job is complete. - actions = [{"pub_sub": {"topic": "{}/topics/{}".format(parent, topic_id)}}] + actions = [{"pub_sub": {"topic": topic}}] # Construct the inspect_job, which defines the entire inspect content task. inspect_job = { @@ -790,11 +792,12 @@ def inspect_bigquery( } } - # Convert the project id into a full resource id. - parent = dlp.project_path(project) + # Convert the project id into full resource ids. + topic = google.cloud.pubsub.PublisherClient.topic_path(project, topic_id) + parent = dlp.location_path(project, 'global') # Tell the API where to send a notification when the job is complete. - actions = [{"pub_sub": {"topic": "{}/topics/{}".format(parent, topic_id)}}] + actions = [{"pub_sub": {"topic": topic}}] # Construct the inspect_job, which defines the entire inspect content task. inspect_job = { diff --git a/dlp/inspect_content_test.py b/dlp/inspect_content_test.py index ea100d16d84a..e2192bdd6c41 100644 --- a/dlp/inspect_content_test.py +++ b/dlp/inspect_content_test.py @@ -40,6 +40,8 @@ BIGQUERY_DATASET_ID = "dlp_test_dataset" + UNIQUE_STRING BIGQUERY_TABLE_ID = "dlp_test_table" + UNIQUE_STRING +TIMEOUT = 300 # 5 minutes + @pytest.fixture(scope="module") def bucket(): @@ -298,6 +300,7 @@ def cancel_operation(out): client.cancel_dlp_job(operation_id) +@pytest.mark.flaky(max_runs=2, min_passes=1) def test_inspect_gcs_file(bucket, topic_id, subscription_id, capsys): try: inspect_content.inspect_gcs_file( @@ -307,15 +310,16 @@ def test_inspect_gcs_file(bucket, topic_id, subscription_id, capsys): topic_id, subscription_id, ["EMAIL_ADDRESS", "PHONE_NUMBER"], - timeout=1 + timeout=TIMEOUT ) out, _ = capsys.readouterr() - assert "Inspection operation started" in out + assert "Info type: EMAIL_ADDRESS" in out finally: cancel_operation(out) +@pytest.mark.flaky(max_runs=2, min_passes=1) def test_inspect_gcs_file_with_custom_info_types( bucket, topic_id, subscription_id, capsys): try: @@ -331,15 +335,16 @@ def test_inspect_gcs_file_with_custom_info_types( [], custom_dictionaries=dictionaries, custom_regexes=regexes, - timeout=1) + timeout=TIMEOUT) out, _ = capsys.readouterr() - assert "Inspection operation started" in out + assert "Info type: EMAIL_ADDRESS" in out finally: cancel_operation(out) +@pytest.mark.flaky(max_runs=2, min_passes=1) def test_inspect_gcs_file_no_results( bucket, topic_id, subscription_id, capsys): try: @@ -350,15 +355,16 @@ def test_inspect_gcs_file_no_results( topic_id, subscription_id, ["EMAIL_ADDRESS", "PHONE_NUMBER"], - timeout=1) + timeout=TIMEOUT) out, _ = capsys.readouterr() - assert "Inspection operation started" in out + assert "No findings" in out finally: cancel_operation(out) +@pytest.mark.flaky(max_runs=2, min_passes=1) def test_inspect_gcs_image_file(bucket, topic_id, subscription_id, capsys): try: inspect_content.inspect_gcs_file( @@ -368,14 +374,15 @@ def test_inspect_gcs_image_file(bucket, topic_id, subscription_id, capsys): topic_id, subscription_id, ["EMAIL_ADDRESS", "PHONE_NUMBER"], - timeout=1) + timeout=TIMEOUT) out, _ = capsys.readouterr() - assert "Inspection operation started" in out + assert "Info type: EMAIL_ADDRESS" in out finally: cancel_operation(out) +@pytest.mark.flaky(max_runs=2, min_passes=1) def test_inspect_gcs_multiple_files(bucket, topic_id, subscription_id, capsys): try: inspect_content.inspect_gcs_file( @@ -385,15 +392,16 @@ def test_inspect_gcs_multiple_files(bucket, topic_id, subscription_id, capsys): topic_id, subscription_id, ["EMAIL_ADDRESS", "PHONE_NUMBER"], - timeout=1) + timeout=TIMEOUT) out, _ = capsys.readouterr() - assert "Inspection operation started" in out + assert "Info type: EMAIL_ADDRESS" in out finally: cancel_operation(out) +@pytest.mark.flaky(max_runs=2, min_passes=1) def test_inspect_datastore( datastore_project, topic_id, subscription_id, capsys): try: @@ -404,14 +412,15 @@ def test_inspect_datastore( topic_id, subscription_id, ["FIRST_NAME", "EMAIL_ADDRESS", "PHONE_NUMBER"], - timeout=1) + timeout=TIMEOUT) out, _ = capsys.readouterr() - assert "Inspection operation started" in out + assert "Info type: EMAIL_ADDRESS" in out finally: cancel_operation(out) +@pytest.mark.flaky(max_runs=2, min_passes=1) def test_inspect_datastore_no_results( datastore_project, topic_id, subscription_id, capsys): try: @@ -422,10 +431,10 @@ def test_inspect_datastore_no_results( topic_id, subscription_id, ["PHONE_NUMBER"], - timeout=1) + timeout=TIMEOUT) out, _ = capsys.readouterr() - assert "Inspection operation started" in out + assert "No findings" in out finally: cancel_operation(out) diff --git a/dlp/risk.py b/dlp/risk.py index a31dfb12c6ef..518f947eee6b 100644 --- a/dlp/risk.py +++ b/dlp/risk.py @@ -59,8 +59,9 @@ def numerical_risk_analysis( # Instantiate a client. dlp = google.cloud.dlp_v2.DlpServiceClient() - # Convert the project id into a full resource id. - parent = dlp.project_path(project) + # Convert the project id into full resource ids. + topic = google.cloud.pubsub.PublisherClient.topic_path(project, topic_id) + parent = dlp.location_path(project, 'global') # Location info of the BigQuery table. source_table = { @@ -70,7 +71,7 @@ def numerical_risk_analysis( } # Tell the API where to send a notification when the job is complete. - actions = [{"pub_sub": {"topic": "{}/topics/{}".format(parent, topic_id)}}] + actions = [{"pub_sub": {"topic": topic}}] # Configure risk analysis job # Give the name of the numeric column to compute risk metrics for @@ -169,8 +170,9 @@ def categorical_risk_analysis( # Instantiate a client. dlp = google.cloud.dlp_v2.DlpServiceClient() - # Convert the project id into a full resource id. - parent = dlp.project_path(project) + # Convert the project id into full resource ids. + topic = google.cloud.pubsub.PublisherClient.topic_path(project, topic_id) + parent = dlp.location_path(project, 'global') # Location info of the BigQuery table. source_table = { @@ -180,7 +182,7 @@ def categorical_risk_analysis( } # Tell the API where to send a notification when the job is complete. - actions = [{"pub_sub": {"topic": "{}/topics/{}".format(parent, topic_id)}}] + actions = [{"pub_sub": {"topic": topic}}] # Configure risk analysis job # Give the name of the numeric column to compute risk metrics for @@ -294,7 +296,8 @@ def get_values(obj): dlp = google.cloud.dlp_v2.DlpServiceClient() # Convert the project id into a full resource id. - parent = dlp.project_path(project) + topic = google.cloud.pubsub.PublisherClient.topic_path(project, topic_id) + parent = dlp.location_path(project, 'global') # Location info of the BigQuery table. source_table = { @@ -310,7 +313,7 @@ def map_fields(field): quasi_ids = map(map_fields, quasi_ids) # Tell the API where to send a notification when the job is complete. - actions = [{"pub_sub": {"topic": "{}/topics/{}".format(parent, topic_id)}}] + actions = [{"pub_sub": {"topic": topic}}] # Configure risk analysis job # Give the name of the numeric column to compute risk metrics for @@ -425,7 +428,8 @@ def get_values(obj): dlp = google.cloud.dlp_v2.DlpServiceClient() # Convert the project id into a full resource id. - parent = dlp.project_path(project) + topic = google.cloud.pubsub.PublisherClient.topic_path(project, topic_id) + parent = dlp.location_path(project, 'global') # Location info of the BigQuery table. source_table = { @@ -441,7 +445,7 @@ def map_fields(field): quasi_ids = map(map_fields, quasi_ids) # Tell the API where to send a notification when the job is complete. - actions = [{"pub_sub": {"topic": "{}/topics/{}".format(parent, topic_id)}}] + actions = [{"pub_sub": {"topic": topic}}] # Configure risk analysis job # Give the name of the numeric column to compute risk metrics for @@ -574,8 +578,9 @@ def get_values(obj): # Instantiate a client. dlp = google.cloud.dlp_v2.DlpServiceClient() - # Convert the project id into a full resource id. - parent = dlp.project_path(project) + # Convert the project id into full resource ids. + topic = google.cloud.pubsub.PublisherClient.topic_path(project, topic_id) + parent = dlp.location_path(project, 'global') # Location info of the BigQuery table. source_table = { @@ -598,7 +603,7 @@ def map_fields(quasi_id, info_type): quasi_ids = map(map_fields, quasi_ids, info_types) # Tell the API where to send a notification when the job is complete. - actions = [{"pub_sub": {"topic": "{}/topics/{}".format(parent, topic_id)}}] + actions = [{"pub_sub": {"topic": topic}}] # Configure risk analysis job # Give the name of the numeric column to compute risk metrics for diff --git a/dlp/risk_test.py b/dlp/risk_test.py index 0164cf3b8c0e..36f7f54a0951 100644 --- a/dlp/risk_test.py +++ b/dlp/risk_test.py @@ -13,7 +13,6 @@ # limitations under the License. import os -import time import uuid import google.cloud.bigquery @@ -37,14 +36,14 @@ BIGQUERY_TABLE_ID = "dlp_test_table" + UNIQUE_STRING BIGQUERY_HARMFUL_TABLE_ID = "harmful" + UNIQUE_STRING -TIMEOUT = 30 +TIMEOUT = 60 # 1 minutes # Create new custom topic/subscription # We observe sometimes all the tests in this file fail. In a # hypothesis where DLP service somehow loses the connection to the # topic, now we use function scope for Pub/Sub fixtures. -@pytest.fixture(scope="function") +@pytest.fixture(scope="module") def topic_id(): # Creates a pubsub topic, and tears it down. publisher = google.cloud.pubsub.PublisherClient() @@ -59,7 +58,7 @@ def topic_id(): publisher.delete_topic(topic_path) -@pytest.fixture(scope="function") +@pytest.fixture(scope="module") def subscription_id(topic_id): # Subscribes to a topic. subscriber = google.cloud.pubsub.SubscriberClient() @@ -166,22 +165,7 @@ def bigquery_project(): bigquery_client.delete_dataset(dataset_ref, delete_contents=True) -def delay(err, *args): - # 20 mins of delay. This sounds like too long a delay, but we - # occasionally observe consequtive time block where operations are - # slow which leads to the test failures. These situations tend to - # get self healed in 20 minutes or so, so I'm trying this strategy. - # - # There are 10 tests, so we don't want the retry delay happening - # for all the tests. When we exhaust the MAX_FLAKY_WAIT, we retry - # the test immediately. - wait_time = min(pytest.MAX_FLAKY_WAIT, 60*20) - pytest.MAX_FLAKY_WAIT -= wait_time - time.sleep(wait_time) - return True - - -@pytest.mark.flaky(max_runs=2, min_passes=1, rerun_filter=delay) +@pytest.mark.flaky(max_runs=2, min_passes=1) def test_numerical_risk_analysis( topic_id, subscription_id, bigquery_project, capsys ): @@ -200,7 +184,7 @@ def test_numerical_risk_analysis( assert "Value Range:" in out -@pytest.mark.flaky(max_runs=2, min_passes=1, rerun_filter=delay) +@pytest.mark.flaky(max_runs=2, min_passes=1) def test_categorical_risk_analysis_on_string_field( topic_id, subscription_id, bigquery_project, capsys ): @@ -219,7 +203,7 @@ def test_categorical_risk_analysis_on_string_field( assert "Most common value occurs" in out -@pytest.mark.flaky(max_runs=2, min_passes=1, rerun_filter=delay) +@pytest.mark.flaky(max_runs=2, min_passes=1) def test_categorical_risk_analysis_on_number_field( topic_id, subscription_id, bigquery_project, capsys ): @@ -238,7 +222,7 @@ def test_categorical_risk_analysis_on_number_field( assert "Most common value occurs" in out -@pytest.mark.flaky(max_runs=2, min_passes=1, rerun_filter=delay) +@pytest.mark.flaky(max_runs=2, min_passes=1) def test_k_anonymity_analysis_single_field( topic_id, subscription_id, bigquery_project, capsys ): @@ -258,7 +242,7 @@ def test_k_anonymity_analysis_single_field( assert "Class size:" in out -@pytest.mark.flaky(max_runs=2, min_passes=1, rerun_filter=delay) +@pytest.mark.flaky(max_runs=2, min_passes=1) def test_k_anonymity_analysis_multiple_fields( topic_id, subscription_id, bigquery_project, capsys ): @@ -278,7 +262,7 @@ def test_k_anonymity_analysis_multiple_fields( assert "Class size:" in out -@pytest.mark.flaky(max_runs=2, min_passes=1, rerun_filter=delay) +@pytest.mark.flaky(max_runs=2, min_passes=1) def test_l_diversity_analysis_single_field( topic_id, subscription_id, bigquery_project, capsys ): @@ -300,7 +284,7 @@ def test_l_diversity_analysis_single_field( assert "Sensitive value" in out -@pytest.mark.flaky(max_runs=2, min_passes=1, rerun_filter=delay) +@pytest.mark.flaky(max_runs=2, min_passes=1) def test_l_diversity_analysis_multiple_field( topic_id, subscription_id, bigquery_project, capsys ): @@ -322,7 +306,7 @@ def test_l_diversity_analysis_multiple_field( assert "Sensitive value" in out -@pytest.mark.flaky(max_runs=2, min_passes=1, rerun_filter=delay) +@pytest.mark.flaky(max_runs=2, min_passes=1) def test_k_map_estimate_analysis_single_field( topic_id, subscription_id, bigquery_project, capsys ): @@ -344,7 +328,7 @@ def test_k_map_estimate_analysis_single_field( assert "Values" in out -@pytest.mark.flaky(max_runs=2, min_passes=1, rerun_filter=delay) +@pytest.mark.flaky(max_runs=2, min_passes=1) def test_k_map_estimate_analysis_multiple_field( topic_id, subscription_id, bigquery_project, capsys ): @@ -366,7 +350,7 @@ def test_k_map_estimate_analysis_multiple_field( assert "Values" in out -@pytest.mark.flaky(max_runs=2, min_passes=1, rerun_filter=delay) +@pytest.mark.flaky(max_runs=2, min_passes=1) def test_k_map_estimate_analysis_quasi_ids_info_types_equal( topic_id, subscription_id, bigquery_project ): diff --git a/endpoints/bookstore-grpc-transcoding/requirements.txt b/endpoints/bookstore-grpc-transcoding/requirements.txt index b1f04322e7f5..1593de5a3698 100644 --- a/endpoints/bookstore-grpc-transcoding/requirements.txt +++ b/endpoints/bookstore-grpc-transcoding/requirements.txt @@ -1,5 +1,4 @@ grpcio==1.29.0 grpcio-tools==1.29.0 -google-auth==1.16.0 +google-auth==1.16.1 six==1.15.0 -googleapis-common-protos==1.51.0 diff --git a/endpoints/bookstore-grpc/requirements.txt b/endpoints/bookstore-grpc/requirements.txt index 5e59fe59ccbf..1593de5a3698 100644 --- a/endpoints/bookstore-grpc/requirements.txt +++ b/endpoints/bookstore-grpc/requirements.txt @@ -1,4 +1,4 @@ grpcio==1.29.0 grpcio-tools==1.29.0 -google-auth==1.16.0 +google-auth==1.16.1 six==1.15.0 diff --git a/endpoints/getting-started/clients/service_to_service_non_default/requirements.txt b/endpoints/getting-started/clients/service_to_service_non_default/requirements.txt index 0f61949724cb..130c1828654a 100644 --- a/endpoints/getting-started/clients/service_to_service_non_default/requirements.txt +++ b/endpoints/getting-started/clients/service_to_service_non_default/requirements.txt @@ -1,3 +1,3 @@ google-api-python-client==1.9.1 -google-auth==1.16.0 +google-auth==1.16.1 google-auth-httplib2==0.0.3 diff --git a/endpoints/getting-started/requirements.txt b/endpoints/getting-started/requirements.txt index 422db74eb2e5..d479821ea234 100644 --- a/endpoints/getting-started/requirements.txt +++ b/endpoints/getting-started/requirements.txt @@ -4,5 +4,5 @@ gunicorn==20.0.4 six==1.15.0 pyyaml==5.3.1 requests==2.23.0 -google-auth==1.16.0 +google-auth==1.16.1 google-auth-oauthlib==0.4.1 diff --git a/functions/billing/requirements.txt b/functions/billing/requirements.txt index cc7add2e812c..e926b00462f5 100644 --- a/functions/billing/requirements.txt +++ b/functions/billing/requirements.txt @@ -1,4 +1,4 @@ -slackclient==2.7.0 +slackclient==2.7.1 # [START functions_billing_limit_appengine_deps] google-api-python-client==1.9.1 # [START functions_billing_limit_appengine_deps] diff --git a/functions/composer/requirements.txt b/functions/composer/requirements.txt index f37f36ad4880..e0acaa9e4cab 100644 --- a/functions/composer/requirements.txt +++ b/functions/composer/requirements.txt @@ -1,2 +1,2 @@ requests_toolbelt==0.9.1 -google-auth==1.16.0 +google-auth==1.16.1 diff --git a/healthcare/api-client/datasets/requirements.txt b/healthcare/api-client/datasets/requirements.txt index bfea15e1b0b4..1a2ecd873d0e 100644 --- a/healthcare/api-client/datasets/requirements.txt +++ b/healthcare/api-client/datasets/requirements.txt @@ -1,4 +1,4 @@ google-api-python-client==1.9.1 google-auth-httplib2==0.0.3 -google-auth==1.16.0 +google-auth==1.16.1 google-cloud==0.34.0 diff --git a/healthcare/api-client/dicom/requirements.txt b/healthcare/api-client/dicom/requirements.txt index 5b869471f2cb..077729a89bc2 100644 --- a/healthcare/api-client/dicom/requirements.txt +++ b/healthcare/api-client/dicom/requirements.txt @@ -1,5 +1,5 @@ google-api-python-client==1.9.1 google-auth-httplib2==0.0.3 -google-auth==1.16.0 +google-auth==1.16.1 google-cloud-pubsub==1.5.0 requests==2.23.0 diff --git a/healthcare/api-client/fhir/requirements.txt b/healthcare/api-client/fhir/requirements.txt index c414eaae4042..645dc54d7b8d 100644 --- a/healthcare/api-client/fhir/requirements.txt +++ b/healthcare/api-client/fhir/requirements.txt @@ -1,6 +1,6 @@ google-api-python-client==1.9.1 google-auth-httplib2==0.0.3 -google-auth==1.16.0 +google-auth==1.16.1 google-cloud==0.34.0 google-cloud-storage==1.28.1 requests==2.23.0 diff --git a/healthcare/api-client/hl7v2/requirements.txt b/healthcare/api-client/hl7v2/requirements.txt index bfea15e1b0b4..1a2ecd873d0e 100644 --- a/healthcare/api-client/hl7v2/requirements.txt +++ b/healthcare/api-client/hl7v2/requirements.txt @@ -1,4 +1,4 @@ google-api-python-client==1.9.1 google-auth-httplib2==0.0.3 -google-auth==1.16.0 +google-auth==1.16.1 google-cloud==0.34.0 diff --git a/healthcare/api-client/v1/datasets/requirements.txt b/healthcare/api-client/v1/datasets/requirements.txt index bfea15e1b0b4..1a2ecd873d0e 100644 --- a/healthcare/api-client/v1/datasets/requirements.txt +++ b/healthcare/api-client/v1/datasets/requirements.txt @@ -1,4 +1,4 @@ google-api-python-client==1.9.1 google-auth-httplib2==0.0.3 -google-auth==1.16.0 +google-auth==1.16.1 google-cloud==0.34.0 diff --git a/healthcare/api-client/v1/dicom/requirements.txt b/healthcare/api-client/v1/dicom/requirements.txt index 5b869471f2cb..077729a89bc2 100644 --- a/healthcare/api-client/v1/dicom/requirements.txt +++ b/healthcare/api-client/v1/dicom/requirements.txt @@ -1,5 +1,5 @@ google-api-python-client==1.9.1 google-auth-httplib2==0.0.3 -google-auth==1.16.0 +google-auth==1.16.1 google-cloud-pubsub==1.5.0 requests==2.23.0 diff --git a/healthcare/api-client/v1/fhir/requirements.txt b/healthcare/api-client/v1/fhir/requirements.txt index 97c6bb6d85cc..f32eafa244b3 100644 --- a/healthcare/api-client/v1/fhir/requirements.txt +++ b/healthcare/api-client/v1/fhir/requirements.txt @@ -1,5 +1,5 @@ google-api-python-client==1.9.1 google-auth-httplib2==0.0.3 -google-auth==1.16.0 +google-auth==1.16.1 google-cloud==0.34.0 google-cloud-storage==1.28.1 diff --git a/healthcare/api-client/v1/hl7v2/requirements.txt b/healthcare/api-client/v1/hl7v2/requirements.txt index bfea15e1b0b4..1a2ecd873d0e 100644 --- a/healthcare/api-client/v1/hl7v2/requirements.txt +++ b/healthcare/api-client/v1/hl7v2/requirements.txt @@ -1,4 +1,4 @@ google-api-python-client==1.9.1 google-auth-httplib2==0.0.3 -google-auth==1.16.0 +google-auth==1.16.1 google-cloud==0.34.0 diff --git a/iam/api-client/requirements.txt b/iam/api-client/requirements.txt index 0f61949724cb..130c1828654a 100644 --- a/iam/api-client/requirements.txt +++ b/iam/api-client/requirements.txt @@ -1,3 +1,3 @@ google-api-python-client==1.9.1 -google-auth==1.16.0 +google-auth==1.16.1 google-auth-httplib2==0.0.3 diff --git a/iap/requirements.txt b/iap/requirements.txt index b8a8ea0fbf49..5f4b57c9bcdb 100644 --- a/iap/requirements.txt +++ b/iap/requirements.txt @@ -1,6 +1,6 @@ cryptography==2.9.2 flask==1.1.2 -google-auth==1.16.0 +google-auth==1.16.1 gunicorn==20.0.4 requests==2.23.0 requests_toolbelt==0.9.1 diff --git a/iot/api-client/end_to_end_example/requirements.txt b/iot/api-client/end_to_end_example/requirements.txt index a941aa8f8bf2..1140fe105365 100644 --- a/iot/api-client/end_to_end_example/requirements.txt +++ b/iot/api-client/end_to_end_example/requirements.txt @@ -1,7 +1,7 @@ cryptography==2.9.2 google-api-python-client==1.9.1 google-auth-httplib2==0.0.3 -google-auth==1.16.0 +google-auth==1.16.1 google-cloud-pubsub==1.5.0 pyjwt==1.7.1 paho-mqtt==1.5.0 diff --git a/iot/api-client/gcs_file_to_device/requirements.txt b/iot/api-client/gcs_file_to_device/requirements.txt index 026895ed1339..0a3ab127dd13 100644 --- a/iot/api-client/gcs_file_to_device/requirements.txt +++ b/iot/api-client/gcs_file_to_device/requirements.txt @@ -1,5 +1,5 @@ google-api-python-client==1.9.1 -google-auth==1.16.0 +google-auth==1.16.1 google-auth-httplib2==0.0.3 google-cloud-iot==1.0.0 google-cloud-pubsub==1.5.0 diff --git a/iot/api-client/http_example/requirements.txt b/iot/api-client/http_example/requirements.txt index 4af81dba7228..9c7936b44029 100644 --- a/iot/api-client/http_example/requirements.txt +++ b/iot/api-client/http_example/requirements.txt @@ -1,7 +1,7 @@ cryptography==2.9.2 google-api-python-client==1.9.1 google-auth-httplib2==0.0.3 -google-auth==1.16.0 +google-auth==1.16.1 google-cloud-iot==1.0.0 google-cloud-pubsub==1.5.0 pyjwt==1.7.1 diff --git a/iot/api-client/manager/requirements.txt b/iot/api-client/manager/requirements.txt index 9e7afcd8db07..12ee2123420d 100644 --- a/iot/api-client/manager/requirements.txt +++ b/iot/api-client/manager/requirements.txt @@ -2,7 +2,7 @@ cryptography==2.9.2 flaky==3.6.1 google-api-python-client==1.9.1 google-auth-httplib2==0.0.3 -google-auth==1.16.0 +google-auth==1.16.1 google-cloud-iot==1.0.0 google-cloud-pubsub==1.5.0 paho-mqtt==1.5.0 diff --git a/iot/api-client/mqtt_example/requirements.txt b/iot/api-client/mqtt_example/requirements.txt index 002868484f64..f63596d94a25 100644 --- a/iot/api-client/mqtt_example/requirements.txt +++ b/iot/api-client/mqtt_example/requirements.txt @@ -1,7 +1,7 @@ cryptography==2.9.2 google-api-python-client==1.9.1 google-auth-httplib2==0.0.3 -google-auth==1.16.0 +google-auth==1.16.1 google-cloud-pubsub==1.5.0 google-cloud-iot==1.0.0 grpc-google-iam-v1==0.12.3 diff --git a/jobs/v3/api_client/requirements.txt b/jobs/v3/api_client/requirements.txt index 0f61949724cb..130c1828654a 100755 --- a/jobs/v3/api_client/requirements.txt +++ b/jobs/v3/api_client/requirements.txt @@ -1,3 +1,3 @@ google-api-python-client==1.9.1 -google-auth==1.16.0 +google-auth==1.16.1 google-auth-httplib2==0.0.3 diff --git a/kubernetes_engine/api-client/requirements.txt b/kubernetes_engine/api-client/requirements.txt index 0f61949724cb..130c1828654a 100644 --- a/kubernetes_engine/api-client/requirements.txt +++ b/kubernetes_engine/api-client/requirements.txt @@ -1,3 +1,3 @@ google-api-python-client==1.9.1 -google-auth==1.16.0 +google-auth==1.16.1 google-auth-httplib2==0.0.3 diff --git a/language/api/requirements.txt b/language/api/requirements.txt index 0f61949724cb..130c1828654a 100644 --- a/language/api/requirements.txt +++ b/language/api/requirements.txt @@ -1,3 +1,3 @@ google-api-python-client==1.9.1 -google-auth==1.16.0 +google-auth==1.16.1 google-auth-httplib2==0.0.3 diff --git a/language/classify_text/requirements.txt b/language/classify_text/requirements.txt index 000eb4bcf2b4..787df584e2e7 100644 --- a/language/classify_text/requirements.txt +++ b/language/classify_text/requirements.txt @@ -1,2 +1,2 @@ google-cloud-language==1.3.0 -numpy==1.18.4 +numpy==1.18.5 diff --git a/ml_engine/online_prediction/requirements.txt b/ml_engine/online_prediction/requirements.txt index 2ba8ac3f8d4a..da49bc1ce985 100644 --- a/ml_engine/online_prediction/requirements.txt +++ b/ml_engine/online_prediction/requirements.txt @@ -1,5 +1,5 @@ tensorflow==2.0.1; python_version > "3.0" tensorflow==1.15.0; python_version < "3.0" google-api-python-client==1.9.1 -google-auth==1.16.0 +google-auth==1.16.1 google-auth-httplib2==0.0.3 diff --git a/monitoring/api/v3/api-client/requirements.txt b/monitoring/api/v3/api-client/requirements.txt index 0f61949724cb..130c1828654a 100644 --- a/monitoring/api/v3/api-client/requirements.txt +++ b/monitoring/api/v3/api-client/requirements.txt @@ -1,3 +1,3 @@ google-api-python-client==1.9.1 -google-auth==1.16.0 +google-auth==1.16.1 google-auth-httplib2==0.0.3 diff --git a/pubsub/cloud-client/iam.py b/pubsub/cloud-client/iam.py index eb0c8246307b..71c55d764c0c 100644 --- a/pubsub/cloud-client/iam.py +++ b/pubsub/cloud-client/iam.py @@ -23,14 +23,18 @@ import argparse -from google.cloud import pubsub_v1 - -def get_topic_policy(project, topic_name): +def get_topic_policy(project, topic_id): """Prints the IAM policy for the given topic.""" # [START pubsub_get_topic_policy] + from google.cloud import pubsub_v1 + + # TODO(developer) + # project_id = "your-project-id" + # topic_id = "your-topic-id" + client = pubsub_v1.PublisherClient() - topic_path = client.topic_path(project, topic_name) + topic_path = client.topic_path(project, topic_id) policy = client.get_iam_policy(topic_path) @@ -40,11 +44,17 @@ def get_topic_policy(project, topic_name): # [END pubsub_get_topic_policy] -def get_subscription_policy(project, subscription_name): +def get_subscription_policy(project, subscription_id): """Prints the IAM policy for the given subscription.""" # [START pubsub_get_subscription_policy] + from google.cloud import pubsub_v1 + + # TODO(developer) + # project_id = "your-project-id" + # subscription_id = "your-subscription-id" + client = pubsub_v1.SubscriberClient() - subscription_path = client.subscription_path(project, subscription_name) + subscription_path = client.subscription_path(project, subscription_id) policy = client.get_iam_policy(subscription_path) @@ -56,11 +66,17 @@ def get_subscription_policy(project, subscription_name): # [END pubsub_get_subscription_policy] -def set_topic_policy(project, topic_name): +def set_topic_policy(project, topic_id): """Sets the IAM policy for a topic.""" # [START pubsub_set_topic_policy] + from google.cloud import pubsub_v1 + + # TODO(developer) + # project_id = "your-project-id" + # topic_id = "your-topic-id" + client = pubsub_v1.PublisherClient() - topic_path = client.topic_path(project, topic_name) + topic_path = client.topic_path(project, topic_id) policy = client.get_iam_policy(topic_path) @@ -75,15 +91,21 @@ def set_topic_policy(project, topic_name): # Set the policy policy = client.set_iam_policy(topic_path, policy) - print("IAM policy for topic {} set: {}".format(topic_name, policy)) + print("IAM policy for topic {} set: {}".format(topic_id, policy)) # [END pubsub_set_topic_policy] -def set_subscription_policy(project, subscription_name): +def set_subscription_policy(project, subscription_id): """Sets the IAM policy for a topic.""" # [START pubsub_set_subscription_policy] + from google.cloud import pubsub_v1 + + # TODO(developer) + # project_id = "your-project-id" + # subscription_id = "your-subscription-id" + client = pubsub_v1.SubscriberClient() - subscription_path = client.subscription_path(project, subscription_name) + subscription_path = client.subscription_path(project, subscription_id) policy = client.get_iam_policy(subscription_path) @@ -91,48 +113,50 @@ def set_subscription_policy(project, subscription_name): policy.bindings.add(role="roles/pubsub.viewer", members=["allUsers"]) # Add a group as an editor. - policy.bindings.add( - role="roles/editor", members=["group:cloud-logs@google.com"] - ) + policy.bindings.add(role="roles/editor", members=["group:cloud-logs@google.com"]) # Set the policy policy = client.set_iam_policy(subscription_path, policy) - print( - "IAM policy for subscription {} set: {}".format( - subscription_name, policy - ) - ) + print("IAM policy for subscription {} set: {}".format(subscription_id, policy)) client.close() # [END pubsub_set_subscription_policy] -def check_topic_permissions(project, topic_name): +def check_topic_permissions(project, topic_id): """Checks to which permissions are available on the given topic.""" # [START pubsub_test_topic_permissions] + from google.cloud import pubsub_v1 + + # TODO(developer) + # project_id = "your-project-id" + # topic_id = "your-topic-id" + client = pubsub_v1.PublisherClient() - topic_path = client.topic_path(project, topic_name) + topic_path = client.topic_path(project, topic_id) permissions_to_check = ["pubsub.topics.publish", "pubsub.topics.update"] - allowed_permissions = client.test_iam_permissions( - topic_path, permissions_to_check - ) + allowed_permissions = client.test_iam_permissions(topic_path, permissions_to_check) print( - "Allowed permissions for topic {}: {}".format( - topic_path, allowed_permissions - ) + "Allowed permissions for topic {}: {}".format(topic_path, allowed_permissions) ) # [END pubsub_test_topic_permissions] -def check_subscription_permissions(project, subscription_name): +def check_subscription_permissions(project, subscription_id): """Checks to which permissions are available on the given subscription.""" # [START pubsub_test_subscription_permissions] + from google.cloud import pubsub_v1 + + # TODO(developer) + # project_id = "your-project-id" + # subscription_id = "your-subscription-id" + client = pubsub_v1.SubscriberClient() - subscription_path = client.subscription_path(project, subscription_name) + subscription_path = client.subscription_path(project, subscription_id) permissions_to_check = [ "pubsub.subscriptions.consume", @@ -155,8 +179,7 @@ def check_subscription_permissions(project, subscription_name): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("project", help="Your Google Cloud project ID") @@ -165,45 +188,44 @@ def check_subscription_permissions(project, subscription_name): get_topic_policy_parser = subparsers.add_parser( "get-topic-policy", help=get_topic_policy.__doc__ ) - get_topic_policy_parser.add_argument("topic_name") + get_topic_policy_parser.add_argument("topic_id") get_subscription_policy_parser = subparsers.add_parser( "get-subscription-policy", help=get_subscription_policy.__doc__ ) - get_subscription_policy_parser.add_argument("subscription_name") + get_subscription_policy_parser.add_argument("subscription_id") set_topic_policy_parser = subparsers.add_parser( "set-topic-policy", help=set_topic_policy.__doc__ ) - set_topic_policy_parser.add_argument("topic_name") + set_topic_policy_parser.add_argument("topic_id") set_subscription_policy_parser = subparsers.add_parser( "set-subscription-policy", help=set_subscription_policy.__doc__ ) - set_subscription_policy_parser.add_argument("subscription_name") + set_subscription_policy_parser.add_argument("subscription_id") check_topic_permissions_parser = subparsers.add_parser( "check-topic-permissions", help=check_topic_permissions.__doc__ ) - check_topic_permissions_parser.add_argument("topic_name") + check_topic_permissions_parser.add_argument("topic_id") check_subscription_permissions_parser = subparsers.add_parser( - "check-subscription-permissions", - help=check_subscription_permissions.__doc__, + "check-subscription-permissions", help=check_subscription_permissions.__doc__, ) - check_subscription_permissions_parser.add_argument("subscription_name") + check_subscription_permissions_parser.add_argument("subscription_id") args = parser.parse_args() if args.command == "get-topic-policy": - get_topic_policy(args.project, args.topic_name) + get_topic_policy(args.project, args.topic_id) elif args.command == "get-subscription-policy": - get_subscription_policy(args.project, args.subscription_name) + get_subscription_policy(args.project, args.subscription_id) elif args.command == "set-topic-policy": - set_topic_policy(args.project, args.topic_name) + set_topic_policy(args.project, args.topic_id) elif args.command == "set-subscription-policy": - set_subscription_policy(args.project, args.subscription_name) + set_subscription_policy(args.project, args.subscription_id) elif args.command == "check-topic-permissions": - check_topic_permissions(args.project, args.topic_name) + check_topic_permissions(args.project, args.topic_id) elif args.command == "check-subscription-permissions": - check_subscription_permissions(args.project, args.subscription_name) + check_subscription_permissions(args.project, args.subscription_id) diff --git a/pubsub/cloud-client/iam_test.py b/pubsub/cloud-client/iam_test.py index f88cde851e7d..31764a056e8d 100644 --- a/pubsub/cloud-client/iam_test.py +++ b/pubsub/cloud-client/iam_test.py @@ -56,9 +56,7 @@ def subscriber_client(): @pytest.fixture def subscription(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION - ) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION) try: subscriber_client.delete_subscription(subscription_path) diff --git a/pubsub/cloud-client/publisher.py b/pubsub/cloud-client/publisher.py index 9e7820fbf305..477b31b9cf71 100644 --- a/pubsub/cloud-client/publisher.py +++ b/pubsub/cloud-client/publisher.py @@ -40,7 +40,7 @@ def list_topics(project_id): # [END pubsub_list_topics] -def create_topic(project_id, topic_name): +def create_topic(project_id, topic_id): """Create a new Pub/Sub topic.""" # [START pubsub_quickstart_create_topic] # [START pubsub_create_topic] @@ -48,10 +48,10 @@ def create_topic(project_id, topic_name): # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" + # topic_id = "your-topic-id" publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project_id, topic_name) + topic_path = publisher.topic_path(project_id, topic_id) topic = publisher.create_topic(topic_path) @@ -60,17 +60,17 @@ def create_topic(project_id, topic_name): # [END pubsub_create_topic] -def delete_topic(project_id, topic_name): +def delete_topic(project_id, topic_id): """Deletes an existing Pub/Sub topic.""" # [START pubsub_delete_topic] from google.cloud import pubsub_v1 # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" + # topic_id = "your-topic-id" publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project_id, topic_name) + topic_path = publisher.topic_path(project_id, topic_id) publisher.delete_topic(topic_path) @@ -78,7 +78,7 @@ def delete_topic(project_id, topic_name): # [END pubsub_delete_topic] -def publish_messages(project_id, topic_name): +def publish_messages(project_id, topic_id): """Publishes multiple messages to a Pub/Sub topic.""" # [START pubsub_quickstart_publisher] # [START pubsub_publish] @@ -86,12 +86,12 @@ def publish_messages(project_id, topic_name): # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" + # topic_id = "your-topic-id" publisher = pubsub_v1.PublisherClient() # The `topic_path` method creates a fully qualified identifier - # in the form `projects/{project_id}/topics/{topic_name}` - topic_path = publisher.topic_path(project_id, topic_name) + # in the form `projects/{project_id}/topics/{topic_id}` + topic_path = publisher.topic_path(project_id, topic_id) for n in range(1, 10): data = u"Message number {}".format(n) @@ -106,7 +106,7 @@ def publish_messages(project_id, topic_name): # [END pubsub_publish] -def publish_messages_with_custom_attributes(project_id, topic_name): +def publish_messages_with_custom_attributes(project_id, topic_id): """Publishes multiple messages with custom attributes to a Pub/Sub topic.""" # [START pubsub_publish_custom_attributes] @@ -114,10 +114,10 @@ def publish_messages_with_custom_attributes(project_id, topic_name): # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" + # topic_id = "your-topic-id" publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project_id, topic_name) + topic_path = publisher.topic_path(project_id, topic_id) for n in range(1, 10): data = u"Message number {}".format(n) @@ -133,7 +133,7 @@ def publish_messages_with_custom_attributes(project_id, topic_name): # [END pubsub_publish_custom_attributes] -def publish_messages_with_error_handler(project_id, topic_name): +def publish_messages_with_error_handler(project_id, topic_id): # [START pubsub_publish_messages_error_handler] """Publishes multiple messages to a Pub/Sub topic with an error handler.""" import time @@ -142,10 +142,10 @@ def publish_messages_with_error_handler(project_id, topic_name): # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" + # topic_id = "your-topic-id" publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project_id, topic_name) + topic_path = publisher.topic_path(project_id, topic_id) futures = dict() @@ -178,14 +178,14 @@ def callback(f): # [END pubsub_publish_messages_error_handler] -def publish_messages_with_batch_settings(project_id, topic_name): +def publish_messages_with_batch_settings(project_id, topic_id): """Publishes multiple messages to a Pub/Sub topic with batch settings.""" # [START pubsub_publisher_batch_settings] from google.cloud import pubsub_v1 # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" + # topic_id = "your-topic-id" # Configure the batch to publish as soon as there is ten messages, # one kilobyte of data, or one second has passed. @@ -195,7 +195,7 @@ def publish_messages_with_batch_settings(project_id, topic_name): max_latency=1, # default 10 ms ) publisher = pubsub_v1.PublisherClient(batch_settings) - topic_path = publisher.topic_path(project_id, topic_name) + topic_path = publisher.topic_path(project_id, topic_id) # Resolve the publish future in a separate thread. def callback(future): @@ -214,14 +214,14 @@ def callback(future): # [END pubsub_publisher_batch_settings] -def publish_messages_with_retry_settings(project_id, topic_name): +def publish_messages_with_retry_settings(project_id, topic_id): """Publishes messages with custom retry settings.""" # [START pubsub_publisher_retry_settings] from google.cloud import pubsub_v1 # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" + # topic_id = "your-topic-id" # Configure the retry settings. Defaults will be overwritten. retry_settings = { @@ -260,7 +260,7 @@ def publish_messages_with_retry_settings(project_id, topic_name): } publisher = pubsub_v1.PublisherClient(client_config=retry_settings) - topic_path = publisher.topic_path(project_id, topic_name) + topic_path = publisher.topic_path(project_id, topic_id) for n in range(1, 10): data = u"Message number {}".format(n) @@ -283,52 +283,52 @@ def publish_messages_with_retry_settings(project_id, topic_name): subparsers.add_parser("list", help=list_topics.__doc__) create_parser = subparsers.add_parser("create", help=create_topic.__doc__) - create_parser.add_argument("topic_name") + create_parser.add_argument("topic_id") delete_parser = subparsers.add_parser("delete", help=delete_topic.__doc__) - delete_parser.add_argument("topic_name") + delete_parser.add_argument("topic_id") publish_parser = subparsers.add_parser("publish", help=publish_messages.__doc__) - publish_parser.add_argument("topic_name") + publish_parser.add_argument("topic_id") publish_with_custom_attributes_parser = subparsers.add_parser( "publish-with-custom-attributes", help=publish_messages_with_custom_attributes.__doc__, ) - publish_with_custom_attributes_parser.add_argument("topic_name") + publish_with_custom_attributes_parser.add_argument("topic_id") publish_with_error_handler_parser = subparsers.add_parser( "publish-with-error-handler", help=publish_messages_with_error_handler.__doc__, ) - publish_with_error_handler_parser.add_argument("topic_name") + publish_with_error_handler_parser.add_argument("topic_id") publish_with_batch_settings_parser = subparsers.add_parser( "publish-with-batch-settings", help=publish_messages_with_batch_settings.__doc__, ) - publish_with_batch_settings_parser.add_argument("topic_name") + publish_with_batch_settings_parser.add_argument("topic_id") publish_with_retry_settings_parser = subparsers.add_parser( "publish-with-retry-settings", help=publish_messages_with_retry_settings.__doc__, ) - publish_with_retry_settings_parser.add_argument("topic_name") + publish_with_retry_settings_parser.add_argument("topic_id") args = parser.parse_args() if args.command == "list": list_topics(args.project_id) elif args.command == "create": - create_topic(args.project_id, args.topic_name) + create_topic(args.project_id, args.topic_id) elif args.command == "delete": - delete_topic(args.project_id, args.topic_name) + delete_topic(args.project_id, args.topic_id) elif args.command == "publish": - publish_messages(args.project_id, args.topic_name) + publish_messages(args.project_id, args.topic_id) elif args.command == "publish-with-custom-attributes": - publish_messages_with_custom_attributes(args.project_id, args.topic_name) + publish_messages_with_custom_attributes(args.project_id, args.topic_id) elif args.command == "publish-with-error-handler": - publish_messages_with_error_handler(args.project_id, args.topic_name) + publish_messages_with_error_handler(args.project_id, args.topic_id) elif args.command == "publish-with-batch-settings": - publish_messages_with_batch_settings(args.project_id, args.topic_name) + publish_messages_with_batch_settings(args.project_id, args.topic_id) elif args.command == "publish-with-retry-settings": - publish_messages_with_retry_settings(args.project_id, args.topic_name) + publish_messages_with_retry_settings(args.project_id, args.topic_id) diff --git a/pubsub/cloud-client/quickstart/pub.py b/pubsub/cloud-client/quickstart/pub.py index a3f8087ecd15..16432c0c3627 100644 --- a/pubsub/cloud-client/quickstart/pub.py +++ b/pubsub/cloud-client/quickstart/pub.py @@ -46,15 +46,15 @@ def callback(api_future): return callback -def pub(project_id, topic_name): +def pub(project_id, topic_id): """Publishes a message to a Pub/Sub topic.""" # [START pubsub_quickstart_pub_client] # Initialize a Publisher client. client = pubsub_v1.PublisherClient() # [END pubsub_quickstart_pub_client] # Create a fully qualified identifier in the form of - # `projects/{project_id}/topics/{topic_name}` - topic_path = client.topic_path(project_id, topic_name) + # `projects/{project_id}/topics/{topic_id}` + topic_path = client.topic_path(project_id, topic_id) # Data sent to Cloud Pub/Sub must be a bytestring. data = b"Hello, World!" @@ -75,13 +75,12 @@ def pub(project_id, topic_name): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("project_id", help="Google Cloud project ID") - parser.add_argument("topic_name", help="Pub/Sub topic name") + parser.add_argument("topic_id", help="Pub/Sub topic ID") args = parser.parse_args() - pub(args.project_id, args.topic_name) + pub(args.project_id, args.topic_id) # [END pubsub_quickstart_pub_all] diff --git a/pubsub/cloud-client/quickstart/sub.py b/pubsub/cloud-client/quickstart/sub.py index 1d90726f5b04..efe00891593e 100644 --- a/pubsub/cloud-client/quickstart/sub.py +++ b/pubsub/cloud-client/quickstart/sub.py @@ -23,23 +23,19 @@ # [END pubsub_quickstart_sub_deps] -def sub(project_id, subscription_name): +def sub(project_id, subscription_id): """Receives messages from a Pub/Sub subscription.""" # [START pubsub_quickstart_sub_client] # Initialize a Subscriber client subscriber_client = pubsub_v1.SubscriberClient() # [END pubsub_quickstart_sub_client] # Create a fully qualified identifier in the form of - # `projects/{project_id}/subscriptions/{subscription_name}` - subscription_path = subscriber_client.subscription_path( - project_id, subscription_name - ) + # `projects/{project_id}/subscriptions/{subscription_id}` + subscription_path = subscriber_client.subscription_path(project_id, subscription_id) def callback(message): print( - "Received message {} of message ID {}\n".format( - message, message.message_id - ) + "Received message {} of message ID {}\n".format(message, message.message_id) ) # Acknowledge the message. Unack'ed messages will be redelivered. message.ack() @@ -62,13 +58,12 @@ def callback(message): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("project_id", help="Google Cloud project ID") - parser.add_argument("subscription_name", help="Pub/Sub subscription name") + parser.add_argument("subscription_id", help="Pub/Sub subscription ID") args = parser.parse_args() - sub(args.project_id, args.subscription_name) + sub(args.project_id, args.subscription_id) # [END pubsub_quickstart_sub_all] diff --git a/pubsub/cloud-client/quickstart/sub_test.py b/pubsub/cloud-client/quickstart/sub_test.py index 2754dc56b5e5..65d5fa111a05 100644 --- a/pubsub/cloud-client/quickstart/sub_test.py +++ b/pubsub/cloud-client/quickstart/sub_test.py @@ -48,9 +48,7 @@ def topic_path(): @pytest.fixture(scope="module") def subscription_path(topic_path): - subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION - ) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION) try: subscription = subscriber_client.create_subscription( @@ -82,9 +80,7 @@ def test_sub(monkeypatch, topic_path, subscription_path, capsys): monkeypatch.setattr(pubsub_v1, "SubscriberClient", mock_client_constructor) def mock_subscribe(subscription_path, callback=None): - real_future = real_client.subscribe( - subscription_path, callback=callback - ) + real_future = real_client.subscribe(subscription_path, callback=callback) mock_future = mock.Mock(spec=real_future, wraps=real_future) def mock_result(): diff --git a/pubsub/cloud-client/subscriber.py b/pubsub/cloud-client/subscriber.py index 5dc468e23a80..677bfe359fd1 100644 --- a/pubsub/cloud-client/subscriber.py +++ b/pubsub/cloud-client/subscriber.py @@ -24,17 +24,17 @@ import argparse -def list_subscriptions_in_topic(project_id, topic_name): +def list_subscriptions_in_topic(project_id, topic_id): """Lists all subscriptions for a given topic.""" # [START pubsub_list_topic_subscriptions] from google.cloud import pubsub_v1 # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" + # topic_id = "your-topic-id" publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project_id, topic_name) + topic_path = publisher.topic_path(project_id, topic_id) for subscription in publisher.list_topic_subscriptions(topic_path): print(subscription) @@ -59,19 +59,19 @@ def list_subscriptions_in_project(project_id): # [END pubsub_list_subscriptions] -def create_subscription(project_id, topic_name, subscription_name): +def create_subscription(project_id, topic_id, subscription_id): """Create a new pull subscription on the given topic.""" # [START pubsub_create_pull_subscription] from google.cloud import pubsub_v1 # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" - # subscription_name = "your-subscription-id" + # topic_id = "your-topic-id" + # subscription_id = "your-subscription-id" subscriber = pubsub_v1.SubscriberClient() - topic_path = subscriber.topic_path(project_id, topic_name) - subscription_path = subscriber.subscription_path(project_id, subscription_name) + topic_path = subscriber.topic_path(project_id, topic_id) + subscription_path = subscriber.subscription_path(project_id, subscription_id) subscription = subscriber.create_subscription(subscription_path, topic_path) @@ -82,7 +82,7 @@ def create_subscription(project_id, topic_name, subscription_name): def create_subscription_with_dead_letter_topic( - project_id, topic_name, subscription_name, dead_letter_topic_name + project_id, topic_id, subscription_id, dead_letter_topic_id ): """Create a subscription with dead letter policy.""" # [START pubsub_dead_letter_create_subscription] @@ -94,17 +94,17 @@ def create_subscription_with_dead_letter_topic( # endpoint = "https://my-test-project.appspot.com/push" # TODO(developer): This is an existing topic that the subscription # with dead letter policy is attached to. - # topic_name = "your-topic-id" + # topic_id = "your-topic-id" # TODO(developer): This is an existing subscription with a dead letter policy. - # subscription_name = "your-subscription-id" + # subscription_id = "your-subscription-id" # TODO(developer): This is an existing dead letter topic that the subscription # with dead letter policy will forward dead letter messages to. - # dead_letter_topic_name = "your-dead-letter-topic-id" + # dead_letter_topic_id = "your-dead-letter-topic-id" subscriber = pubsub_v1.SubscriberClient() - topic_path = subscriber.topic_path(project_id, topic_name) - subscription_path = subscriber.subscription_path(project_id, subscription_name) - dead_letter_topic_path = subscriber.topic_path(project_id, dead_letter_topic_name) + topic_path = subscriber.topic_path(project_id, topic_id) + subscription_path = subscriber.subscription_path(project_id, subscription_id) + dead_letter_topic_path = subscriber.topic_path(project_id, dead_letter_topic_id) dead_letter_policy = DeadLetterPolicy( dead_letter_topic=dead_letter_topic_path, max_delivery_attempts=10 @@ -129,20 +129,20 @@ def create_subscription_with_dead_letter_topic( # [END pubsub_dead_letter_create_subscription] -def create_push_subscription(project_id, topic_name, subscription_name, endpoint): +def create_push_subscription(project_id, topic_id, subscription_id, endpoint): """Create a new push subscription on the given topic.""" # [START pubsub_create_push_subscription] from google.cloud import pubsub_v1 # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" - # subscription_name = "your-subscription-id" + # topic_id = "your-topic-id" + # subscription_id = "your-subscription-id" # endpoint = "https://my-test-project.appspot.com/push" subscriber = pubsub_v1.SubscriberClient() - topic_path = subscriber.topic_path(project_id, topic_name) - subscription_path = subscriber.subscription_path(project_id, subscription_name) + topic_path = subscriber.topic_path(project_id, topic_id) + subscription_path = subscriber.subscription_path(project_id, subscription_id) push_config = pubsub_v1.types.PushConfig(push_endpoint=endpoint) @@ -157,17 +157,17 @@ def create_push_subscription(project_id, topic_name, subscription_name, endpoint # [END pubsub_create_push_subscription] -def delete_subscription(project_id, subscription_name): +def delete_subscription(project_id, subscription_id): """Deletes an existing Pub/Sub topic.""" # [START pubsub_delete_subscription] from google.cloud import pubsub_v1 # TODO(developer) # project_id = "your-project-id" - # subscription_name = "your-subscription-id" + # subscription_id = "your-subscription-id" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_id) subscriber.delete_subscription(subscription_path) @@ -177,7 +177,7 @@ def delete_subscription(project_id, subscription_name): # [END pubsub_delete_subscription] -def update_push_subscription(project_id, topic_name, subscription_name, endpoint): +def update_push_subscription(project_id, topic_id, subscription_id, endpoint): """ Updates an existing Pub/Sub subscription's push endpoint URL. Note that certain properties of a subscription, such as @@ -188,17 +188,17 @@ def update_push_subscription(project_id, topic_name, subscription_name, endpoint # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" - # subscription_name = "your-subscription-id" + # topic_id = "your-topic-id" + # subscription_id = "your-subscription-id" # endpoint = "https://my-test-project.appspot.com/push" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_id) push_config = pubsub_v1.types.PushConfig(push_endpoint=endpoint) subscription = pubsub_v1.types.Subscription( - name=subscription_path, topic=topic_name, push_config=push_config + name=subscription_path, topic=topic_id, push_config=push_config ) update_mask = {"paths": {"push_config"}} @@ -213,7 +213,7 @@ def update_push_subscription(project_id, topic_name, subscription_name, endpoint def update_subscription_with_dead_letter_policy( - project_id, topic_name, subscription_name, dead_letter_topic_name + project_id, topic_id, subscription_id, dead_letter_topic_id ): """Update a subscription's dead letter policy.""" # [START pubsub_dead_letter_update_subscription] @@ -224,17 +224,17 @@ def update_subscription_with_dead_letter_policy( # project_id = "your-project-id" # TODO(developer): This is an existing topic that the subscription # with dead letter policy is attached to. - # topic_name = "your-topic-name" + # topic_id = "your-topic-id" # TODO(developer): This is an existing subscription with a dead letter policy. - # subscription_name = "your-subscription-id" + # subscription_id = "your-subscription-id" # TODO(developer): This is an existing dead letter topic that the subscription # with dead letter policy will forward dead letter messages to. - # dead_letter_topic_name = "your-dead-letter-topic-id" + # dead_letter_topic_id = "your-dead-letter-topic-id" subscriber = pubsub_v1.SubscriberClient() - topic_path = subscriber.topic_path(project_id, topic_name) - subscription_path = subscriber.subscription_path(project_id, subscription_name) - dead_letter_topic_path = subscriber.topic_path(project_id, dead_letter_topic_name) + topic_path = subscriber.topic_path(project_id, topic_id) + subscription_path = subscriber.subscription_path(project_id, subscription_id) + dead_letter_topic_path = subscriber.topic_path(project_id, dead_letter_topic_id) subscription_before_update = subscriber.get_subscription(subscription_path) print("Before the update: {}".format(subscription_before_update)) @@ -264,7 +264,7 @@ def update_subscription_with_dead_letter_policy( return subscription_after_update -def remove_dead_letter_policy(project_id, topic_name, subscription_name): +def remove_dead_letter_policy(project_id, topic_id, subscription_id): """Remove dead letter policy from a subscription.""" # [START pubsub_dead_letter_remove] from google.cloud import pubsub_v1 @@ -274,13 +274,13 @@ def remove_dead_letter_policy(project_id, topic_name, subscription_name): # project_id = "your-project-id" # TODO(developer): This is an existing topic that the subscription # with dead letter policy is attached to. - # topic_name = "your-topic-name" + # topic_id = "your-topic-id" # TODO(developer): This is an existing subscription with a dead letter policy. - # subscription_name = "your-subscription-id" + # subscription_id = "your-subscription-id" subscriber = pubsub_v1.SubscriberClient() - topic_path = subscriber.topic_path(project_id, topic_name) - subscription_path = subscriber.subscription_path(project_id, subscription_name) + topic_path = subscriber.topic_path(project_id, topic_id) + subscription_path = subscriber.subscription_path(project_id, subscription_id) subscription_before_update = subscriber.get_subscription(subscription_path) print("Before removing the policy: {}".format(subscription_before_update)) @@ -309,7 +309,7 @@ def remove_dead_letter_policy(project_id, topic_name, subscription_name): return subscription_after_update -def receive_messages(project_id, subscription_name, timeout=None): +def receive_messages(project_id, subscription_id, timeout=None): """Receives messages from a pull subscription.""" # [START pubsub_subscriber_async_pull] # [START pubsub_quickstart_subscriber] @@ -318,14 +318,14 @@ def receive_messages(project_id, subscription_name, timeout=None): # TODO(developer) # project_id = "your-project-id" - # subscription_name = "your-subscription-id" + # subscription_id = "your-subscription-id" # Number of seconds the subscriber should listen for messages # timeout = 5.0 subscriber = pubsub_v1.SubscriberClient() # The `subscription_path` method creates a fully qualified identifier - # in the form `projects/{project_id}/subscriptions/{subscription_name}` - subscription_path = subscriber.subscription_path(project_id, subscription_name) + # in the form `projects/{project_id}/subscriptions/{subscription_id}` + subscription_path = subscriber.subscription_path(project_id, subscription_id) def callback(message): print("Received message: {}".format(message)) @@ -346,9 +346,7 @@ def callback(message): # [END pubsub_quickstart_subscriber] -def receive_messages_with_custom_attributes( - project_id, subscription_name, timeout=None -): +def receive_messages_with_custom_attributes(project_id, subscription_id, timeout=None): """Receives messages from a pull subscription.""" # [START pubsub_subscriber_sync_pull_custom_attributes] # [START pubsub_subscriber_async_pull_custom_attributes] @@ -357,12 +355,12 @@ def receive_messages_with_custom_attributes( # TODO(developer) # project_id = "your-project-id" - # subscription_name = "your-subscription-id" + # subscription_id = "your-subscription-id" # Number of seconds the subscriber should listen for messages # timeout = 5.0 subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_id) def callback(message): print("Received message: {}".format(message.data)) @@ -388,7 +386,7 @@ def callback(message): # [END pubsub_subscriber_sync_pull_custom_attributes] -def receive_messages_with_flow_control(project_id, subscription_name, timeout=None): +def receive_messages_with_flow_control(project_id, subscription_id, timeout=None): """Receives messages from a pull subscription with flow control.""" # [START pubsub_subscriber_flow_settings] from concurrent.futures import TimeoutError @@ -396,12 +394,12 @@ def receive_messages_with_flow_control(project_id, subscription_name, timeout=No # TODO(developer) # project_id = "your-project-id" - # subscription_name = "your-subscription-id" + # subscription_id = "your-subscription-id" # Number of seconds the subscriber should listen for messages # timeout = 5.0 subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_id) def callback(message): print("Received message: {}".format(message.data)) @@ -426,17 +424,17 @@ def callback(message): # [END pubsub_subscriber_flow_settings] -def synchronous_pull(project_id, subscription_name): +def synchronous_pull(project_id, subscription_id): """Pulling messages synchronously.""" # [START pubsub_subscriber_sync_pull] from google.cloud import pubsub_v1 # TODO(developer) # project_id = "your-project-id" - # subscription_name = "your-subscription-id" + # subscription_id = "your-subscription-id" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_id) NUM_MESSAGES = 3 @@ -461,7 +459,7 @@ def synchronous_pull(project_id, subscription_name): # [END pubsub_subscriber_sync_pull] -def synchronous_pull_with_lease_management(project_id, subscription_name): +def synchronous_pull_with_lease_management(project_id, subscription_id): """Pulling messages synchronously with lease management""" # [START pubsub_subscriber_sync_pull_with_lease] import logging @@ -473,10 +471,10 @@ def synchronous_pull_with_lease_management(project_id, subscription_name): # TODO(developer) # project_id = "your-project-id" - # subscription_name = "your-subscription-id" + # subscription_id = "your-subscription-id" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_id) NUM_MESSAGES = 2 ACK_DEADLINE = 30 @@ -547,19 +545,19 @@ def worker(msg): # [END pubsub_subscriber_sync_pull_with_lease] -def listen_for_errors(project_id, subscription_name, timeout=None): +def listen_for_errors(project_id, subscription_id, timeout=None): """Receives messages and catches errors from a pull subscription.""" # [START pubsub_subscriber_error_listener] from google.cloud import pubsub_v1 # TODO(developer) - # project_id = "Your Google Cloud Project ID" - # subscription_name = "Your Pubsub subscription name" + # project_id = "your-project-id" + # subscription_id = "your-subscription-id" # Number of seconds the subscriber should listen for messages # timeout = 5.0 subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_id) def callback(message): print("Received message: {}".format(message)) @@ -578,25 +576,23 @@ def callback(message): streaming_pull_future.cancel() print( "Listening for messages on {} threw an exception: {}.".format( - subscription_name, e + subscription_id, e ) ) # [END pubsub_subscriber_error_listener] -def receive_messages_with_delivery_attempts( - project_id, subscription_name, timeout=None -): +def receive_messages_with_delivery_attempts(project_id, subscription_id, timeout=None): # [START pubsub_dead_letter_delivery_attempt] from concurrent.futures import TimeoutError from google.cloud import pubsub_v1 # TODO(developer) # project_id = "your-project-id" - # subscription_name = "your-subscription-id" + # subscription_id = "your-subscription-id" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_id) def callback(message): print("Received message: {}".format(message)) @@ -627,64 +623,64 @@ def callback(message): list_in_topic_parser = subparsers.add_parser( "list-in-topic", help=list_subscriptions_in_topic.__doc__ ) - list_in_topic_parser.add_argument("topic_name") + list_in_topic_parser.add_argument("topic_id") list_in_project_parser = subparsers.add_parser( "list-in-project", help=list_subscriptions_in_project.__doc__ ) create_parser = subparsers.add_parser("create", help=create_subscription.__doc__) - create_parser.add_argument("topic_name") - create_parser.add_argument("subscription_name") + create_parser.add_argument("topic_id") + create_parser.add_argument("subscription_id") create_with_dead_letter_policy_parser = subparsers.add_parser( "create-with-dead-letter-policy", help=create_subscription_with_dead_letter_topic.__doc__, ) - create_with_dead_letter_policy_parser.add_argument("topic_name") - create_with_dead_letter_policy_parser.add_argument("subscription_name") - create_with_dead_letter_policy_parser.add_argument("dead_letter_topic_name") + create_with_dead_letter_policy_parser.add_argument("topic_id") + create_with_dead_letter_policy_parser.add_argument("subscription_id") + create_with_dead_letter_policy_parser.add_argument("dead_letter_topic_id") create_push_parser = subparsers.add_parser( "create-push", help=create_push_subscription.__doc__ ) - create_push_parser.add_argument("topic_name") - create_push_parser.add_argument("subscription_name") + create_push_parser.add_argument("topic_id") + create_push_parser.add_argument("subscription_id") create_push_parser.add_argument("endpoint") delete_parser = subparsers.add_parser("delete", help=delete_subscription.__doc__) - delete_parser.add_argument("subscription_name") + delete_parser.add_argument("subscription_id") update_push_parser = subparsers.add_parser( "update-push", help=update_push_subscription.__doc__ ) - update_push_parser.add_argument("topic_name") - update_push_parser.add_argument("subscription_name") + update_push_parser.add_argument("topic_id") + update_push_parser.add_argument("subscription_id") update_push_parser.add_argument("endpoint") update_dead_letter_policy_parser = subparsers.add_parser( "update-dead-letter-policy", help=update_subscription_with_dead_letter_policy.__doc__, ) - update_dead_letter_policy_parser.add_argument("topic_name") - update_dead_letter_policy_parser.add_argument("subscription_name") - update_dead_letter_policy_parser.add_argument("dead_letter_topic_name") + update_dead_letter_policy_parser.add_argument("topic_id") + update_dead_letter_policy_parser.add_argument("subscription_id") + update_dead_letter_policy_parser.add_argument("dead_letter_topic_id") remove_dead_letter_policy_parser = subparsers.add_parser( "remove-dead-letter-policy", help=remove_dead_letter_policy.__doc__ ) - remove_dead_letter_policy_parser.add_argument("topic_name") - remove_dead_letter_policy_parser.add_argument("subscription_name") + remove_dead_letter_policy_parser.add_argument("topic_id") + remove_dead_letter_policy_parser.add_argument("subscription_id") receive_parser = subparsers.add_parser("receive", help=receive_messages.__doc__) - receive_parser.add_argument("subscription_name") + receive_parser.add_argument("subscription_id") receive_parser.add_argument("timeout", default=None, type=float, nargs="?") receive_with_custom_attributes_parser = subparsers.add_parser( "receive-custom-attributes", help=receive_messages_with_custom_attributes.__doc__, ) - receive_with_custom_attributes_parser.add_argument("subscription_name") + receive_with_custom_attributes_parser.add_argument("subscription_id") receive_with_custom_attributes_parser.add_argument( "timeout", default=None, type=float, nargs="?" ) @@ -692,7 +688,7 @@ def callback(message): receive_with_flow_control_parser = subparsers.add_parser( "receive-flow-control", help=receive_messages_with_flow_control.__doc__ ) - receive_with_flow_control_parser.add_argument("subscription_name") + receive_with_flow_control_parser.add_argument("subscription_id") receive_with_flow_control_parser.add_argument( "timeout", default=None, type=float, nargs="?" ) @@ -700,18 +696,18 @@ def callback(message): synchronous_pull_parser = subparsers.add_parser( "receive-synchronously", help=synchronous_pull.__doc__ ) - synchronous_pull_parser.add_argument("subscription_name") + synchronous_pull_parser.add_argument("subscription_id") synchronous_pull_with_lease_management_parser = subparsers.add_parser( "receive-synchronously-with-lease", help=synchronous_pull_with_lease_management.__doc__, ) - synchronous_pull_with_lease_management_parser.add_argument("subscription_name") + synchronous_pull_with_lease_management_parser.add_argument("subscription_id") listen_for_errors_parser = subparsers.add_parser( "listen-for-errors", help=listen_for_errors.__doc__ ) - listen_for_errors_parser.add_argument("subscription_name") + listen_for_errors_parser.add_argument("subscription_id") listen_for_errors_parser.add_argument( "timeout", default=None, type=float, nargs="?" ) @@ -720,7 +716,7 @@ def callback(message): "receive-messages-with-delivery-attempts", help=receive_messages_with_delivery_attempts.__doc__, ) - receive_messages_with_delivery_attempts_parser.add_argument("subscription_name") + receive_messages_with_delivery_attempts_parser.add_argument("subscription_id") receive_messages_with_delivery_attempts_parser.add_argument( "timeout", default=None, type=float, nargs="?" ) @@ -728,56 +724,54 @@ def callback(message): args = parser.parse_args() if args.command == "list-in-topic": - list_subscriptions_in_topic(args.project_id, args.topic_name) + list_subscriptions_in_topic(args.project_id, args.topic_id) elif args.command == "list-in-project": list_subscriptions_in_project(args.project_id) elif args.command == "create": - create_subscription(args.project_id, args.topic_name, args.subscription_name) + create_subscription(args.project_id, args.topic_id, args.subscription_id) elif args.command == "create-with-dead-letter-policy": create_subscription_with_dead_letter_topic( args.project_id, - args.topic_name, - args.subscription_name, - args.dead_letter_topic_name, + args.topic_id, + args.subscription_id, + args.dead_letter_topic_id, ) elif args.command == "create-push": create_push_subscription( - args.project_id, args.topic_name, args.subscription_name, args.endpoint, + args.project_id, args.topic_id, args.subscription_id, args.endpoint, ) elif args.command == "delete": - delete_subscription(args.project_id, args.subscription_name) + delete_subscription(args.project_id, args.subscription_id) elif args.command == "update-push": update_push_subscription( - args.project_id, args.topic_name, args.subscription_name, args.endpoint, + args.project_id, args.topic_id, args.subscription_id, args.endpoint, ) elif args.command == "update-dead-letter-policy": update_subscription_with_dead_letter_policy( args.project_id, - args.topic_name, - args.subscription_name, - args.dead_letter_topic_name, + args.topic_id, + args.subscription_id, + args.dead_letter_topic_id, ) elif args.command == "remove-dead-letter-policy": - remove_dead_letter_policy( - args.project_id, args.topic_name, args.subscription_name - ) + remove_dead_letter_policy(args.project_id, args.topic_id, args.subscription_id) elif args.command == "receive": - receive_messages(args.project_id, args.subscription_name, args.timeout) + receive_messages(args.project_id, args.subscription_id, args.timeout) elif args.command == "receive-custom-attributes": receive_messages_with_custom_attributes( - args.project_id, args.subscription_name, args.timeout + args.project_id, args.subscription_id, args.timeout ) elif args.command == "receive-flow-control": receive_messages_with_flow_control( - args.project_id, args.subscription_name, args.timeout + args.project_id, args.subscription_id, args.timeout ) elif args.command == "receive-synchronously": - synchronous_pull(args.project_id, args.subscription_name) + synchronous_pull(args.project_id, args.subscription_id) elif args.command == "receive-synchronously-with-lease": - synchronous_pull_with_lease_management(args.project_id, args.subscription_name) + synchronous_pull_with_lease_management(args.project_id, args.subscription_id) elif args.command == "listen-for-errors": - listen_for_errors(args.project_id, args.subscription_name, args.timeout) + listen_for_errors(args.project_id, args.subscription_id, args.timeout) elif args.command == "receive-messages-with-delivery-attempts": receive_messages_with_delivery_attempts( - args.project_id, args.subscription_name, args.timeout + args.project_id, args.subscription_id, args.timeout ) diff --git a/run/markdown-preview/noxfile_config.py b/run/markdown-preview/noxfile_config.py index 43abbb029d78..3b544da5c981 100644 --- a/run/markdown-preview/noxfile_config.py +++ b/run/markdown-preview/noxfile_config.py @@ -23,8 +23,8 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - # We only run the e2e test in py37 session. - 'ignored_versions': ["2.7", "3.6", "3.8"], + # We only run the cloud run tests in py38 session. + 'ignored_versions': ["2.7", "3.6", "3.7"], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a diff --git a/run/system-package/e2e_test.py b/run/system-package/e2e_test.py new file mode 100644 index 000000000000..8a5712ab139d --- /dev/null +++ b/run/system-package/e2e_test.py @@ -0,0 +1,102 @@ +# Copyright 2020 Google, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This sample creates a secure two-service application running on Cloud Run. +# This test builds and deploys the two secure services +# to test that they interact properly together. + +import os +import subprocess +from urllib import request +import uuid + +import pytest + + +@pytest.fixture() +def services(): + # Unique suffix to create distinct service names + suffix = uuid.uuid4().hex + project = os.environ['GOOGLE_CLOUD_PROJECT'] + + # Build and Deploy Cloud Run Services + subprocess.run( + [ + "gcloud", + "builds", + "submit", + "--project", + project, + "--substitutions", + f"_SUFFIX={suffix}", + "--config", + "e2e_test_setup.yaml", + "--quiet", + ], check=True + ) + + # Get the URL for the service and the token + service = subprocess.run( + [ + "gcloud", + "run", + "--project", + project, + "--platform=managed", + "--region=us-central1", + "services", + "describe", + f"sys-package-{suffix}", + "--format=value(status.url)", + ], + stdout=subprocess.PIPE, + check=True + ).stdout.strip() + + token = subprocess.run( + ["gcloud", "auth", "print-identity-token"], stdout=subprocess.PIPE, + check=True + ).stdout.strip() + + yield service, token + + subprocess.run( + ["gcloud", "run", "services", "delete", f"sys-package-{suffix}", + "--project", project, "--platform", "managed", "--region", + "us-central1", "--quiet"], + check=True + ) + + +def test_end_to_end(services): + service = services[0].decode() + token = services[1].decode() + data = ( + "diagram.png?dot=digraph G { A -> {B, C, D} -> {F} }".replace(" ", "%20")) + print(service) + print(f"{service}{data}") + + req = request.Request( + f"{service}/{data}", + headers={ + "Authorization": f"Bearer {token}", + }, + ) + + response = request.urlopen(req) + assert response.status == 200 + + body = response.read() + # Response is a png + assert b"PNG" in body diff --git a/run/system-package/e2e_test_setup.yaml b/run/system-package/e2e_test_setup.yaml new file mode 100644 index 000000000000..be2df2c55489 --- /dev/null +++ b/run/system-package/e2e_test_setup.yaml @@ -0,0 +1,40 @@ +# Copyright 2020 Google, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: +- # Build the renderer image + name: gcr.io/cloud-builders/docker:latest + args: ['build', '--tag=gcr.io/$PROJECT_ID/sys-package-${_SUFFIX}', '.'] + +- # Push the container image to Container Registry + name: gcr.io/cloud-builders/docker + args: ['push', 'gcr.io/$PROJECT_ID/sys-package-${_SUFFIX}'] + +- # Deploy to Cloud Run + name: gcr.io/cloud-builders/gcloud + args: + - run + - deploy + - sys-package-${_SUFFIX} + - --image + - gcr.io/$PROJECT_ID/sys-package-${_SUFFIX} + - --region + - us-central1 + - --platform + - managed + - --no-allow-unauthenticated + +images: + - 'gcr.io/$PROJECT_ID/sys-package-${_SUFFIX}' + diff --git a/run/system-package/noxfile_config.py b/run/system-package/noxfile_config.py new file mode 100644 index 000000000000..3b544da5c981 --- /dev/null +++ b/run/system-package/noxfile_config.py @@ -0,0 +1,39 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default TEST_CONFIG_OVERRIDE for python repos. + +# You can copy this file into your directory, then it will be inported from +# the noxfile.py. + +# The source of truth: +# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py + +TEST_CONFIG_OVERRIDE = { + # You can opt out from the test for specific Python versions. + + # We only run the cloud run tests in py38 session. + 'ignored_versions': ["2.7", "3.6", "3.7"], + + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + 'gcloud_project_env': 'GCLOUD_PROJECT', + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + 'envs': {}, +} diff --git a/scripts/readme-gen/templates/README.tmpl.rst b/scripts/readme-gen/templates/README.tmpl.rst index 30ad03d050d8..1d0432d0d927 100644 --- a/scripts/readme-gen/templates/README.tmpl.rst +++ b/scripts/readme-gen/templates/README.tmpl.rst @@ -23,6 +23,13 @@ To run the sample, you need to enable the API at: {{required_api_url}} To run the sample, you need to have `{{required_role}}` role. {% endif %} +{% if required_roles %} +To run the sample, you need to have the following roles: +{% for role in required_roles %} +* `{{role}}` +{% endfor %} +{% endif %} + {{other_required_steps}} {% if setup %} diff --git a/spanner/cloud-client/snippets.py b/spanner/cloud-client/snippets.py index b600964088aa..26402bcd3be3 100644 --- a/spanner/cloud-client/snippets.py +++ b/spanner/cloud-client/snippets.py @@ -106,14 +106,23 @@ def delete_data(instance_id, database_id): instance = spanner_client.instance(instance_id) database = instance.database(database_id) - singers_to_delete = spanner.KeySet( - keys=[[1], [2], [3], [4], [5]]) + # Delete individual rows albums_to_delete = spanner.KeySet( - keys=[[1, 1], [1, 2], [2, 1], [2, 2], [2, 3]]) + keys=[[2, 1], [2, 3]]) + + # Delete a range of rows where the column key is >=3 and <5 + singers_range = spanner.KeyRange(start_closed=[3], end_open=[5]) + singers_to_delete = spanner.KeySet( + ranges=[singers_range]) + + # Delete remaining Singers rows, which will also delete the remaining + # Albums rows because Albums was defined with ON DELETE CASCADE + remaining_singers = spanner.KeySet(all_=True) with database.batch() as batch: batch.delete('Albums', albums_to_delete) batch.delete('Singers', singers_to_delete) + batch.delete('Singers', remaining_singers) print('Deleted data.') # [END spanner_delete_data] diff --git a/storage/api/requirements.txt b/storage/api/requirements.txt index 0f61949724cb..130c1828654a 100644 --- a/storage/api/requirements.txt +++ b/storage/api/requirements.txt @@ -1,3 +1,3 @@ google-api-python-client==1.9.1 -google-auth==1.16.0 +google-auth==1.16.1 google-auth-httplib2==0.0.3 diff --git a/storage/s3-sdk/requirements.txt b/storage/s3-sdk/requirements.txt index 266192819c54..754be494bf99 100644 --- a/storage/s3-sdk/requirements.txt +++ b/storage/s3-sdk/requirements.txt @@ -1 +1 @@ -boto3==1.13.22 +boto3==1.13.23 diff --git a/storage/signed_urls/requirements.txt b/storage/signed_urls/requirements.txt index 13adc4e7c494..bd8bdf4fe3c8 100644 --- a/storage/signed_urls/requirements.txt +++ b/storage/signed_urls/requirements.txt @@ -1,3 +1,3 @@ google-cloud-storage==1.28.1 -google-auth==1.16.0 +google-auth==1.16.1 six==1.15.0 diff --git a/storage/transfer_service/requirements.txt b/storage/transfer_service/requirements.txt index 0f61949724cb..130c1828654a 100644 --- a/storage/transfer_service/requirements.txt +++ b/storage/transfer_service/requirements.txt @@ -1,3 +1,3 @@ google-api-python-client==1.9.1 -google-auth==1.16.0 +google-auth==1.16.1 google-auth-httplib2==0.0.3 diff --git a/tasks/requirements.txt b/tasks/requirements.txt index e5b2c4310060..b7bc58a3b74e 100644 --- a/tasks/requirements.txt +++ b/tasks/requirements.txt @@ -1,2 +1 @@ google-cloud-tasks==1.5.0 -googleapis-common-protos==1.51.0 diff --git a/translate/cloud-client/hybrid_glossaries/hybrid_tutorial.py b/translate/cloud-client/hybrid_glossaries/hybrid_tutorial.py index 0da84b722078..1b320e5b2a49 100644 --- a/translate/cloud-client/hybrid_glossaries/hybrid_tutorial.py +++ b/translate/cloud-client/hybrid_glossaries/hybrid_tutorial.py @@ -193,21 +193,28 @@ def text_to_speech(text, outfile): client = texttospeech.TextToSpeechClient() # Sets the text input to be synthesized - synthesis_input = texttospeech.types.SynthesisInput(ssml=ssml) + synthesis_input = texttospeech.SynthesisInput(ssml=ssml) # Builds the voice request, selects the language code ("en-US") and # the SSML voice gender ("MALE") - voice = texttospeech.types.VoiceSelectionParams( + voice = texttospeech.VoiceSelectionParams( language_code='en-US', - ssml_gender=texttospeech.enums.SsmlVoiceGender.MALE) + ssml_gender=texttospeech.SsmlVoiceGender.MALE) # Selects the type of audio file to return - audio_config = texttospeech.types.AudioConfig( - audio_encoding=texttospeech.enums.AudioEncoding.MP3) + audio_config = texttospeech.AudioConfig( + audio_encoding=texttospeech.AudioEncoding.MP3) # Performs the text-to-speech request on the text input with the selected # voice parameters and audio file type - response = client.synthesize_speech(synthesis_input, voice, audio_config) + + request = texttospeech.SynthesizeSpeechRequest( + input=synthesis_input, + voice=voice, + audio_config=audio_config + ) + + response = client.synthesize_speech(request=request) # Writes the synthetic audio to the output file. with open(outfile, 'wb') as out: diff --git a/translate/cloud-client/hybrid_glossaries/hybrid_tutorial_tests.py b/translate/cloud-client/hybrid_glossaries/hybrid_tutorial_test.py similarity index 93% rename from translate/cloud-client/hybrid_glossaries/hybrid_tutorial_tests.py rename to translate/cloud-client/hybrid_glossaries/hybrid_tutorial_test.py index 5224ca524a91..f99be330b92d 100644 --- a/translate/cloud-client/hybrid_glossaries/hybrid_tutorial_tests.py +++ b/translate/cloud-client/hybrid_glossaries/hybrid_tutorial_test.py @@ -13,7 +13,9 @@ # limitations under the License. import os +import re import sys +import uuid from hybrid_tutorial import create_glossary from hybrid_tutorial import pic_to_text @@ -28,14 +30,10 @@ def test_vision_standard_format(): - - expected_text = 'This is\na test!\n' - alt_expected_text = 'This\nis\na test!\n' - # Generate text using Vision API text = pic_to_text('resources/standard_format.jpeg') - assert (text == expected_text) or (text == alt_expected_text) + assert re.match(r"This\s?is\s?a\s?test!\s?", text) def test_vision_non_standard_format(): @@ -58,7 +56,7 @@ def test_create_and_delete_glossary(): from beta_snippets import delete_glossary languages = ['fr', 'en'] - glossary_name = 'test-glossary' + glossary_name = f'test-glossary-{uuid.uuid4()}' glossary_uri = 'gs://cloud-samples-data/translation/bistro_glossary.csv' # create_glossary will raise an exception if creation fails diff --git a/translate/cloud-client/hybrid_glossaries/requirements.txt b/translate/cloud-client/hybrid_glossaries/requirements.txt index d13c5f7d8080..ed9bbd1eae82 100644 --- a/translate/cloud-client/hybrid_glossaries/requirements.txt +++ b/translate/cloud-client/hybrid_glossaries/requirements.txt @@ -1,3 +1,3 @@ google-cloud-translate==2.0.1 google-cloud-vision==1.0.0 -google-cloud-texttospeech==1.0.1 +google-cloud-texttospeech==2.0.0 diff --git a/translate/cloud-client/hybrid_glossaries/resources/non_standard_format.txt b/translate/cloud-client/hybrid_glossaries/resources/non_standard_format.txt index 8a6e3c113d18..529799ee94b1 100644 --- a/translate/cloud-client/hybrid_glossaries/resources/non_standard_format.txt +++ b/translate/cloud-client/hybrid_glossaries/resources/non_standard_format.txt @@ -19,11 +19,11 @@ $10 Firebase Fruit Salad watermelon, honeydew melon, and pineapple +$6 Compute Engine Burger quarter-pound burger with cheddar cheese $10 -$6 BigQuery BLT bacon, lettuce, and tomato sandwich diff --git a/translate/cloud-client/pytest.ini b/translate/cloud-client/pytest.ini new file mode 100644 index 000000000000..f139880676b3 --- /dev/null +++ b/translate/cloud-client/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +# don't recurse into hybrid_glossaries directory +norecursedirs = * \ No newline at end of file