From a740170d19b8aebcc860f2945ca4c5bb08682443 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Fri, 10 Jul 2020 14:30:41 +0200 Subject: [PATCH 001/101] Revert "samples: add samples from pubsub/cloud-client (#134)" This reverts commit e204b86e34346b6d456771ef4d4cfe5e15e53238. We want to merge the samples branch *unsquashed* to preserve samples commit history. --- .github/CODEOWNERS | 11 - samples/AUTHORING_GUIDE.md | 1 - samples/CONTRIBUTING.md | 1 - samples/snippets/README.rst | 282 --------- samples/snippets/README.rst.in | 30 - samples/snippets/iam.py | 231 ------- samples/snippets/iam_test.py | 118 ---- samples/snippets/noxfile.py | 224 ------- samples/snippets/publisher.py | 334 ---------- samples/snippets/publisher_test.py | 146 ----- samples/snippets/quickstart/pub.py | 86 --- samples/snippets/quickstart/pub_test.py | 56 -- samples/snippets/quickstart/sub.py | 69 --- samples/snippets/quickstart/sub_test.py | 102 --- samples/snippets/requirements-test.txt | 3 - samples/snippets/subscriber.py | 783 ------------------------ samples/snippets/subscriber_test.py | 341 ----------- synth.py | 11 +- 18 files changed, 1 insertion(+), 2828 deletions(-) delete mode 100644 .github/CODEOWNERS delete mode 100644 samples/AUTHORING_GUIDE.md delete mode 100644 samples/CONTRIBUTING.md delete mode 100644 samples/snippets/README.rst delete mode 100644 samples/snippets/README.rst.in delete mode 100644 samples/snippets/iam.py delete mode 100644 samples/snippets/iam_test.py delete mode 100644 samples/snippets/noxfile.py delete mode 100644 samples/snippets/publisher.py delete mode 100644 samples/snippets/publisher_test.py delete mode 100644 samples/snippets/quickstart/pub.py delete mode 100644 samples/snippets/quickstart/pub_test.py delete mode 100644 samples/snippets/quickstart/sub.py delete mode 100644 samples/snippets/quickstart/sub_test.py delete mode 100644 samples/snippets/requirements-test.txt delete mode 100644 samples/snippets/subscriber.py delete mode 100644 samples/snippets/subscriber_test.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index cf01548a9..000000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,11 +0,0 @@ -# Code owners file. -# This file controls who is tagged for review for any given pull request. -# -# For syntax help see: -# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax - - -# The python-samples-owners team is the default owner for anything not -# explicitly taken by someone else. - - /samples/ @anguillanneuf @hongalex @googleapis/python-samples-owners diff --git a/samples/AUTHORING_GUIDE.md b/samples/AUTHORING_GUIDE.md deleted file mode 100644 index 55c97b32f..000000000 --- a/samples/AUTHORING_GUIDE.md +++ /dev/null @@ -1 +0,0 @@ -See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/AUTHORING_GUIDE.md \ No newline at end of file diff --git a/samples/CONTRIBUTING.md b/samples/CONTRIBUTING.md deleted file mode 100644 index 34c882b6f..000000000 --- a/samples/CONTRIBUTING.md +++ /dev/null @@ -1 +0,0 @@ -See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/CONTRIBUTING.md \ No newline at end of file diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst deleted file mode 100644 index 2676680af..000000000 --- a/samples/snippets/README.rst +++ /dev/null @@ -1,282 +0,0 @@ - -.. This file is automatically generated. Do not edit this file directly. - -Google Cloud Pub/Sub Python Samples -=============================================================================== - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/README.rst - - -This directory contains samples for Google Cloud Pub/Sub. `Google Cloud Pub/Sub`_ is a fully-managed real-time messaging service that allows you to send and receive messages between independent applications. - - - - -.. _Google Cloud Pub/Sub: https://cloud.google.com/pubsub/docs - - -Setup -------------------------------------------------------------------------------- - - - -Authentication -++++++++++++++ - -This sample requires you to have authentication setup. Refer to the -`Authentication Getting Started Guide`_ for instructions on setting up -credentials for applications. - -.. _Authentication Getting Started Guide: - https://cloud.google.com/docs/authentication/getting-started - - - - -Install Dependencies -++++++++++++++++++++ - -#. Clone python-docs-samples and change directory to the sample directory you want to use. - - .. code-block:: bash - - $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git - -#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. - - .. _Python Development Environment Setup Guide: - https://cloud.google.com/python/setup - -#. Create a virtualenv. Samples are compatible with Python 3.6+. - - .. code-block:: bash - - $ virtualenv env - $ source env/bin/activate - -#. Install the dependencies needed to run the samples. - - .. code-block:: bash - - $ pip install -r requirements.txt - -.. _pip: https://pip.pypa.io/ -.. _virtualenv: https://virtualenv.pypa.io/ - - - - - - -Samples -------------------------------------------------------------------------------- - - -Quickstart -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/quickstart.py,pubsub/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python quickstart.py - - - - -Publisher -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/publisher.py,pubsub/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python publisher.py - - - usage: publisher.py [-h] - project_id - {list,create,delete,publish,publish-with-custom-attributes,publish-with-error-handler,publish-with-batch-settings,publish-with-retry-settings} - ... - - This application demonstrates how to perform basic operations on topics - with the Cloud Pub/Sub API. - - For more information, see the README.md under /pubsub and the documentation - at https://cloud.google.com/pubsub/docs. - - positional arguments: - project_id Your Google Cloud project ID - {list,create,delete,publish,publish-with-custom-attributes,publish-with-error-handler,publish-with-batch-settings,publish-with-retry-settings} - list Lists all Pub/Sub topics in the given project. - create Create a new Pub/Sub topic. - delete Deletes an existing Pub/Sub topic. - publish Publishes multiple messages to a Pub/Sub topic. - publish-with-custom-attributes - Publishes multiple messages with custom attributes to - a Pub/Sub topic. - publish-with-error-handler - Publishes multiple messages to a Pub/Sub topic with an - error handler. - publish-with-batch-settings - Publishes multiple messages to a Pub/Sub topic with - batch settings. - publish-with-retry-settings - Publishes messages with custom retry settings. - - optional arguments: - -h, --help show this help message and exit - - - - - -Subscribers -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/subscriber.py,pubsub/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python subscriber.py - - - usage: subscriber.py [-h] - project_id - {list-in-topic,list-in-project,create,create-with-dead-letter-policy,create-push,delete,update-push,update-dead-letter-policy,remove-dead-letter-policy,receive,receive-custom-attributes,receive-flow-control,receive-synchronously,receive-synchronously-with-lease,listen-for-errors,receive-messages-with-delivery-attempts} - ... - - This application demonstrates how to perform basic operations on - subscriptions with the Cloud Pub/Sub API. - - For more information, see the README.md under /pubsub and the documentation - at https://cloud.google.com/pubsub/docs. - - positional arguments: - project_id Your Google Cloud project ID - {list-in-topic,list-in-project,create,create-with-dead-letter-policy,create-push,delete,update-push,update-dead-letter-policy,remove-dead-letter-policy,receive,receive-custom-attributes,receive-flow-control,receive-synchronously,receive-synchronously-with-lease,listen-for-errors,receive-messages-with-delivery-attempts} - list-in-topic Lists all subscriptions for a given topic. - list-in-project Lists all subscriptions in the current project. - create Create a new pull subscription on the given topic. - create-with-dead-letter-policy - Create a subscription with dead letter policy. - create-push Create a new push subscription on the given topic. - delete Deletes an existing Pub/Sub topic. - update-push Updates an existing Pub/Sub subscription's push - endpoint URL. Note that certain properties of a - subscription, such as its topic, are not modifiable. - update-dead-letter-policy - Update a subscription's dead letter policy. - remove-dead-letter-policy - Remove dead letter policy from a subscription. - receive Receives messages from a pull subscription. - receive-custom-attributes - Receives messages from a pull subscription. - receive-flow-control - Receives messages from a pull subscription with flow - control. - receive-synchronously - Pulling messages synchronously. - receive-synchronously-with-lease - Pulling messages synchronously with lease management - listen-for-errors Receives messages and catches errors from a pull - subscription. - receive-messages-with-delivery-attempts - - optional arguments: - -h, --help show this help message and exit - - - - - -Identity and Access Management -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/iam.py,pubsub/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python iam.py - - - usage: iam.py [-h] - project - {get-topic-policy,get-subscription-policy,set-topic-policy,set-subscription-policy,check-topic-permissions,check-subscription-permissions} - ... - - This application demonstrates how to perform basic operations on IAM - policies with the Cloud Pub/Sub API. - - For more information, see the README.md under /pubsub and the documentation - at https://cloud.google.com/pubsub/docs. - - positional arguments: - project Your Google Cloud project ID - {get-topic-policy,get-subscription-policy,set-topic-policy,set-subscription-policy,check-topic-permissions,check-subscription-permissions} - get-topic-policy Prints the IAM policy for the given topic. - get-subscription-policy - Prints the IAM policy for the given subscription. - set-topic-policy Sets the IAM policy for a topic. - set-subscription-policy - Sets the IAM policy for a topic. - check-topic-permissions - Checks to which permissions are available on the given - topic. - check-subscription-permissions - Checks to which permissions are available on the given - subscription. - - optional arguments: - -h, --help show this help message and exit - - - - - - - - - -The client library -------------------------------------------------------------------------------- - -This sample uses the `Google Cloud Client Library for Python`_. -You can read the documentation for more details on API usage and use GitHub -to `browse the source`_ and `report issues`_. - -.. _Google Cloud Client Library for Python: - https://googlecloudplatform.github.io/google-cloud-python/ -.. _browse the source: - https://github.com/GoogleCloudPlatform/google-cloud-python -.. _report issues: - https://github.com/GoogleCloudPlatform/google-cloud-python/issues - - - -.. _Google Cloud SDK: https://cloud.google.com/sdk/ diff --git a/samples/snippets/README.rst.in b/samples/snippets/README.rst.in deleted file mode 100644 index ddbc64712..000000000 --- a/samples/snippets/README.rst.in +++ /dev/null @@ -1,30 +0,0 @@ -# This file is used to generate README.rst - -product: - name: Google Cloud Pub/Sub - short_name: Cloud Pub/Sub - url: https://cloud.google.com/pubsub/docs - description: > - `Google Cloud Pub/Sub`_ is a fully-managed real-time messaging service that - allows you to send and receive messages between independent applications. - -setup: -- auth -- install_deps - -samples: -- name: Quickstart - file: quickstart.py -- name: Publisher - file: publisher.py - show_help: true -- name: Subscribers - file: subscriber.py - show_help: true -- name: Identity and Access Management - file: iam.py - show_help: true - -cloud_client_library: true - -folder: pubsub/cloud-client \ No newline at end of file diff --git a/samples/snippets/iam.py b/samples/snippets/iam.py deleted file mode 100644 index 71c55d764..000000000 --- a/samples/snippets/iam.py +++ /dev/null @@ -1,231 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2019 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This application demonstrates how to perform basic operations on IAM -policies with the Cloud Pub/Sub API. - -For more information, see the README.md under /pubsub and the documentation -at https://cloud.google.com/pubsub/docs. -""" - -import argparse - - -def get_topic_policy(project, topic_id): - """Prints the IAM policy for the given topic.""" - # [START pubsub_get_topic_policy] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # topic_id = "your-topic-id" - - client = pubsub_v1.PublisherClient() - topic_path = client.topic_path(project, topic_id) - - policy = client.get_iam_policy(topic_path) - - print("Policy for topic {}:".format(topic_path)) - for binding in policy.bindings: - print("Role: {}, Members: {}".format(binding.role, binding.members)) - # [END pubsub_get_topic_policy] - - -def get_subscription_policy(project, subscription_id): - """Prints the IAM policy for the given subscription.""" - # [START pubsub_get_subscription_policy] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # subscription_id = "your-subscription-id" - - client = pubsub_v1.SubscriberClient() - subscription_path = client.subscription_path(project, subscription_id) - - policy = client.get_iam_policy(subscription_path) - - print("Policy for subscription {}:".format(subscription_path)) - for binding in policy.bindings: - print("Role: {}, Members: {}".format(binding.role, binding.members)) - - client.close() - # [END pubsub_get_subscription_policy] - - -def set_topic_policy(project, topic_id): - """Sets the IAM policy for a topic.""" - # [START pubsub_set_topic_policy] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # topic_id = "your-topic-id" - - client = pubsub_v1.PublisherClient() - topic_path = client.topic_path(project, topic_id) - - policy = client.get_iam_policy(topic_path) - - # Add all users as viewers. - policy.bindings.add(role="roles/pubsub.viewer", members=["allUsers"]) - - # Add a group as a publisher. - policy.bindings.add( - role="roles/pubsub.publisher", members=["group:cloud-logs@google.com"] - ) - - # Set the policy - policy = client.set_iam_policy(topic_path, policy) - - print("IAM policy for topic {} set: {}".format(topic_id, policy)) - # [END pubsub_set_topic_policy] - - -def set_subscription_policy(project, subscription_id): - """Sets the IAM policy for a topic.""" - # [START pubsub_set_subscription_policy] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # subscription_id = "your-subscription-id" - - client = pubsub_v1.SubscriberClient() - subscription_path = client.subscription_path(project, subscription_id) - - policy = client.get_iam_policy(subscription_path) - - # Add all users as viewers. - policy.bindings.add(role="roles/pubsub.viewer", members=["allUsers"]) - - # Add a group as an editor. - policy.bindings.add(role="roles/editor", members=["group:cloud-logs@google.com"]) - - # Set the policy - policy = client.set_iam_policy(subscription_path, policy) - - print("IAM policy for subscription {} set: {}".format(subscription_id, policy)) - - client.close() - # [END pubsub_set_subscription_policy] - - -def check_topic_permissions(project, topic_id): - """Checks to which permissions are available on the given topic.""" - # [START pubsub_test_topic_permissions] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # topic_id = "your-topic-id" - - client = pubsub_v1.PublisherClient() - topic_path = client.topic_path(project, topic_id) - - permissions_to_check = ["pubsub.topics.publish", "pubsub.topics.update"] - - allowed_permissions = client.test_iam_permissions(topic_path, permissions_to_check) - - print( - "Allowed permissions for topic {}: {}".format(topic_path, allowed_permissions) - ) - # [END pubsub_test_topic_permissions] - - -def check_subscription_permissions(project, subscription_id): - """Checks to which permissions are available on the given subscription.""" - # [START pubsub_test_subscription_permissions] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # subscription_id = "your-subscription-id" - - client = pubsub_v1.SubscriberClient() - subscription_path = client.subscription_path(project, subscription_id) - - permissions_to_check = [ - "pubsub.subscriptions.consume", - "pubsub.subscriptions.update", - ] - - allowed_permissions = client.test_iam_permissions( - subscription_path, permissions_to_check - ) - - print( - "Allowed permissions for subscription {}: {}".format( - subscription_path, allowed_permissions - ) - ) - - client.close() - # [END pubsub_test_subscription_permissions] - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument("project", help="Your Google Cloud project ID") - - subparsers = parser.add_subparsers(dest="command") - - get_topic_policy_parser = subparsers.add_parser( - "get-topic-policy", help=get_topic_policy.__doc__ - ) - get_topic_policy_parser.add_argument("topic_id") - - get_subscription_policy_parser = subparsers.add_parser( - "get-subscription-policy", help=get_subscription_policy.__doc__ - ) - get_subscription_policy_parser.add_argument("subscription_id") - - set_topic_policy_parser = subparsers.add_parser( - "set-topic-policy", help=set_topic_policy.__doc__ - ) - set_topic_policy_parser.add_argument("topic_id") - - set_subscription_policy_parser = subparsers.add_parser( - "set-subscription-policy", help=set_subscription_policy.__doc__ - ) - set_subscription_policy_parser.add_argument("subscription_id") - - check_topic_permissions_parser = subparsers.add_parser( - "check-topic-permissions", help=check_topic_permissions.__doc__ - ) - check_topic_permissions_parser.add_argument("topic_id") - - check_subscription_permissions_parser = subparsers.add_parser( - "check-subscription-permissions", help=check_subscription_permissions.__doc__, - ) - check_subscription_permissions_parser.add_argument("subscription_id") - - args = parser.parse_args() - - if args.command == "get-topic-policy": - get_topic_policy(args.project, args.topic_id) - elif args.command == "get-subscription-policy": - get_subscription_policy(args.project, args.subscription_id) - elif args.command == "set-topic-policy": - set_topic_policy(args.project, args.topic_id) - elif args.command == "set-subscription-policy": - set_subscription_policy(args.project, args.subscription_id) - elif args.command == "check-topic-permissions": - check_topic_permissions(args.project, args.topic_id) - elif args.command == "check-subscription-permissions": - check_subscription_permissions(args.project, args.subscription_id) diff --git a/samples/snippets/iam_test.py b/samples/snippets/iam_test.py deleted file mode 100644 index d196953f6..000000000 --- a/samples/snippets/iam_test.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import uuid - -from google.cloud import pubsub_v1 -import pytest - -import iam - -UUID = uuid.uuid4().hex -PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] -TOPIC = "iam-test-topic-" + UUID -SUBSCRIPTION = "iam-test-subscription-" + UUID - - -@pytest.fixture(scope="module") -def publisher_client(): - yield pubsub_v1.PublisherClient() - - -@pytest.fixture(scope="module") -def topic(publisher_client): - topic_path = publisher_client.topic_path(PROJECT, TOPIC) - - try: - publisher_client.delete_topic(topic_path) - except Exception: - pass - - publisher_client.create_topic(topic_path) - - yield topic_path - - publisher_client.delete_topic(topic_path) - - -@pytest.fixture(scope="module") -def subscriber_client(): - subscriber_client = pubsub_v1.SubscriberClient() - yield subscriber_client - subscriber_client.close() - - -@pytest.fixture -def subscription(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION) - - try: - subscriber_client.delete_subscription(subscription_path) - except Exception: - pass - - subscriber_client.create_subscription(subscription_path, topic=topic) - - yield subscription_path - - subscriber_client.delete_subscription(subscription_path) - - -def test_get_topic_policy(topic, capsys): - iam.get_topic_policy(PROJECT, TOPIC) - - out, _ = capsys.readouterr() - assert topic in out - - -def test_get_subscription_policy(subscription, capsys): - iam.get_subscription_policy(PROJECT, SUBSCRIPTION) - - out, _ = capsys.readouterr() - assert subscription in out - - -def test_set_topic_policy(publisher_client, topic): - iam.set_topic_policy(PROJECT, TOPIC) - - policy = publisher_client.get_iam_policy(topic) - assert "roles/pubsub.publisher" in str(policy) - assert "allUsers" in str(policy) - - -def test_set_subscription_policy(subscriber_client, subscription): - iam.set_subscription_policy(PROJECT, SUBSCRIPTION) - - policy = subscriber_client.get_iam_policy(subscription) - assert "roles/pubsub.viewer" in str(policy) - assert "allUsers" in str(policy) - - -def test_check_topic_permissions(topic, capsys): - iam.check_topic_permissions(PROJECT, TOPIC) - - out, _ = capsys.readouterr() - - assert topic in out - assert "pubsub.topics.publish" in out - - -def test_check_subscription_permissions(subscription, capsys): - iam.check_subscription_permissions(PROJECT, SUBSCRIPTION) - - out, _ = capsys.readouterr() - - assert subscription in out - assert "pubsub.subscriptions.consume" in out diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py deleted file mode 100644 index ba55d7ce5..000000000 --- a/samples/snippets/noxfile.py +++ /dev/null @@ -1,224 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import os -from pathlib import Path -import sys - -import nox - - -# WARNING - WARNING - WARNING - WARNING - WARNING -# WARNING - WARNING - WARNING - WARNING - WARNING -# DO NOT EDIT THIS FILE EVER! -# WARNING - WARNING - WARNING - WARNING - WARNING -# WARNING - WARNING - WARNING - WARNING - WARNING - -# Copy `noxfile_config.py` to your directory and modify it instead. - - -# `TEST_CONFIG` dict is a configuration hook that allows users to -# modify the test configurations. The values here should be in sync -# with `noxfile_config.py`. Users will copy `noxfile_config.py` into -# their directory and modify it. - -TEST_CONFIG = { - # You can opt out from the test for specific Python versions. - 'ignored_versions': ["2.7"], - - # An envvar key for determining the project id to use. Change it - # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a - # build specific Cloud project. You can also use your own string - # to use your own Cloud project. - 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', - # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', - - # A dictionary you want to inject into your test. Don't put any - # secrets here. These values will override predefined values. - 'envs': {}, -} - - -try: - # Ensure we can import noxfile_config in the project's directory. - sys.path.append('.') - from noxfile_config import TEST_CONFIG_OVERRIDE -except ImportError as e: - print("No user noxfile_config found: detail: {}".format(e)) - TEST_CONFIG_OVERRIDE = {} - -# Update the TEST_CONFIG with the user supplied values. -TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) - - -def get_pytest_env_vars(): - """Returns a dict for pytest invocation.""" - ret = {} - - # Override the GCLOUD_PROJECT and the alias. - env_key = TEST_CONFIG['gcloud_project_env'] - # This should error out if not set. - ret['GOOGLE_CLOUD_PROJECT'] = os.environ[env_key] - - # Apply user supplied envs. - ret.update(TEST_CONFIG['envs']) - return ret - - -# DO NOT EDIT - automatically generated. -# All versions used to tested samples. -ALL_VERSIONS = ["2.7", "3.6", "3.7", "3.8"] - -# Any default versions that should be ignored. -IGNORED_VERSIONS = TEST_CONFIG['ignored_versions'] - -TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) - -INSTALL_LIBRARY_FROM_SOURCE = bool(os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False)) -# -# Style Checks -# - - -def _determine_local_import_names(start_dir): - """Determines all import names that should be considered "local". - - This is used when running the linter to insure that import order is - properly checked. - """ - file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] - return [ - basename - for basename, extension in file_ext_pairs - if extension == ".py" - or os.path.isdir(os.path.join(start_dir, basename)) - and basename not in ("__pycache__") - ] - - -# Linting with flake8. -# -# We ignore the following rules: -# E203: whitespace before ‘:’ -# E266: too many leading ‘#’ for block comment -# E501: line too long -# I202: Additional newline in a section of imports -# -# We also need to specify the rules which are ignored by default: -# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] -FLAKE8_COMMON_ARGS = [ - "--show-source", - "--builtin=gettext", - "--max-complexity=20", - "--import-order-style=google", - "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", - "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", - "--max-line-length=88", -] - - -@nox.session -def lint(session): - session.install("flake8", "flake8-import-order") - - local_names = _determine_local_import_names(".") - args = FLAKE8_COMMON_ARGS + [ - "--application-import-names", - ",".join(local_names), - "." - ] - session.run("flake8", *args) - - -# -# Sample Tests -# - - -PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] - - -def _session_tests(session, post_install=None): - """Runs py.test for a particular project.""" - if os.path.exists("requirements.txt"): - session.install("-r", "requirements.txt") - - if os.path.exists("requirements-test.txt"): - session.install("-r", "requirements-test.txt") - - if INSTALL_LIBRARY_FROM_SOURCE: - session.install("-e", _get_repo_root()) - - if post_install: - post_install(session) - - session.run( - "pytest", - *(PYTEST_COMMON_ARGS + session.posargs), - # Pytest will return 5 when no tests are collected. This can happen - # on travis where slow and flaky tests are excluded. - # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html - success_codes=[0, 5], - env=get_pytest_env_vars() - ) - - -@nox.session(python=ALL_VERSIONS) -def py(session): - """Runs py.test for a sample using the specified version of Python.""" - if session.python in TESTED_VERSIONS: - _session_tests(session) - else: - session.skip("SKIPPED: {} tests are disabled for this sample.".format( - session.python - )) - - -# -# Readmegen -# - - -def _get_repo_root(): - """ Returns the root folder of the project. """ - # Get root of this repository. Assume we don't have directories nested deeper than 10 items. - p = Path(os.getcwd()) - for i in range(10): - if p is None: - break - if Path(p / ".git").exists(): - return str(p) - p = p.parent - raise Exception("Unable to detect repository root.") - - -GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) - - -@nox.session -@nox.parametrize("path", GENERATED_READMES) -def readmegen(session, path): - """(Re-)generates the readme for a sample.""" - session.install("jinja2", "pyyaml") - dir_ = os.path.dirname(path) - - if os.path.exists(os.path.join(dir_, "requirements.txt")): - session.install("-r", os.path.join(dir_, "requirements.txt")) - - in_file = os.path.join(dir_, "README.rst.in") - session.run( - "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file - ) diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py deleted file mode 100644 index 477b31b9c..000000000 --- a/samples/snippets/publisher.py +++ /dev/null @@ -1,334 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This application demonstrates how to perform basic operations on topics -with the Cloud Pub/Sub API. - -For more information, see the README.md under /pubsub and the documentation -at https://cloud.google.com/pubsub/docs. -""" - -import argparse - - -def list_topics(project_id): - """Lists all Pub/Sub topics in the given project.""" - # [START pubsub_list_topics] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - - publisher = pubsub_v1.PublisherClient() - project_path = publisher.project_path(project_id) - - for topic in publisher.list_topics(project_path): - print(topic) - # [END pubsub_list_topics] - - -def create_topic(project_id, topic_id): - """Create a new Pub/Sub topic.""" - # [START pubsub_quickstart_create_topic] - # [START pubsub_create_topic] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # topic_id = "your-topic-id" - - publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project_id, topic_id) - - topic = publisher.create_topic(topic_path) - - print("Topic created: {}".format(topic)) - # [END pubsub_quickstart_create_topic] - # [END pubsub_create_topic] - - -def delete_topic(project_id, topic_id): - """Deletes an existing Pub/Sub topic.""" - # [START pubsub_delete_topic] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # topic_id = "your-topic-id" - - publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project_id, topic_id) - - publisher.delete_topic(topic_path) - - print("Topic deleted: {}".format(topic_path)) - # [END pubsub_delete_topic] - - -def publish_messages(project_id, topic_id): - """Publishes multiple messages to a Pub/Sub topic.""" - # [START pubsub_quickstart_publisher] - # [START pubsub_publish] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # topic_id = "your-topic-id" - - publisher = pubsub_v1.PublisherClient() - # The `topic_path` method creates a fully qualified identifier - # in the form `projects/{project_id}/topics/{topic_id}` - topic_path = publisher.topic_path(project_id, topic_id) - - for n in range(1, 10): - data = u"Message number {}".format(n) - # Data must be a bytestring - data = data.encode("utf-8") - # When you publish a message, the client returns a future. - future = publisher.publish(topic_path, data=data) - print(future.result()) - - print("Published messages.") - # [END pubsub_quickstart_publisher] - # [END pubsub_publish] - - -def publish_messages_with_custom_attributes(project_id, topic_id): - """Publishes multiple messages with custom attributes - to a Pub/Sub topic.""" - # [START pubsub_publish_custom_attributes] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # topic_id = "your-topic-id" - - publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project_id, topic_id) - - for n in range(1, 10): - data = u"Message number {}".format(n) - # Data must be a bytestring - data = data.encode("utf-8") - # Add two attributes, origin and username, to the message - future = publisher.publish( - topic_path, data, origin="python-sample", username="gcp" - ) - print(future.result()) - - print("Published messages with custom attributes.") - # [END pubsub_publish_custom_attributes] - - -def publish_messages_with_error_handler(project_id, topic_id): - # [START pubsub_publish_messages_error_handler] - """Publishes multiple messages to a Pub/Sub topic with an error handler.""" - import time - - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # topic_id = "your-topic-id" - - publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project_id, topic_id) - - futures = dict() - - def get_callback(f, data): - def callback(f): - try: - print(f.result()) - futures.pop(data) - except: # noqa - print("Please handle {} for {}.".format(f.exception(), data)) - - return callback - - for i in range(10): - data = str(i) - futures.update({data: None}) - # When you publish a message, the client returns a future. - future = publisher.publish( - topic_path, data=data.encode("utf-8") # data must be a bytestring. - ) - futures[data] = future - # Publish failures shall be handled in the callback function. - future.add_done_callback(get_callback(future, data)) - - # Wait for all the publish futures to resolve before exiting. - while futures: - time.sleep(5) - - print("Published message with error handler.") - # [END pubsub_publish_messages_error_handler] - - -def publish_messages_with_batch_settings(project_id, topic_id): - """Publishes multiple messages to a Pub/Sub topic with batch settings.""" - # [START pubsub_publisher_batch_settings] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # topic_id = "your-topic-id" - - # Configure the batch to publish as soon as there is ten messages, - # one kilobyte of data, or one second has passed. - batch_settings = pubsub_v1.types.BatchSettings( - max_messages=10, # default 100 - max_bytes=1024, # default 1 MB - max_latency=1, # default 10 ms - ) - publisher = pubsub_v1.PublisherClient(batch_settings) - topic_path = publisher.topic_path(project_id, topic_id) - - # Resolve the publish future in a separate thread. - def callback(future): - message_id = future.result() - print(message_id) - - for n in range(1, 10): - data = u"Message number {}".format(n) - # Data must be a bytestring - data = data.encode("utf-8") - future = publisher.publish(topic_path, data=data) - # Non-blocking. Allow the publisher client to batch multiple messages. - future.add_done_callback(callback) - - print("Published messages with batch settings.") - # [END pubsub_publisher_batch_settings] - - -def publish_messages_with_retry_settings(project_id, topic_id): - """Publishes messages with custom retry settings.""" - # [START pubsub_publisher_retry_settings] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # topic_id = "your-topic-id" - - # Configure the retry settings. Defaults will be overwritten. - retry_settings = { - "interfaces": { - "google.pubsub.v1.Publisher": { - "retry_codes": { - "publish": [ - "ABORTED", - "CANCELLED", - "DEADLINE_EXCEEDED", - "INTERNAL", - "RESOURCE_EXHAUSTED", - "UNAVAILABLE", - "UNKNOWN", - ] - }, - "retry_params": { - "messaging": { - "initial_retry_delay_millis": 100, # default: 100 - "retry_delay_multiplier": 1.3, # default: 1.3 - "max_retry_delay_millis": 60000, # default: 60000 - "initial_rpc_timeout_millis": 5000, # default: 25000 - "rpc_timeout_multiplier": 1.0, # default: 1.0 - "max_rpc_timeout_millis": 600000, # default: 30000 - "total_timeout_millis": 600000, # default: 600000 - } - }, - "methods": { - "Publish": { - "retry_codes_name": "publish", - "retry_params_name": "messaging", - } - }, - } - } - } - - publisher = pubsub_v1.PublisherClient(client_config=retry_settings) - topic_path = publisher.topic_path(project_id, topic_id) - - for n in range(1, 10): - data = u"Message number {}".format(n) - # Data must be a bytestring - data = data.encode("utf-8") - future = publisher.publish(topic_path, data=data) - print(future.result()) - - print("Published messages with retry settings.") - # [END pubsub_publisher_retry_settings] - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument("project_id", help="Your Google Cloud project ID") - - subparsers = parser.add_subparsers(dest="command") - subparsers.add_parser("list", help=list_topics.__doc__) - - create_parser = subparsers.add_parser("create", help=create_topic.__doc__) - create_parser.add_argument("topic_id") - - delete_parser = subparsers.add_parser("delete", help=delete_topic.__doc__) - delete_parser.add_argument("topic_id") - - publish_parser = subparsers.add_parser("publish", help=publish_messages.__doc__) - publish_parser.add_argument("topic_id") - - publish_with_custom_attributes_parser = subparsers.add_parser( - "publish-with-custom-attributes", - help=publish_messages_with_custom_attributes.__doc__, - ) - publish_with_custom_attributes_parser.add_argument("topic_id") - - publish_with_error_handler_parser = subparsers.add_parser( - "publish-with-error-handler", help=publish_messages_with_error_handler.__doc__, - ) - publish_with_error_handler_parser.add_argument("topic_id") - - publish_with_batch_settings_parser = subparsers.add_parser( - "publish-with-batch-settings", - help=publish_messages_with_batch_settings.__doc__, - ) - publish_with_batch_settings_parser.add_argument("topic_id") - - publish_with_retry_settings_parser = subparsers.add_parser( - "publish-with-retry-settings", - help=publish_messages_with_retry_settings.__doc__, - ) - publish_with_retry_settings_parser.add_argument("topic_id") - - args = parser.parse_args() - - if args.command == "list": - list_topics(args.project_id) - elif args.command == "create": - create_topic(args.project_id, args.topic_id) - elif args.command == "delete": - delete_topic(args.project_id, args.topic_id) - elif args.command == "publish": - publish_messages(args.project_id, args.topic_id) - elif args.command == "publish-with-custom-attributes": - publish_messages_with_custom_attributes(args.project_id, args.topic_id) - elif args.command == "publish-with-error-handler": - publish_messages_with_error_handler(args.project_id, args.topic_id) - elif args.command == "publish-with-batch-settings": - publish_messages_with_batch_settings(args.project_id, args.topic_id) - elif args.command == "publish-with-retry-settings": - publish_messages_with_retry_settings(args.project_id, args.topic_id) diff --git a/samples/snippets/publisher_test.py b/samples/snippets/publisher_test.py deleted file mode 100644 index b5c2ea1ea..000000000 --- a/samples/snippets/publisher_test.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import time -import uuid - -import backoff -from google.cloud import pubsub_v1 -import mock -import pytest - -import publisher - -UUID = uuid.uuid4().hex -PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] -TOPIC_ADMIN = "publisher-test-topic-admin-" + UUID -TOPIC_PUBLISH = "publisher-test-topic-publish-" + UUID - - -@pytest.fixture -def client(): - yield pubsub_v1.PublisherClient() - - -@pytest.fixture -def topic_admin(client): - topic_path = client.topic_path(PROJECT, TOPIC_ADMIN) - - try: - topic = client.get_topic(topic_path) - except: # noqa - topic = client.create_topic(topic_path) - - yield topic.name - # Teardown of `topic_admin` is handled in `test_delete()`. - - -@pytest.fixture -def topic_publish(client): - topic_path = client.topic_path(PROJECT, TOPIC_PUBLISH) - - try: - topic = client.get_topic(topic_path) - except: # noqa - topic = client.create_topic(topic_path) - - yield topic.name - - client.delete_topic(topic.name) - - -def _make_sleep_patch(): - real_sleep = time.sleep - - def new_sleep(period): - if period == 60: - real_sleep(5) - raise RuntimeError("sigil") - else: - real_sleep(period) - - return mock.patch("time.sleep", new=new_sleep) - - -def test_list(client, topic_admin, capsys): - @backoff.on_exception(backoff.expo, AssertionError, max_time=60) - def eventually_consistent_test(): - publisher.list_topics(PROJECT) - out, _ = capsys.readouterr() - assert topic_admin in out - - eventually_consistent_test() - - -def test_create(client): - topic_path = client.topic_path(PROJECT, TOPIC_ADMIN) - try: - client.delete_topic(topic_path) - except Exception: - pass - - publisher.create_topic(PROJECT, TOPIC_ADMIN) - - @backoff.on_exception(backoff.expo, AssertionError, max_time=60) - def eventually_consistent_test(): - assert client.get_topic(topic_path) - - eventually_consistent_test() - - -def test_delete(client, topic_admin): - publisher.delete_topic(PROJECT, TOPIC_ADMIN) - - @backoff.on_exception(backoff.expo, AssertionError, max_time=60) - def eventually_consistent_test(): - with pytest.raises(Exception): - client.get_topic(client.topic_path(PROJECT, TOPIC_ADMIN)) - - eventually_consistent_test() - - -def test_publish(topic_publish, capsys): - publisher.publish_messages(PROJECT, TOPIC_PUBLISH) - - out, _ = capsys.readouterr() - assert "Published" in out - - -def test_publish_with_custom_attributes(topic_publish, capsys): - publisher.publish_messages_with_custom_attributes(PROJECT, TOPIC_PUBLISH) - - out, _ = capsys.readouterr() - assert "Published" in out - - -def test_publish_with_batch_settings(topic_publish, capsys): - publisher.publish_messages_with_batch_settings(PROJECT, TOPIC_PUBLISH) - - out, _ = capsys.readouterr() - assert "Published" in out - - -def test_publish_with_retry_settings(topic_publish, capsys): - publisher.publish_messages_with_retry_settings(PROJECT, TOPIC_PUBLISH) - - out, _ = capsys.readouterr() - assert "Published" in out - - -def test_publish_with_error_handler(topic_publish, capsys): - publisher.publish_messages_with_error_handler(PROJECT, TOPIC_PUBLISH) - - out, _ = capsys.readouterr() - assert "Published" in out diff --git a/samples/snippets/quickstart/pub.py b/samples/snippets/quickstart/pub.py deleted file mode 100644 index 16432c0c3..000000000 --- a/samples/snippets/quickstart/pub.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START pubsub_quickstart_pub_all] -import argparse -import time - -# [START pubsub_quickstart_pub_deps] -from google.cloud import pubsub_v1 - -# [END pubsub_quickstart_pub_deps] - - -def get_callback(api_future, data, ref): - """Wrap message data in the context of the callback function.""" - - def callback(api_future): - try: - print( - "Published message {} now has message ID {}".format( - data, api_future.result() - ) - ) - ref["num_messages"] += 1 - except Exception: - print( - "A problem occurred when publishing {}: {}\n".format( - data, api_future.exception() - ) - ) - raise - - return callback - - -def pub(project_id, topic_id): - """Publishes a message to a Pub/Sub topic.""" - # [START pubsub_quickstart_pub_client] - # Initialize a Publisher client. - client = pubsub_v1.PublisherClient() - # [END pubsub_quickstart_pub_client] - # Create a fully qualified identifier in the form of - # `projects/{project_id}/topics/{topic_id}` - topic_path = client.topic_path(project_id, topic_id) - - # Data sent to Cloud Pub/Sub must be a bytestring. - data = b"Hello, World!" - - # Keep track of the number of published messages. - ref = dict({"num_messages": 0}) - - # When you publish a message, the client returns a future. - api_future = client.publish(topic_path, data=data) - api_future.add_done_callback(get_callback(api_future, data, ref)) - - # Keep the main thread from exiting while the message future - # gets resolved in the background. - while api_future.running(): - time.sleep(0.5) - print("Published {} message(s).".format(ref["num_messages"])) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument("project_id", help="Google Cloud project ID") - parser.add_argument("topic_id", help="Pub/Sub topic ID") - - args = parser.parse_args() - - pub(args.project_id, args.topic_id) -# [END pubsub_quickstart_pub_all] diff --git a/samples/snippets/quickstart/pub_test.py b/samples/snippets/quickstart/pub_test.py deleted file mode 100644 index 6f5cc06c4..000000000 --- a/samples/snippets/quickstart/pub_test.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import uuid - -from google.api_core.exceptions import AlreadyExists -from google.cloud import pubsub_v1 -import pytest - -import pub # noqa - - -UUID = uuid.uuid4().hex -PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] -TOPIC = "quickstart-pub-test-topic-" + UUID - - -@pytest.fixture(scope="module") -def publisher_client(): - yield pubsub_v1.PublisherClient() - - -@pytest.fixture(scope="module") -def topic(publisher_client): - topic_path = publisher_client.topic_path(PROJECT, TOPIC) - - try: - publisher_client.create_topic(topic_path) - except AlreadyExists: - pass - - yield TOPIC - - publisher_client.delete_topic(topic_path) - - -def test_pub(publisher_client, topic, capsys): - pub.pub(PROJECT, topic) - - out, _ = capsys.readouterr() - - assert "Hello, World!" in out diff --git a/samples/snippets/quickstart/sub.py b/samples/snippets/quickstart/sub.py deleted file mode 100644 index efe008915..000000000 --- a/samples/snippets/quickstart/sub.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# [START pubsub_quickstart_sub_all] -import argparse - -# [START pubsub_quickstart_sub_deps] -from google.cloud import pubsub_v1 - -# [END pubsub_quickstart_sub_deps] - - -def sub(project_id, subscription_id): - """Receives messages from a Pub/Sub subscription.""" - # [START pubsub_quickstart_sub_client] - # Initialize a Subscriber client - subscriber_client = pubsub_v1.SubscriberClient() - # [END pubsub_quickstart_sub_client] - # Create a fully qualified identifier in the form of - # `projects/{project_id}/subscriptions/{subscription_id}` - subscription_path = subscriber_client.subscription_path(project_id, subscription_id) - - def callback(message): - print( - "Received message {} of message ID {}\n".format(message, message.message_id) - ) - # Acknowledge the message. Unack'ed messages will be redelivered. - message.ack() - print("Acknowledged message {}\n".format(message.message_id)) - - streaming_pull_future = subscriber_client.subscribe( - subscription_path, callback=callback - ) - print("Listening for messages on {}..\n".format(subscription_path)) - - try: - # Calling result() on StreamingPullFuture keeps the main thread from - # exiting while messages get processed in the callbacks. - streaming_pull_future.result() - except: # noqa - streaming_pull_future.cancel() - - subscriber_client.close() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument("project_id", help="Google Cloud project ID") - parser.add_argument("subscription_id", help="Pub/Sub subscription ID") - - args = parser.parse_args() - - sub(args.project_id, args.subscription_id) -# [END pubsub_quickstart_sub_all] diff --git a/samples/snippets/quickstart/sub_test.py b/samples/snippets/quickstart/sub_test.py deleted file mode 100644 index 38047422a..000000000 --- a/samples/snippets/quickstart/sub_test.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import uuid - -from google.api_core.exceptions import AlreadyExists -from google.cloud import pubsub_v1 -import mock -import pytest - -import sub # noqa - - -UUID = uuid.uuid4().hex -PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] -TOPIC = "quickstart-sub-test-topic-" + UUID -SUBSCRIPTION = "quickstart-sub-test-topic-sub-" + UUID - -publisher_client = pubsub_v1.PublisherClient() -subscriber_client = pubsub_v1.SubscriberClient() - - -@pytest.fixture(scope="module") -def topic_path(): - topic_path = publisher_client.topic_path(PROJECT, TOPIC) - - try: - topic = publisher_client.create_topic(topic_path) - yield topic.name - except AlreadyExists: - yield topic_path - - publisher_client.delete_topic(topic_path) - - -@pytest.fixture(scope="module") -def subscription_path(topic_path): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION) - - try: - subscription = subscriber_client.create_subscription( - subscription_path, topic_path - ) - yield subscription.name - except AlreadyExists: - yield subscription_path - - subscriber_client.delete_subscription(subscription_path) - subscriber_client.close() - - -def _publish_messages(topic_path): - publish_future = publisher_client.publish(topic_path, data=b"Hello World!") - publish_future.result() - - -def test_sub(monkeypatch, topic_path, subscription_path, capsys): - - real_client = pubsub_v1.SubscriberClient() - mock_client = mock.Mock(spec=pubsub_v1.SubscriberClient, wraps=real_client) - - # Attributes on mock_client_constructor uses the corresponding - # attributes on pubsub_v1.SubscriberClient. - mock_client_constructor = mock.create_autospec(pubsub_v1.SubscriberClient) - mock_client_constructor.return_value = mock_client - - monkeypatch.setattr(pubsub_v1, "SubscriberClient", mock_client_constructor) - - def mock_subscribe(subscription_path, callback=None): - real_future = real_client.subscribe(subscription_path, callback=callback) - mock_future = mock.Mock(spec=real_future, wraps=real_future) - - def mock_result(): - return real_future.result(timeout=10) - - mock_future.result.side_effect = mock_result - return mock_future - - mock_client.subscribe.side_effect = mock_subscribe - - _publish_messages(topic_path) - - sub.sub(PROJECT, SUBSCRIPTION) - - out, _ = capsys.readouterr() - assert "Received message" in out - assert "Acknowledged message" in out - - real_client.close() diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt deleted file mode 100644 index adf26b9f9..000000000 --- a/samples/snippets/requirements-test.txt +++ /dev/null @@ -1,3 +0,0 @@ -backoff==1.10.0 -pytest==5.3.2 -mock==3.0.5 diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py deleted file mode 100644 index f079e7d42..000000000 --- a/samples/snippets/subscriber.py +++ /dev/null @@ -1,783 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This application demonstrates how to perform basic operations on -subscriptions with the Cloud Pub/Sub API. - -For more information, see the README.md under /pubsub and the documentation -at https://cloud.google.com/pubsub/docs. -""" - -import argparse - - -def list_subscriptions_in_topic(project_id, topic_id): - """Lists all subscriptions for a given topic.""" - # [START pubsub_list_topic_subscriptions] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # topic_id = "your-topic-id" - - publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project_id, topic_id) - - for subscription in publisher.list_topic_subscriptions(topic_path): - print(subscription) - # [END pubsub_list_topic_subscriptions] - - -def list_subscriptions_in_project(project_id): - """Lists all subscriptions in the current project.""" - # [START pubsub_list_subscriptions] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - - subscriber = pubsub_v1.SubscriberClient() - project_path = subscriber.project_path(project_id) - - # Wrap the subscriber in a 'with' block to automatically call close() to - # close the underlying gRPC channel when done. - with subscriber: - for subscription in subscriber.list_subscriptions(project_path): - print(subscription.name) - # [END pubsub_list_subscriptions] - - -def create_subscription(project_id, topic_id, subscription_id): - """Create a new pull subscription on the given topic.""" - # [START pubsub_create_pull_subscription] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # topic_id = "your-topic-id" - # subscription_id = "your-subscription-id" - - subscriber = pubsub_v1.SubscriberClient() - topic_path = subscriber.topic_path(project_id, topic_id) - subscription_path = subscriber.subscription_path(project_id, subscription_id) - - # Wrap the subscriber in a 'with' block to automatically call close() to - # close the underlying gRPC channel when done. - with subscriber: - subscription = subscriber.create_subscription(subscription_path, topic_path) - - print("Subscription created: {}".format(subscription)) - # [END pubsub_create_pull_subscription] - - -def create_subscription_with_dead_letter_topic( - project_id, topic_id, subscription_id, dead_letter_topic_id -): - """Create a subscription with dead letter policy.""" - # [START pubsub_dead_letter_create_subscription] - from google.cloud import pubsub_v1 - from google.cloud.pubsub_v1.types import DeadLetterPolicy - - # TODO(developer) - # project_id = "your-project-id" - # endpoint = "https://my-test-project.appspot.com/push" - # TODO(developer): This is an existing topic that the subscription - # with dead letter policy is attached to. - # topic_id = "your-topic-id" - # TODO(developer): This is an existing subscription with a dead letter policy. - # subscription_id = "your-subscription-id" - # TODO(developer): This is an existing dead letter topic that the subscription - # with dead letter policy will forward dead letter messages to. - # dead_letter_topic_id = "your-dead-letter-topic-id" - - subscriber = pubsub_v1.SubscriberClient() - topic_path = subscriber.topic_path(project_id, topic_id) - subscription_path = subscriber.subscription_path(project_id, subscription_id) - dead_letter_topic_path = subscriber.topic_path(project_id, dead_letter_topic_id) - - dead_letter_policy = DeadLetterPolicy( - dead_letter_topic=dead_letter_topic_path, max_delivery_attempts=10 - ) - - with subscriber: - subscription = subscriber.create_subscription( - subscription_path, topic_path, dead_letter_policy=dead_letter_policy - ) - - print("Subscription created: {}".format(subscription.name)) - print( - "It will forward dead letter messages to: {}".format( - subscription.dead_letter_policy.dead_letter_topic - ) - ) - print( - "After {} delivery attempts.".format( - subscription.dead_letter_policy.max_delivery_attempts - ) - ) - # [END pubsub_dead_letter_create_subscription] - - -def create_push_subscription(project_id, topic_id, subscription_id, endpoint): - """Create a new push subscription on the given topic.""" - # [START pubsub_create_push_subscription] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # topic_id = "your-topic-id" - # subscription_id = "your-subscription-id" - # endpoint = "https://my-test-project.appspot.com/push" - - subscriber = pubsub_v1.SubscriberClient() - topic_path = subscriber.topic_path(project_id, topic_id) - subscription_path = subscriber.subscription_path(project_id, subscription_id) - - push_config = pubsub_v1.types.PushConfig(push_endpoint=endpoint) - - # Wrap the subscriber in a 'with' block to automatically call close() to - # close the underlying gRPC channel when done. - with subscriber: - subscription = subscriber.create_subscription( - subscription_path, topic_path, push_config - ) - - print("Push subscription created: {}".format(subscription)) - print("Endpoint for subscription is: {}".format(endpoint)) - # [END pubsub_create_push_subscription] - - -def delete_subscription(project_id, subscription_id): - """Deletes an existing Pub/Sub topic.""" - # [START pubsub_delete_subscription] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # subscription_id = "your-subscription-id" - - subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_id) - - # Wrap the subscriber in a 'with' block to automatically call close() to - # close the underlying gRPC channel when done. - with subscriber: - subscriber.delete_subscription(subscription_path) - - print("Subscription deleted: {}".format(subscription_path)) - # [END pubsub_delete_subscription] - - -def update_push_subscription(project_id, topic_id, subscription_id, endpoint): - """ - Updates an existing Pub/Sub subscription's push endpoint URL. - Note that certain properties of a subscription, such as - its topic, are not modifiable. - """ - # [START pubsub_update_push_configuration] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # topic_id = "your-topic-id" - # subscription_id = "your-subscription-id" - # endpoint = "https://my-test-project.appspot.com/push" - - subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_id) - - push_config = pubsub_v1.types.PushConfig(push_endpoint=endpoint) - - subscription = pubsub_v1.types.Subscription( - name=subscription_path, topic=topic_id, push_config=push_config - ) - - update_mask = {"paths": {"push_config"}} - - # Wrap the subscriber in a 'with' block to automatically call close() to - # close the underlying gRPC channel when done. - with subscriber: - result = subscriber.update_subscription(subscription, update_mask) - - print("Subscription updated: {}".format(subscription_path)) - print("New endpoint for subscription is: {}".format(result.push_config)) - # [END pubsub_update_push_configuration] - - -def update_subscription_with_dead_letter_policy( - project_id, topic_id, subscription_id, dead_letter_topic_id -): - """Update a subscription's dead letter policy.""" - # [START pubsub_dead_letter_update_subscription] - from google.cloud import pubsub_v1 - from google.cloud.pubsub_v1.types import DeadLetterPolicy, FieldMask - - # TODO(developer) - # project_id = "your-project-id" - # TODO(developer): This is an existing topic that the subscription - # with dead letter policy is attached to. - # topic_id = "your-topic-id" - # TODO(developer): This is an existing subscription with a dead letter policy. - # subscription_id = "your-subscription-id" - # TODO(developer): This is an existing dead letter topic that the subscription - # with dead letter policy will forward dead letter messages to. - # dead_letter_topic_id = "your-dead-letter-topic-id" - - subscriber = pubsub_v1.SubscriberClient() - topic_path = subscriber.topic_path(project_id, topic_id) - subscription_path = subscriber.subscription_path(project_id, subscription_id) - dead_letter_topic_path = subscriber.topic_path(project_id, dead_letter_topic_id) - - subscription_before_update = subscriber.get_subscription(subscription_path) - print("Before the update: {}".format(subscription_before_update)) - - # Indicates which fields in the provided subscription to update. - update_mask = FieldMask(paths=["dead_letter_policy.max_delivery_attempts"]) - - # Construct a dead letter policy you expect to have after the update. - dead_letter_policy = DeadLetterPolicy( - dead_letter_topic=dead_letter_topic_path, max_delivery_attempts=20 - ) - - # Construct the subscription with the dead letter policy you expect to have - # after the update. Here, values in the required fields (name, topic) help - # identify the subscription. - subscription = pubsub_v1.types.Subscription( - name=subscription_path, topic=topic_path, dead_letter_policy=dead_letter_policy, - ) - - with subscriber: - subscription_after_update = subscriber.update_subscription( - subscription, update_mask - ) - - print("After the update: {}".format(subscription_after_update)) - # [END pubsub_dead_letter_update_subscription] - return subscription_after_update - - -def remove_dead_letter_policy(project_id, topic_id, subscription_id): - """Remove dead letter policy from a subscription.""" - # [START pubsub_dead_letter_remove] - from google.cloud import pubsub_v1 - from google.cloud.pubsub_v1.types import FieldMask - - # TODO(developer) - # project_id = "your-project-id" - # TODO(developer): This is an existing topic that the subscription - # with dead letter policy is attached to. - # topic_id = "your-topic-id" - # TODO(developer): This is an existing subscription with a dead letter policy. - # subscription_id = "your-subscription-id" - - subscriber = pubsub_v1.SubscriberClient() - topic_path = subscriber.topic_path(project_id, topic_id) - subscription_path = subscriber.subscription_path(project_id, subscription_id) - - subscription_before_update = subscriber.get_subscription(subscription_path) - print("Before removing the policy: {}".format(subscription_before_update)) - - # Indicates which fields in the provided subscription to update. - update_mask = FieldMask( - paths=[ - "dead_letter_policy.dead_letter_topic", - "dead_letter_policy.max_delivery_attempts", - ] - ) - - # Construct the subscription (without any dead letter policy) that you - # expect to have after the update. - subscription = pubsub_v1.types.Subscription( - name=subscription_path, topic=topic_path - ) - - with subscriber: - subscription_after_update = subscriber.update_subscription( - subscription, update_mask - ) - - print("After removing the policy: {}".format(subscription_after_update)) - # [END pubsub_dead_letter_remove] - return subscription_after_update - - -def receive_messages(project_id, subscription_id, timeout=None): - """Receives messages from a pull subscription.""" - # [START pubsub_subscriber_async_pull] - # [START pubsub_quickstart_subscriber] - from concurrent.futures import TimeoutError - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # subscription_id = "your-subscription-id" - # Number of seconds the subscriber should listen for messages - # timeout = 5.0 - - subscriber = pubsub_v1.SubscriberClient() - # The `subscription_path` method creates a fully qualified identifier - # in the form `projects/{project_id}/subscriptions/{subscription_id}` - subscription_path = subscriber.subscription_path(project_id, subscription_id) - - def callback(message): - print("Received message: {}".format(message)) - message.ack() - - streaming_pull_future = subscriber.subscribe(subscription_path, callback=callback) - print("Listening for messages on {}..\n".format(subscription_path)) - - # Wrap subscriber in a 'with' block to automatically call close() when done. - with subscriber: - try: - # When `timeout` is not set, result() will block indefinitely, - # unless an exception is encountered first. - streaming_pull_future.result(timeout=timeout) - except TimeoutError: - streaming_pull_future.cancel() - # [END pubsub_subscriber_async_pull] - # [END pubsub_quickstart_subscriber] - - -def receive_messages_with_custom_attributes(project_id, subscription_id, timeout=None): - """Receives messages from a pull subscription.""" - # [START pubsub_subscriber_async_pull_custom_attributes] - from concurrent.futures import TimeoutError - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # subscription_id = "your-subscription-id" - # Number of seconds the subscriber should listen for messages - # timeout = 5.0 - - subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_id) - - def callback(message): - print("Received message: {}".format(message.data)) - if message.attributes: - print("Attributes:") - for key in message.attributes: - value = message.attributes.get(key) - print("{}: {}".format(key, value)) - message.ack() - - streaming_pull_future = subscriber.subscribe(subscription_path, callback=callback) - print("Listening for messages on {}..\n".format(subscription_path)) - - # Wrap subscriber in a 'with' block to automatically call close() when done. - with subscriber: - try: - # When `timeout` is not set, result() will block indefinitely, - # unless an exception is encountered first. - streaming_pull_future.result(timeout=timeout) - except TimeoutError: - streaming_pull_future.cancel() - # [END pubsub_subscriber_async_pull_custom_attributes] - - -def receive_messages_with_flow_control(project_id, subscription_id, timeout=None): - """Receives messages from a pull subscription with flow control.""" - # [START pubsub_subscriber_flow_settings] - from concurrent.futures import TimeoutError - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # subscription_id = "your-subscription-id" - # Number of seconds the subscriber should listen for messages - # timeout = 5.0 - - subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_id) - - def callback(message): - print("Received message: {}".format(message.data)) - message.ack() - - # Limit the subscriber to only have ten outstanding messages at a time. - flow_control = pubsub_v1.types.FlowControl(max_messages=10) - - streaming_pull_future = subscriber.subscribe( - subscription_path, callback=callback, flow_control=flow_control - ) - print("Listening for messages on {}..\n".format(subscription_path)) - - # Wrap subscriber in a 'with' block to automatically call close() when done. - with subscriber: - try: - # When `timeout` is not set, result() will block indefinitely, - # unless an exception is encountered first. - streaming_pull_future.result(timeout=timeout) - except TimeoutError: - streaming_pull_future.cancel() - # [END pubsub_subscriber_flow_settings] - - -def synchronous_pull(project_id, subscription_id): - """Pulling messages synchronously.""" - # [START pubsub_subscriber_sync_pull] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # subscription_id = "your-subscription-id" - - subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_id) - - NUM_MESSAGES = 3 - - # Wrap the subscriber in a 'with' block to automatically call close() to - # close the underlying gRPC channel when done. - with subscriber: - # The subscriber pulls a specific number of messages. - response = subscriber.pull(subscription_path, max_messages=NUM_MESSAGES) - - ack_ids = [] - for received_message in response.received_messages: - print("Received: {}".format(received_message.message.data)) - ack_ids.append(received_message.ack_id) - - # Acknowledges the received messages so they will not be sent again. - subscriber.acknowledge(subscription_path, ack_ids) - - print( - "Received and acknowledged {} messages. Done.".format( - len(response.received_messages) - ) - ) - # [END pubsub_subscriber_sync_pull] - - -def synchronous_pull_with_lease_management(project_id, subscription_id): - """Pulling messages synchronously with lease management""" - # [START pubsub_subscriber_sync_pull_with_lease] - import logging - import multiprocessing - import random - import time - - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # subscription_id = "your-subscription-id" - - subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_id) - - NUM_MESSAGES = 2 - ACK_DEADLINE = 30 - SLEEP_TIME = 10 - - # The subscriber pulls a specific number of messages. - response = subscriber.pull(subscription_path, max_messages=NUM_MESSAGES) - - multiprocessing.log_to_stderr() - logger = multiprocessing.get_logger() - logger.setLevel(logging.INFO) - - def worker(msg): - """Simulates a long-running process.""" - RUN_TIME = random.randint(1, 60) - logger.info( - "{}: Running {} for {}s".format( - time.strftime("%X", time.gmtime()), msg.message.data, RUN_TIME - ) - ) - time.sleep(RUN_TIME) - - # `processes` stores process as key and ack id and message as values. - processes = dict() - for message in response.received_messages: - process = multiprocessing.Process(target=worker, args=(message,)) - processes[process] = (message.ack_id, message.message.data) - process.start() - - while processes: - for process in list(processes): - ack_id, msg_data = processes[process] - # If the process is still running, reset the ack deadline as - # specified by ACK_DEADLINE once every while as specified - # by SLEEP_TIME. - if process.is_alive(): - # `ack_deadline_seconds` must be between 10 to 600. - subscriber.modify_ack_deadline( - subscription_path, [ack_id], ack_deadline_seconds=ACK_DEADLINE, - ) - logger.info( - "{}: Reset ack deadline for {} for {}s".format( - time.strftime("%X", time.gmtime()), msg_data, ACK_DEADLINE, - ) - ) - - # If the processs is finished, acknowledges using `ack_id`. - else: - subscriber.acknowledge(subscription_path, [ack_id]) - logger.info( - "{}: Acknowledged {}".format( - time.strftime("%X", time.gmtime()), msg_data - ) - ) - processes.pop(process) - - # If there are still processes running, sleeps the thread. - if processes: - time.sleep(SLEEP_TIME) - - print( - "Received and acknowledged {} messages. Done.".format( - len(response.received_messages) - ) - ) - - # Close the underlying gPRC channel. Alternatively, wrap subscriber in - # a 'with' block to automatically call close() when done. - subscriber.close() - # [END pubsub_subscriber_sync_pull_with_lease] - - -def listen_for_errors(project_id, subscription_id, timeout=None): - """Receives messages and catches errors from a pull subscription.""" - # [START pubsub_subscriber_error_listener] - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # subscription_id = "your-subscription-id" - # Number of seconds the subscriber should listen for messages - # timeout = 5.0 - - subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_id) - - def callback(message): - print("Received message: {}".format(message)) - message.ack() - - streaming_pull_future = subscriber.subscribe(subscription_path, callback=callback) - print("Listening for messages on {}..\n".format(subscription_path)) - - # Wrap subscriber in a 'with' block to automatically call close() when done. - with subscriber: - # When `timeout` is not set, result() will block indefinitely, - # unless an exception is encountered first. - try: - streaming_pull_future.result(timeout=timeout) - except Exception as e: - streaming_pull_future.cancel() - print( - "Listening for messages on {} threw an exception: {}.".format( - subscription_id, e - ) - ) - # [END pubsub_subscriber_error_listener] - - -def receive_messages_with_delivery_attempts(project_id, subscription_id, timeout=None): - # [START pubsub_dead_letter_delivery_attempt] - from concurrent.futures import TimeoutError - from google.cloud import pubsub_v1 - - # TODO(developer) - # project_id = "your-project-id" - # subscription_id = "your-subscription-id" - - subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_id) - - def callback(message): - print("Received message: {}".format(message)) - print("With delivery attempts: {}".format(message.delivery_attempt)) - message.ack() - - streaming_pull_future = subscriber.subscribe(subscription_path, callback=callback) - print("Listening for messages on {}..\n".format(subscription_path)) - - # Wrap subscriber in a 'with' block to automatically call close() when done. - with subscriber: - # When `timeout` is not set, result() will block indefinitely, - # unless an exception is encountered first. - try: - streaming_pull_future.result(timeout=timeout) - except TimeoutError: - streaming_pull_future.cancel() - # [END pubsub_dead_letter_delivery_attempt] - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument("project_id", help="Your Google Cloud project ID") - - subparsers = parser.add_subparsers(dest="command") - list_in_topic_parser = subparsers.add_parser( - "list-in-topic", help=list_subscriptions_in_topic.__doc__ - ) - list_in_topic_parser.add_argument("topic_id") - - list_in_project_parser = subparsers.add_parser( - "list-in-project", help=list_subscriptions_in_project.__doc__ - ) - - create_parser = subparsers.add_parser("create", help=create_subscription.__doc__) - create_parser.add_argument("topic_id") - create_parser.add_argument("subscription_id") - - create_with_dead_letter_policy_parser = subparsers.add_parser( - "create-with-dead-letter-policy", - help=create_subscription_with_dead_letter_topic.__doc__, - ) - create_with_dead_letter_policy_parser.add_argument("topic_id") - create_with_dead_letter_policy_parser.add_argument("subscription_id") - create_with_dead_letter_policy_parser.add_argument("dead_letter_topic_id") - - create_push_parser = subparsers.add_parser( - "create-push", help=create_push_subscription.__doc__ - ) - create_push_parser.add_argument("topic_id") - create_push_parser.add_argument("subscription_id") - create_push_parser.add_argument("endpoint") - - delete_parser = subparsers.add_parser("delete", help=delete_subscription.__doc__) - delete_parser.add_argument("subscription_id") - - update_push_parser = subparsers.add_parser( - "update-push", help=update_push_subscription.__doc__ - ) - update_push_parser.add_argument("topic_id") - update_push_parser.add_argument("subscription_id") - update_push_parser.add_argument("endpoint") - - update_dead_letter_policy_parser = subparsers.add_parser( - "update-dead-letter-policy", - help=update_subscription_with_dead_letter_policy.__doc__, - ) - update_dead_letter_policy_parser.add_argument("topic_id") - update_dead_letter_policy_parser.add_argument("subscription_id") - update_dead_letter_policy_parser.add_argument("dead_letter_topic_id") - - remove_dead_letter_policy_parser = subparsers.add_parser( - "remove-dead-letter-policy", help=remove_dead_letter_policy.__doc__ - ) - remove_dead_letter_policy_parser.add_argument("topic_id") - remove_dead_letter_policy_parser.add_argument("subscription_id") - - receive_parser = subparsers.add_parser("receive", help=receive_messages.__doc__) - receive_parser.add_argument("subscription_id") - receive_parser.add_argument("timeout", default=None, type=float, nargs="?") - - receive_with_custom_attributes_parser = subparsers.add_parser( - "receive-custom-attributes", - help=receive_messages_with_custom_attributes.__doc__, - ) - receive_with_custom_attributes_parser.add_argument("subscription_id") - receive_with_custom_attributes_parser.add_argument( - "timeout", default=None, type=float, nargs="?" - ) - - receive_with_flow_control_parser = subparsers.add_parser( - "receive-flow-control", help=receive_messages_with_flow_control.__doc__ - ) - receive_with_flow_control_parser.add_argument("subscription_id") - receive_with_flow_control_parser.add_argument( - "timeout", default=None, type=float, nargs="?" - ) - - synchronous_pull_parser = subparsers.add_parser( - "receive-synchronously", help=synchronous_pull.__doc__ - ) - synchronous_pull_parser.add_argument("subscription_id") - - synchronous_pull_with_lease_management_parser = subparsers.add_parser( - "receive-synchronously-with-lease", - help=synchronous_pull_with_lease_management.__doc__, - ) - synchronous_pull_with_lease_management_parser.add_argument("subscription_id") - - listen_for_errors_parser = subparsers.add_parser( - "listen-for-errors", help=listen_for_errors.__doc__ - ) - listen_for_errors_parser.add_argument("subscription_id") - listen_for_errors_parser.add_argument( - "timeout", default=None, type=float, nargs="?" - ) - - receive_messages_with_delivery_attempts_parser = subparsers.add_parser( - "receive-messages-with-delivery-attempts", - help=receive_messages_with_delivery_attempts.__doc__, - ) - receive_messages_with_delivery_attempts_parser.add_argument("subscription_id") - receive_messages_with_delivery_attempts_parser.add_argument( - "timeout", default=None, type=float, nargs="?" - ) - - args = parser.parse_args() - - if args.command == "list-in-topic": - list_subscriptions_in_topic(args.project_id, args.topic_id) - elif args.command == "list-in-project": - list_subscriptions_in_project(args.project_id) - elif args.command == "create": - create_subscription(args.project_id, args.topic_id, args.subscription_id) - elif args.command == "create-with-dead-letter-policy": - create_subscription_with_dead_letter_topic( - args.project_id, - args.topic_id, - args.subscription_id, - args.dead_letter_topic_id, - ) - elif args.command == "create-push": - create_push_subscription( - args.project_id, args.topic_id, args.subscription_id, args.endpoint, - ) - elif args.command == "delete": - delete_subscription(args.project_id, args.subscription_id) - elif args.command == "update-push": - update_push_subscription( - args.project_id, args.topic_id, args.subscription_id, args.endpoint, - ) - elif args.command == "update-dead-letter-policy": - update_subscription_with_dead_letter_policy( - args.project_id, - args.topic_id, - args.subscription_id, - args.dead_letter_topic_id, - ) - elif args.command == "remove-dead-letter-policy": - remove_dead_letter_policy(args.project_id, args.topic_id, args.subscription_id) - elif args.command == "receive": - receive_messages(args.project_id, args.subscription_id, args.timeout) - elif args.command == "receive-custom-attributes": - receive_messages_with_custom_attributes( - args.project_id, args.subscription_id, args.timeout - ) - elif args.command == "receive-flow-control": - receive_messages_with_flow_control( - args.project_id, args.subscription_id, args.timeout - ) - elif args.command == "receive-synchronously": - synchronous_pull(args.project_id, args.subscription_id) - elif args.command == "receive-synchronously-with-lease": - synchronous_pull_with_lease_management(args.project_id, args.subscription_id) - elif args.command == "listen-for-errors": - listen_for_errors(args.project_id, args.subscription_id, args.timeout) - elif args.command == "receive-messages-with-delivery-attempts": - receive_messages_with_delivery_attempts( - args.project_id, args.subscription_id, args.timeout - ) diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py deleted file mode 100644 index a7f7c139c..000000000 --- a/samples/snippets/subscriber_test.py +++ /dev/null @@ -1,341 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import uuid - -import backoff -from google.cloud import pubsub_v1 -import pytest - -import subscriber - -UUID = uuid.uuid4().hex -PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] -TOPIC = "subscription-test-topic-" + UUID -DEAD_LETTER_TOPIC = "subscription-test-dead-letter-topic-" + UUID -SUBSCRIPTION_ADMIN = "subscription-test-subscription-admin-" + UUID -SUBSCRIPTION_ASYNC = "subscription-test-subscription-async-" + UUID -SUBSCRIPTION_SYNC = "subscription-test-subscription-sync-" + UUID -SUBSCRIPTION_DLQ = "subscription-test-subscription-dlq-" + UUID -ENDPOINT = "https://{}.appspot.com/push".format(PROJECT) -NEW_ENDPOINT = "https://{}.appspot.com/push2".format(PROJECT) - - -@pytest.fixture(scope="module") -def publisher_client(): - yield pubsub_v1.PublisherClient() - - -@pytest.fixture(scope="module") -def topic(publisher_client): - topic_path = publisher_client.topic_path(PROJECT, TOPIC) - - try: - topic = publisher_client.get_topic(topic_path) - except: # noqa - topic = publisher_client.create_topic(topic_path) - - yield topic.name - - publisher_client.delete_topic(topic.name) - - -@pytest.fixture(scope="module") -def dead_letter_topic(publisher_client): - topic_path = publisher_client.topic_path(PROJECT, DEAD_LETTER_TOPIC) - - try: - dead_letter_topic = publisher_client.get_topic(topic_path) - except: # noqa - dead_letter_topic = publisher_client.create_topic(topic_path) - - yield dead_letter_topic.name - - publisher_client.delete_topic(dead_letter_topic.name) - - -@pytest.fixture(scope="module") -def subscriber_client(): - subscriber_client = pubsub_v1.SubscriberClient() - yield subscriber_client - subscriber_client.close() - - -@pytest.fixture(scope="module") -def subscription_admin(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_ADMIN) - - try: - subscription = subscriber_client.get_subscription(subscription_path) - except: # noqa - subscription = subscriber_client.create_subscription( - subscription_path, topic=topic - ) - - yield subscription.name - - -@pytest.fixture(scope="module") -def subscription_sync(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_SYNC) - - try: - subscription = subscriber_client.get_subscription(subscription_path) - except: # noqa - subscription = subscriber_client.create_subscription( - subscription_path, topic=topic - ) - - yield subscription.name - - subscriber_client.delete_subscription(subscription.name) - - -@pytest.fixture(scope="module") -def subscription_async(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_ASYNC) - - try: - subscription = subscriber_client.get_subscription(subscription_path) - except: # noqa - subscription = subscriber_client.create_subscription( - subscription_path, topic=topic - ) - - yield subscription.name - - subscriber_client.delete_subscription(subscription.name) - - -@pytest.fixture(scope="module") -def subscription_dlq(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_DLQ) - - try: - subscription = subscriber_client.get_subscription(subscription_path) - except: # noqa - subscription = subscriber_client.create_subscription( - subscription_path, topic=topic - ) - - yield subscription.name - - subscriber_client.delete_subscription(subscription.name) - - -def test_list_in_topic(subscription_admin, capsys): - @backoff.on_exception(backoff.expo, AssertionError, max_time=60) - def eventually_consistent_test(): - subscriber.list_subscriptions_in_topic(PROJECT, TOPIC) - out, _ = capsys.readouterr() - assert subscription_admin in out - - eventually_consistent_test() - - -def test_list_in_project(subscription_admin, capsys): - @backoff.on_exception(backoff.expo, AssertionError, max_time=60) - def eventually_consistent_test(): - subscriber.list_subscriptions_in_project(PROJECT) - out, _ = capsys.readouterr() - assert subscription_admin in out - - eventually_consistent_test() - - -def test_create(subscriber_client): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_ADMIN) - - try: - subscriber_client.delete_subscription(subscription_path) - except Exception: - pass - - subscriber.create_subscription(PROJECT, TOPIC, SUBSCRIPTION_ADMIN) - - @backoff.on_exception(backoff.expo, AssertionError, max_time=60) - def eventually_consistent_test(): - assert subscriber_client.get_subscription(subscription_path) - - eventually_consistent_test() - - -def test_create_subscription_with_dead_letter_policy( - subscriber_client, publisher_client, topic, dead_letter_topic, capsys -): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_DLQ) - dead_letter_topic_path = publisher_client.topic_path(PROJECT, DEAD_LETTER_TOPIC) - - try: - subscriber_client.delete_subscription(subscription_path) - except Exception: - pass - - subscriber.create_subscription_with_dead_letter_topic( - PROJECT, TOPIC, SUBSCRIPTION_DLQ, DEAD_LETTER_TOPIC - ) - - out, _ = capsys.readouterr() - assert "Subscription created: " + subscription_path in out - assert "It will forward dead letter messages to: " + dead_letter_topic_path in out - assert "After 10 delivery attempts." in out - - -def test_create_push(subscriber_client): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_ADMIN) - try: - subscriber_client.delete_subscription(subscription_path) - except Exception: - pass - - subscriber.create_push_subscription(PROJECT, TOPIC, SUBSCRIPTION_ADMIN, ENDPOINT) - - @backoff.on_exception(backoff.expo, AssertionError, max_time=60) - def eventually_consistent_test(): - assert subscriber_client.get_subscription(subscription_path) - - eventually_consistent_test() - - -def test_update(subscriber_client, subscription_admin, capsys): - subscriber.update_push_subscription( - PROJECT, TOPIC, SUBSCRIPTION_ADMIN, NEW_ENDPOINT - ) - - out, _ = capsys.readouterr() - assert "Subscription updated" in out - - -def test_update_dead_letter_policy( - subscriber_client, topic, subscription_dlq, dead_letter_topic, capsys -): - _ = subscriber.update_subscription_with_dead_letter_policy( - PROJECT, TOPIC, SUBSCRIPTION_DLQ, DEAD_LETTER_TOPIC - ) - - out, _ = capsys.readouterr() - assert "max_delivery_attempts: 20" in out - - -def test_delete(subscriber_client, subscription_admin): - subscriber.delete_subscription(PROJECT, SUBSCRIPTION_ADMIN) - - @backoff.on_exception(backoff.expo, AssertionError, max_time=60) - def eventually_consistent_test(): - with pytest.raises(Exception): - subscriber_client.get_subscription(subscription_admin) - - eventually_consistent_test() - - -def _publish_messages(publisher_client, topic): - for n in range(5): - data = u"message {}".format(n).encode("utf-8") - publish_future = publisher_client.publish( - topic, data=data, origin="python-sample" - ) - publish_future.result() - - -def test_receive(publisher_client, topic, subscription_async, capsys): - _publish_messages(publisher_client, topic) - - subscriber.receive_messages(PROJECT, SUBSCRIPTION_ASYNC, 5) - - out, _ = capsys.readouterr() - assert "Listening" in out - assert subscription_async in out - assert "message" in out - - -def test_receive_with_custom_attributes( - publisher_client, topic, subscription_async, capsys -): - - _publish_messages(publisher_client, topic) - - subscriber.receive_messages_with_custom_attributes(PROJECT, SUBSCRIPTION_ASYNC, 5) - - out, _ = capsys.readouterr() - assert "message" in out - assert "origin" in out - assert "python-sample" in out - - -def test_receive_with_flow_control(publisher_client, topic, subscription_async, capsys): - - _publish_messages(publisher_client, topic) - - subscriber.receive_messages_with_flow_control(PROJECT, SUBSCRIPTION_ASYNC, 5) - - out, _ = capsys.readouterr() - assert "Listening" in out - assert subscription_async in out - assert "message" in out - - -def test_receive_synchronously(publisher_client, topic, subscription_sync, capsys): - _publish_messages(publisher_client, topic) - - subscriber.synchronous_pull(PROJECT, SUBSCRIPTION_SYNC) - - out, _ = capsys.readouterr() - assert "Done." in out - - -def test_receive_synchronously_with_lease( - publisher_client, topic, subscription_sync, capsys -): - _publish_messages(publisher_client, topic) - - subscriber.synchronous_pull_with_lease_management(PROJECT, SUBSCRIPTION_SYNC) - - out, _ = capsys.readouterr() - assert "Done." in out - - -def test_listen_for_errors(publisher_client, topic, subscription_async, capsys): - - _publish_messages(publisher_client, topic) - - subscriber.listen_for_errors(PROJECT, SUBSCRIPTION_ASYNC, 5) - - out, _ = capsys.readouterr() - assert "Listening" in out - assert subscription_async in out - assert "threw an exception" in out - - -def test_receive_with_delivery_attempts( - publisher_client, topic, subscription_dlq, dead_letter_topic, capsys -): - _publish_messages(publisher_client, topic) - - subscriber.receive_messages_with_delivery_attempts(PROJECT, SUBSCRIPTION_DLQ, 10) - - out, _ = capsys.readouterr() - assert "Listening" in out - assert subscription_dlq in out - assert "Received message: " in out - assert "message 4" in out - assert "With delivery attempts: " in out - - -def test_remove_dead_letter_policy(subscriber_client, subscription_dlq): - subscription_after_update = subscriber.remove_dead_letter_policy( - PROJECT, TOPIC, SUBSCRIPTION_DLQ - ) - - assert subscription_after_update.dead_letter_policy.dead_letter_topic == "" diff --git a/synth.py b/synth.py index 0e2c96e42..b44cc0acf 100644 --- a/synth.py +++ b/synth.py @@ -18,7 +18,6 @@ import synthtool as s from synthtool import gcp -from synthtool.languages import python gapic = gcp.GAPICBazel() common = gcp.CommonTemplates() @@ -267,16 +266,8 @@ def _merge_dict(d1, d2): # Add templated files # ---------------------------------------------------------------------------- templated_files = gcp.CommonTemplates().py_library( - unit_cov_level=97, - cov_level=99, - system_test_external_dependencies=["psutil"], - samples=True, + unit_cov_level=97, cov_level=99, system_test_external_dependencies=["psutil"], ) s.move(templated_files) -# ---------------------------------------------------------------------------- -# Samples templates -# ---------------------------------------------------------------------------- -python.py_samples() - s.shell.run(["nox", "-s", "blacken"], hide_output=False) From 7bbaba00ca2febc811f8af7b977108b219f06b8d Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Wed, 24 Aug 2016 14:51:23 -0700 Subject: [PATCH 002/101] Add pubsub publisher and subscriber samples Change-Id: I38b90c10aef72c37188c4520897302933b9d2ea7 --- samples/snippets/README.md | 16 ++++ samples/snippets/publisher.py | 107 +++++++++++++++++++++++ samples/snippets/publisher_test.py | 67 +++++++++++++++ samples/snippets/requirements.txt | 2 +- samples/snippets/subscriber.py | 127 ++++++++++++++++++++++++++++ samples/snippets/subscriber_test.py | 83 ++++++++++++++++++ 6 files changed, 401 insertions(+), 1 deletion(-) create mode 100644 samples/snippets/README.md create mode 100644 samples/snippets/publisher.py create mode 100644 samples/snippets/publisher_test.py create mode 100644 samples/snippets/subscriber.py create mode 100644 samples/snippets/subscriber_test.py diff --git a/samples/snippets/README.md b/samples/snippets/README.md new file mode 100644 index 000000000..8691c9017 --- /dev/null +++ b/samples/snippets/README.md @@ -0,0 +1,16 @@ +# Google Cloud Pub/Sub Samples + + + + +## Prerequisites + +All samples require a [Google Cloud Project](https://console.cloud.google.com). + +Use the [Cloud SDK](https://cloud.google.com/sdk) to provide authentication: + + gcloud beta auth application-default login + +Run the samples: + + TODO diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py new file mode 100644 index 000000000..61387d67e --- /dev/null +++ b/samples/snippets/publisher.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform basic operations on topics +with the Cloud Pub/Sub API. + +For more information, see the README.md under /pubsub and the documentation +at https://cloud.google.com/pubsub/docs. +""" + +import argparse + +from gcloud import pubsub + + +def list_topics(): + """Lists all Pub/Sub topics in the current project.""" + pubsub_client = pubsub.Client() + + topics = [] + next_page_token = None + while True: + page, next_page_token = pubsub_client.list_topics() + topics.extend(page) + if not next_page_token: + break + + for topic in topics: + print(topic.name) + + +def create_topic(topic_name): + """Create a new Pub/Sub topic.""" + pubsub_client = pubsub.Client() + topic = pubsub_client.topic(topic_name) + + topic.create() + + print('Topic {} created.'.format(topic.name)) + + +def delete_topic(topic_name): + """Deletes an existing Pub/Sub topic.""" + pubsub_client = pubsub.Client() + topic = pubsub_client.topic(topic_name) + + topic.delete() + + print('Topic {} deleted.'.format(topic.name)) + + +def publish_message(topic_name, data): + """Publishes a message to a Pub/Sub topic with the given data.""" + pubsub_client = pubsub.Client() + topic = pubsub_client.topic(topic_name) + + # Data must be a bytestring + data = data.encode('utf-8') + + message_id = topic.publish(data) + + print('Message {} published.'.format(message_id)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + + subparsers = parser.add_subparsers(dest='command') + subparsers.add_parser('list', help=list_topics.__doc__) + + create_parser = subparsers.add_parser('create', help=create_topic.__doc__) + create_parser.add_argument('topic_name') + + delete_parser = subparsers.add_parser('delete', help=delete_topic.__doc__) + delete_parser.add_argument('topic_name') + + publish_parser = subparsers.add_parser( + 'publish', help=publish_message.__doc__) + publish_parser.add_argument('topic_name') + publish_parser.add_argument('data') + + args = parser.parse_args() + + if args.command == 'list': + list_topics() + elif args.command == 'create': + create_topic(args.topic_name) + elif args.command == 'delete': + delete_topic(args.topic_name) + elif args.command == 'publish': + publish_message(args.topic_name, args.data) diff --git a/samples/snippets/publisher_test.py b/samples/snippets/publisher_test.py new file mode 100644 index 000000000..3cce3c962 --- /dev/null +++ b/samples/snippets/publisher_test.py @@ -0,0 +1,67 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from gcloud import pubsub +from gcp.testing import eventually_consistent +import pytest + +import publisher + +TEST_TOPIC = 'publisher-test-topic' + + +@pytest.fixture +def test_topic(): + client = pubsub.Client() + topic = client.topic(TEST_TOPIC) + yield topic + if topic.exists(): + topic.delete() + + +def test_list(test_topic, capsys): + test_topic.create() + + @eventually_consistent.call + def _(): + publisher.list_topics() + out, _ = capsys.readouterr() + assert test_topic.name in out + + +def test_create(test_topic): + publisher.create_topic(test_topic.name) + + @eventually_consistent.call + def _(): + assert test_topic.exists() + + +def test_delete(test_topic): + test_topic.create() + + publisher.delete_topic(test_topic.name) + + @eventually_consistent.call + def _(): + assert not test_topic.exists() + + +def test_publish(test_topic, capsys): + test_topic.create() + + publisher.publish_message(test_topic.name, 'hello') + + out, _ = capsys.readouterr() + assert 'published' in out diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 42ab449b1..2beeafe63 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==1.6.1 +gcloud==0.18.1 diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py new file mode 100644 index 000000000..2b3371cd6 --- /dev/null +++ b/samples/snippets/subscriber.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform basic operations on +subscriptions with the Cloud Pub/Sub API. + +For more information, see the README.md under /pubsub and the documentation +at https://cloud.google.com/pubsub/docs. +""" + +import argparse + +from gcloud import pubsub + + +def list_subscriptions(topic_name): + """Lists all subscriptions for a given topic.""" + pubsub_client = pubsub.Client() + topic = pubsub_client.topic(topic_name) + + subscriptions = [] + next_page_token = None + while True: + page, next_page_token = topic.list_subscriptions() + subscriptions.extend(page) + if not next_page_token: + break + + for subscription in subscriptions: + print(subscription.name) + + +def create_subscription(topic_name, subscription_name): + """Create a new pull subscription on the given topic.""" + pubsub_client = pubsub.Client() + topic = pubsub_client.topic(topic_name) + + subscription = topic.subscription(subscription_name) + subscription.create() + + print('Subscription {} created on topic {}.'.format( + subscription.name, topic.name)) + + +def delete_subscription(topic_name, subscription_name): + """Deletes an existing Pub/Sub topic.""" + pubsub_client = pubsub.Client() + topic = pubsub_client.topic(topic_name) + subscription = topic.subscription(subscription_name) + + subscription.delete() + + print('Subscription {} deleted on topic {}.'.format( + subscription.name, topic.name)) + + +def receive_message(topic_name, subscription_name): + """Receives a message from a pull subscription.""" + pubsub_client = pubsub.Client() + topic = pubsub_client.topic(topic_name) + subscription = topic.subscription(subscription_name) + + # Change return_immediately=False to block until messages are + # received. + results = subscription.pull(return_immediately=True) + + print('Received {} messages.'.format(len(results))) + + for ack_id, message in results: + print('* {}: {}, {}'.format( + message.message_id, message.data, message.attributes)) + + # Acknowledge received messages. If you do not acknowledge, Pub/Sub will + # redeliver the message. + if results: + subscription.acknowledge([ack_id for ack_id, message in results]) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + + subparsers = parser.add_subparsers(dest='command') + list_parser = subparsers.add_parser( + 'list', help=list_subscriptions.__doc__) + list_parser.add_argument('topic_name') + + create_parser = subparsers.add_parser( + 'create', help=create_subscription.__doc__) + create_parser.add_argument('topic_name') + create_parser.add_argument('subscription_name') + + delete_parser = subparsers.add_parser( + 'delete', help=delete_subscription.__doc__) + delete_parser.add_argument('topic_name') + delete_parser.add_argument('subscription_name') + + receive_parser = subparsers.add_parser( + 'receive', help=receive_message.__doc__) + receive_parser.add_argument('topic_name') + receive_parser.add_argument('subscription_name') + + args = parser.parse_args() + + if args.command == 'list': + list_subscriptions(args.topic_name) + elif args.command == 'create': + create_subscription(args.topic_name, args.subscription_name) + elif args.command == 'delete': + delete_subscription(args.topic_name, args.subscription_name) + elif args.command == 'receive': + receive_message(args.topic_name, args.subscription_name) diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py new file mode 100644 index 000000000..6335aa973 --- /dev/null +++ b/samples/snippets/subscriber_test.py @@ -0,0 +1,83 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from gcloud import pubsub +from gcp.testing import eventually_consistent +import pytest + +import subscriber + +TEST_TOPIC = 'subscription-test-topic' +TEST_SUBSCRIPTION = 'subscription-test-subscription' + + +@pytest.fixture +def test_topic(): + client = pubsub.Client() + topic = client.topic(TEST_TOPIC) + topic.create() + yield topic + if topic.exists(): + topic.delete() + + +@pytest.fixture +def test_subscription(test_topic): + subscription = test_topic.subscription(TEST_SUBSCRIPTION) + yield subscription + if subscription.exists(): + subscription.delete() + + +def test_list(test_subscription, capsys): + test_subscription.create() + + @eventually_consistent.call + def _(): + subscriber.list_subscriptions(test_subscription.topic.name) + out, _ = capsys.readouterr() + assert test_subscription.name in out + + +def test_create(test_subscription): + subscriber.create_subscription( + test_subscription.topic.name, test_subscription.name) + + @eventually_consistent.call + def _(): + assert test_subscription.exists() + + +def test_delete(test_subscription): + test_subscription.create() + + subscriber.delete_subscription( + test_subscription.topic.name, test_subscription.name) + + @eventually_consistent.call + def _(): + assert not test_subscription.exists() + + +def test_receive(test_subscription, capsys): + topic = test_subscription.topic + test_subscription.create() + + topic.publish('hello'.encode('utf-8')) + + @eventually_consistent.call + def _(): + subscriber.receive_message(topic.name, test_subscription.name) + out, _ = capsys.readouterr() + assert 'hello' in out From bcf2693e7a954085285d4d5e4d17e70aabbbae15 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Wed, 24 Aug 2016 14:56:47 -0700 Subject: [PATCH 003/101] Update readme Change-Id: Ie95e2e1556a8d97b5321dc86bf8de431aa36a2d5 --- samples/snippets/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/snippets/README.md b/samples/snippets/README.md index 8691c9017..00c7d2f7a 100644 --- a/samples/snippets/README.md +++ b/samples/snippets/README.md @@ -13,4 +13,5 @@ Use the [Cloud SDK](https://cloud.google.com/sdk) to provide authentication: Run the samples: - TODO + python publisher.py -h + python subscriber.py -h From b444e38a55c67d78f97ea2ba8167ba27937cbece Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 25 Aug 2016 11:26:06 -0700 Subject: [PATCH 004/101] Add pubsub iam samples Change-Id: I12c407d3cdf4a3f9736dfaeca6f20b31df6d310a --- samples/snippets/iam.py | 182 +++++++++++++++++++++++++++++++++++ samples/snippets/iam_test.py | 102 ++++++++++++++++++++ 2 files changed, 284 insertions(+) create mode 100644 samples/snippets/iam.py create mode 100644 samples/snippets/iam_test.py diff --git a/samples/snippets/iam.py b/samples/snippets/iam.py new file mode 100644 index 000000000..3828195bc --- /dev/null +++ b/samples/snippets/iam.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform basic operations on IAM +policies with the Cloud Pub/Sub API. + +For more information, see the README.md under /pubsub and the documentation +at https://cloud.google.com/pubsub/docs. +""" + +import argparse + +from gcloud import pubsub + + +def get_topic_policy(topic_name): + """Prints the IAM policy for the given topic.""" + pubsub_client = pubsub.Client() + topic = pubsub_client.topic(topic_name) + + policy = topic.get_iam_policy() + + print('Policy for topic {}:'.format(topic.name)) + print('Version: {}'.format(policy.version)) + print('Owners: {}'.format(policy.owners)) + print('Editors: {}'.format(policy.editors)) + print('Viewers: {}'.format(policy.viewers)) + print('Publishers: {}'.format(policy.publishers)) + print('Subscribers: {}'.format(policy.subscribers)) + + +def get_subscription_policy(topic_name, subscription_name): + """Prints the IAM policy for the given subscription.""" + pubsub_client = pubsub.Client() + topic = pubsub_client.topic(topic_name) + subscription = topic.subscription(subscription_name) + + policy = subscription.get_iam_policy() + + print('Policy for subscription {} on topic {}:'.format( + subscription.name, topic.name)) + print('Version: {}'.format(policy.version)) + print('Owners: {}'.format(policy.owners)) + print('Editors: {}'.format(policy.editors)) + print('Viewers: {}'.format(policy.viewers)) + print('Publishers: {}'.format(policy.publishers)) + print('Subscribers: {}'.format(policy.subscribers)) + + +def set_topic_policy(topic_name): + """Sets the IAM policy for a topic.""" + pubsub_client = pubsub.Client() + topic = pubsub_client.topic(topic_name) + policy = topic.get_iam_policy() + + # Add all users as viewers. + policy.viewers.add(policy.all_users()) + # Add a group as editors. + policy.editors.add(policy.group('cloud-logs@google.com')) + + # Set the policy + topic.set_iam_policy(policy) + + print('IAM policy for topic {} set.'.format(topic.name)) + + +def set_subscription_policy(topic_name, subscription_name): + """Sets the IAM policy for a topic.""" + pubsub_client = pubsub.Client() + topic = pubsub_client.topic(topic_name) + subscription = topic.subscription(subscription_name) + policy = subscription.get_iam_policy() + + # Add all users as viewers. + policy.viewers.add(policy.all_users()) + # Add a group as editors. + policy.editors.add(policy.group('cloud-logs@google.com')) + + # Set the policy + subscription.set_iam_policy(policy) + + print('IAM policy for subscription {} on topic {} set.'.format( + topic.name, subscription.name)) + + +def check_topic_permissions(topic_name): + """Checks to which permissions are available on the given topic.""" + pubsub_client = pubsub.Client() + topic = pubsub_client.topic(topic_name) + + permissions_to_check = [ + 'pubsub.topics.publish', + 'pubsub.topics.update' + ] + + allowed_permissions = topic.check_iam_permissions(permissions_to_check) + + print('Allowed permissions for topic {}: {}'.format( + topic.name, allowed_permissions)) + + +def check_subscription_permissions(topic_name, subscription_name): + """Checks to which permissions are available on the given subscription.""" + pubsub_client = pubsub.Client() + topic = pubsub_client.topic(topic_name) + subscription = topic.subscription(subscription_name) + + permissions_to_check = [ + 'pubsub.subscriptions.consume', + 'pubsub.subscriptions.update' + ] + + allowed_permissions = subscription.check_iam_permissions( + permissions_to_check) + + print('Allowed permissions for subscription {} on topic {}: {}'.format( + subscription.name, topic.name, allowed_permissions)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + + subparsers = parser.add_subparsers(dest='command') + + get_topic_policy_parser = subparsers.add_parser( + 'get-topic-policy', help=get_topic_policy.__doc__) + get_topic_policy_parser.add_argument('topic_name') + + get_subscription_policy_parser = subparsers.add_parser( + 'get-subscription-policy', help=get_subscription_policy.__doc__) + get_subscription_policy_parser.add_argument('topic_name') + get_subscription_policy_parser.add_argument('subscription_name') + + set_topic_policy_parser = subparsers.add_parser( + 'set-topic-policy', help=set_topic_policy.__doc__) + set_topic_policy_parser.add_argument('topic_name') + + set_subscription_policy_parser = subparsers.add_parser( + 'set-subscription-policy', help=set_subscription_policy.__doc__) + set_subscription_policy_parser.add_argument('topic_name') + set_subscription_policy_parser.add_argument('subscription_name') + + check_topic_permissions_parser = subparsers.add_parser( + 'check-topic-permissions', help=check_topic_permissions.__doc__) + check_topic_permissions_parser.add_argument('topic_name') + + check_subscription_permissions_parser = subparsers.add_parser( + 'check-subscription-permissions', + help=check_subscription_permissions.__doc__) + check_subscription_permissions_parser.add_argument('topic_name') + check_subscription_permissions_parser.add_argument('subscription_name') + + args = parser.parse_args() + + if args.command == 'get-topic-policy': + get_topic_policy(args.topic_name) + elif args.command == 'get-subscription-policy': + get_subscription_policy(args.topic_name, args.subscription_name) + elif args.command == 'set-topic-policy': + set_topic_policy(args.topic_name) + elif args.command == 'set-subscription-policy': + set_subscription_policy(args.topic_name, args.subscription_name) + elif args.command == 'check-topic-permissions': + check_topic_permissions(args.topic_name) + elif args.command == 'check-subscription-permissions': + check_subscription_permissions(args.topic_name, args.subscription_name) diff --git a/samples/snippets/iam_test.py b/samples/snippets/iam_test.py new file mode 100644 index 000000000..43f53cd3d --- /dev/null +++ b/samples/snippets/iam_test.py @@ -0,0 +1,102 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from gcloud import pubsub +from gcp.testing import eventually_consistent +import pytest + +import iam + +TEST_TOPIC = 'iam-test-topic' +TEST_SUBSCRIPTION = 'iam-test-subscription' + + +@pytest.fixture +def test_topic(): + client = pubsub.Client() + topic = client.topic(TEST_TOPIC) + topic.create() + yield topic + if topic.exists(): + topic.delete() + + +@pytest.fixture +def test_subscription(test_topic): + subscription = test_topic.subscription(TEST_SUBSCRIPTION) + yield subscription + if subscription.exists(): + subscription.delete() + + +def test_get_topic_policy(test_topic, capsys): + iam.get_topic_policy(test_topic.name) + + out, _ = capsys.readouterr() + assert test_topic.name in out + + +def test_get_subscription_policy(test_subscription, capsys): + test_subscription.create() + + iam.get_subscription_policy( + test_subscription.topic.name, + test_subscription.name) + + out, _ = capsys.readouterr() + assert test_subscription.topic.name in out + assert test_subscription.name in out + + +def test_set_topic_policy(test_topic): + iam.set_topic_policy(test_topic.name) + + policy = test_topic.get_iam_policy() + assert policy.viewers + assert policy.editors + + +def test_set_subscription_policy(test_subscription): + test_subscription.create() + + iam.set_subscription_policy( + test_subscription.topic.name, + test_subscription.name) + + policy = test_subscription.get_iam_policy() + assert policy.viewers + assert policy.editors + + +def test_check_topic_permissions(test_topic, capsys): + iam.check_topic_permissions(test_topic.name) + + out, _ = capsys.readouterr() + + assert test_topic.name in out + assert 'pubsub.topics.publish' in out + + +def test_check_subscription_permissions(test_subscription, capsys): + test_subscription.create() + + iam.check_subscription_permissions( + test_subscription.topic.name, + test_subscription.name) + + out, _ = capsys.readouterr() + + assert test_subscription.topic.name in out + assert test_subscription.name in out + assert 'pubsub.subscriptions.consume' in out From 709a9acaf45d5d79749825fa069a67099a9f5e81 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 1 Sep 2016 11:01:09 -0700 Subject: [PATCH 005/101] Fix lint issue Change-Id: Ifebdab0b974cc3d3fe8900a23ca7416fed9e026a --- samples/snippets/iam_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/samples/snippets/iam_test.py b/samples/snippets/iam_test.py index 43f53cd3d..0e662fcb5 100644 --- a/samples/snippets/iam_test.py +++ b/samples/snippets/iam_test.py @@ -13,7 +13,6 @@ # limitations under the License. from gcloud import pubsub -from gcp.testing import eventually_consistent import pytest import iam From bb28d992c120a70c4e870760672295669c6c2b73 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Fri, 23 Sep 2016 09:48:46 -0700 Subject: [PATCH 006/101] Auto-update dependencies. [(#540)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/540) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 2beeafe63..dfb42aaaa 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -gcloud==0.18.1 +gcloud==0.18.2 From 6f37fbad05da00ed2b278e38e59a99fe1009f0fb Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 26 Sep 2016 11:34:45 -0700 Subject: [PATCH 007/101] Auto-update dependencies. [(#542)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/542) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index dfb42aaaa..97a207d3a 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -gcloud==0.18.2 +gcloud==0.18.3 From 81f6ae9d15f32f7535d9ab7a9ee7d24a41b3ecb7 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 29 Sep 2016 20:51:47 -0700 Subject: [PATCH 008/101] Move to google-cloud [(#544)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/544) --- samples/snippets/iam.py | 2 +- samples/snippets/iam_test.py | 2 +- samples/snippets/publisher.py | 2 +- samples/snippets/publisher_test.py | 2 +- samples/snippets/requirements.txt | 2 +- samples/snippets/subscriber.py | 2 +- samples/snippets/subscriber_test.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/samples/snippets/iam.py b/samples/snippets/iam.py index 3828195bc..1f97e4570 100644 --- a/samples/snippets/iam.py +++ b/samples/snippets/iam.py @@ -23,7 +23,7 @@ import argparse -from gcloud import pubsub +from google.cloud import pubsub def get_topic_policy(topic_name): diff --git a/samples/snippets/iam_test.py b/samples/snippets/iam_test.py index 0e662fcb5..7adf87056 100644 --- a/samples/snippets/iam_test.py +++ b/samples/snippets/iam_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from gcloud import pubsub +from google.cloud import pubsub import pytest import iam diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index 61387d67e..d527b54cb 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -23,7 +23,7 @@ import argparse -from gcloud import pubsub +from google.cloud import pubsub def list_topics(): diff --git a/samples/snippets/publisher_test.py b/samples/snippets/publisher_test.py index 3cce3c962..6c0a9be00 100644 --- a/samples/snippets/publisher_test.py +++ b/samples/snippets/publisher_test.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from gcloud import pubsub from gcp.testing import eventually_consistent +from google.cloud import pubsub import pytest import publisher diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 97a207d3a..7aa8ce1fb 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -gcloud==0.18.3 +google-cloud-pubsub==0.20.0 diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 2b3371cd6..b4fdd9573 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -23,7 +23,7 @@ import argparse -from gcloud import pubsub +from google.cloud import pubsub def list_subscriptions(topic_name): diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index 6335aa973..3b51d48af 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from gcloud import pubsub from gcp.testing import eventually_consistent +from google.cloud import pubsub import pytest import subscriber From 448a0664256f40a8d6d465876c82fcf23e3fbfb6 Mon Sep 17 00:00:00 2001 From: Jason Dobry Date: Wed, 5 Oct 2016 09:56:04 -0700 Subject: [PATCH 009/101] Add new "quickstart" samples [(#547)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/547) --- samples/snippets/quickstart.py | 40 ++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 samples/snippets/quickstart.py diff --git a/samples/snippets/quickstart.py b/samples/snippets/quickstart.py new file mode 100644 index 000000000..fdcb45003 --- /dev/null +++ b/samples/snippets/quickstart.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def run_quickstart(): + # [START pubsub_quickstart] + # Imports the Google Cloud client library + from google.cloud import pubsub + + # Instantiates a client + pubsub_client = pubsub.Client() + + # The name for the new topic + topic_name = 'my-new-topic' + + # Prepares the new topic + topic = pubsub_client.topic(topic_name) + + # Creates the new topic + topic.create() + + print('Topic {} created.'.format(topic.name)) + # [END pubsub_quickstart] + + +if __name__ == '__main__': + run_quickstart() From 7041fbefc6176b0674cd541c9186625ab1598ba4 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Wed, 12 Oct 2016 10:48:57 -0700 Subject: [PATCH 010/101] Quickstart tests [(#569)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/569) * Add tests for quickstarts * Update secrets --- samples/snippets/quickstart_test.py | 45 +++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 samples/snippets/quickstart_test.py diff --git a/samples/snippets/quickstart_test.py b/samples/snippets/quickstart_test.py new file mode 100644 index 000000000..bbb3bd75f --- /dev/null +++ b/samples/snippets/quickstart_test.py @@ -0,0 +1,45 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import pubsub +import pytest + +import quickstart + + +# Must match the dataset listed in quickstart.py (there's no easy way to +# extract this). +TOPIC_NAME = 'my-new-topic' + + +@pytest.fixture +def temporary_topic(): + """Fixture that ensures the test dataset does not exist before or + after a test.""" + pubsub_client = pubsub.Client() + topic = pubsub_client.topic(TOPIC_NAME) + + if topic.exists(): + topic.delete() + + yield + + if topic.exists(): + topic.delete() + + +def test_quickstart(capsys, temporary_topic): + quickstart.run_quickstart() + out, _ = capsys.readouterr() + assert TOPIC_NAME in out From 955535603098e97ad95a53b508c91812f5d0cb66 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Mon, 24 Oct 2016 11:03:17 -0700 Subject: [PATCH 011/101] Generate readmes for most service samples [(#599)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/599) --- samples/snippets/README.md | 17 --- samples/snippets/README.rst | 203 +++++++++++++++++++++++++++++++++ samples/snippets/README.rst.in | 28 +++++ 3 files changed, 231 insertions(+), 17 deletions(-) delete mode 100644 samples/snippets/README.md create mode 100644 samples/snippets/README.rst create mode 100644 samples/snippets/README.rst.in diff --git a/samples/snippets/README.md b/samples/snippets/README.md deleted file mode 100644 index 00c7d2f7a..000000000 --- a/samples/snippets/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Google Cloud Pub/Sub Samples - - - - -## Prerequisites - -All samples require a [Google Cloud Project](https://console.cloud.google.com). - -Use the [Cloud SDK](https://cloud.google.com/sdk) to provide authentication: - - gcloud beta auth application-default login - -Run the samples: - - python publisher.py -h - python subscriber.py -h diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst new file mode 100644 index 000000000..8ea32bab0 --- /dev/null +++ b/samples/snippets/README.rst @@ -0,0 +1,203 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Pub/Sub Python Samples +=============================================================================== + +This directory contains samples for Google Cloud Pub/Sub. `Google Cloud Pub/Sub`_ is a fully-managed real-time messaging service that allows you to send and receive messages between independent applications. + + + + +.. _Google Cloud Pub/Sub: https://cloud.google.com/pubsub/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +Authentication is typically done through `Application Default Credentials`_, +which means you do not have to change the code to authenticate as long as +your environment has credentials. You have a few options for setting up +authentication: + +#. When running locally, use the `Google Cloud SDK`_ + + .. code-block:: bash + + gcloud beta auth application-default login + + +#. When running on App Engine or Compute Engine, credentials are already + set-up. However, you may need to configure your Compute Engine instance + with `additional scopes`_. + +#. You can create a `Service Account key file`_. This file can be used to + authenticate to Google Cloud Platform services from any environment. To use + the file, set the ``GOOGLE_APPLICATION_CREDENTIALS`` environment variable to + the path to the key file, for example: + + .. code-block:: bash + + export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service_account.json + +.. _Application Default Credentials: https://cloud.google.com/docs/authentication#getting_credentials_for_server-centric_flow +.. _additional scopes: https://cloud.google.com/compute/docs/authentication#using +.. _Service Account key file: https://developers.google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount + +Install Dependencies +++++++++++++++++++++ + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Quickstart ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python quickstart.py + + +Publisher ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python publisher.py + + usage: publisher.py [-h] {list,create,delete,publish} ... + + This application demonstrates how to perform basic operations on topics + with the Cloud Pub/Sub API. + + For more information, see the README.md under /pubsub and the documentation + at https://cloud.google.com/pubsub/docs. + + positional arguments: + {list,create,delete,publish} + list Lists all Pub/Sub topics in the current project. + create Create a new Pub/Sub topic. + delete Deletes an existing Pub/Sub topic. + publish Publishes a message to a Pub/Sub topic with the given + data. + + optional arguments: + -h, --help show this help message and exit + + +Subscribers ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python subscriber.py + + usage: subscriber.py [-h] {list,create,delete,receive} ... + + This application demonstrates how to perform basic operations on + subscriptions with the Cloud Pub/Sub API. + + For more information, see the README.md under /pubsub and the documentation + at https://cloud.google.com/pubsub/docs. + + positional arguments: + {list,create,delete,receive} + list Lists all subscriptions for a given topic. + create Create a new pull subscription on the given topic. + delete Deletes an existing Pub/Sub topic. + receive Receives a message from a pull subscription. + + optional arguments: + -h, --help show this help message and exit + + +Identity and Access Management ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + + +To run this sample: + +.. code-block:: bash + + $ python iam.py + + usage: iam.py [-h] + {get-topic-policy,get-subscription-policy,set-topic-policy,set-subscription-policy,check-topic-permissions,check-subscription-permissions} + ... + + This application demonstrates how to perform basic operations on IAM + policies with the Cloud Pub/Sub API. + + For more information, see the README.md under /pubsub and the documentation + at https://cloud.google.com/pubsub/docs. + + positional arguments: + {get-topic-policy,get-subscription-policy,set-topic-policy,set-subscription-policy,check-topic-permissions,check-subscription-permissions} + get-topic-policy Prints the IAM policy for the given topic. + get-subscription-policy + Prints the IAM policy for the given subscription. + set-topic-policy Sets the IAM policy for a topic. + set-subscription-policy + Sets the IAM policy for a topic. + check-topic-permissions + Checks to which permissions are available on the given + topic. + check-subscription-permissions + Checks to which permissions are available on the given + subscription. + + optional arguments: + -h, --help show this help message and exit + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/samples/snippets/README.rst.in b/samples/snippets/README.rst.in new file mode 100644 index 000000000..6a9fd00c7 --- /dev/null +++ b/samples/snippets/README.rst.in @@ -0,0 +1,28 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Pub/Sub + short_name: Cloud Pub/Sub + url: https://cloud.google.com/pubsub/docs + description: > + `Google Cloud Pub/Sub`_ is a fully-managed real-time messaging service that + allows you to send and receive messages between independent applications. + +setup: +- auth +- install_deps + +samples: +- name: Quickstart + file: quickstart.py +- name: Publisher + file: publisher.py + show_help: true +- name: Subscribers + file: subscriber.py + show_help: true +- name: Identity and Access Management + file: iam.py + show_help: true + +cloud_client_library: true From 2dd4785a3946ca8fbd4086cd01b7856e59ab761a Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 15 Nov 2016 14:58:27 -0800 Subject: [PATCH 012/101] Update samples to support latest Google Cloud Python [(#656)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/656) --- samples/snippets/publisher.py | 10 +--------- samples/snippets/requirements.txt | 2 +- samples/snippets/subscriber.py | 10 +--------- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index d527b54cb..465047560 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -30,15 +30,7 @@ def list_topics(): """Lists all Pub/Sub topics in the current project.""" pubsub_client = pubsub.Client() - topics = [] - next_page_token = None - while True: - page, next_page_token = pubsub_client.list_topics() - topics.extend(page) - if not next_page_token: - break - - for topic in topics: + for topic in pubsub_client.list_topics(): print(topic.name) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 7aa8ce1fb..f2fc2b4a2 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.20.0 +google-cloud-pubsub==0.21.0 diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index b4fdd9573..9a564496e 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -31,15 +31,7 @@ def list_subscriptions(topic_name): pubsub_client = pubsub.Client() topic = pubsub_client.topic(topic_name) - subscriptions = [] - next_page_token = None - while True: - page, next_page_token = topic.list_subscriptions() - subscriptions.extend(page) - if not next_page_token: - break - - for subscription in subscriptions: + for subscription in topic.list_subscriptions(): print(subscription.name) From fdee3c4da485cfe30617abcd1444b890df8e5b63 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 13 Dec 2016 09:54:02 -0800 Subject: [PATCH 013/101] Auto-update dependencies. [(#715)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/715) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index f2fc2b4a2..b53a39956 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.21.0 +google-cloud-pubsub==0.22.0 From d47fb0fd64ff2bd0ae778f3d486f98908d93e9af Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 16 Feb 2017 13:39:16 -0800 Subject: [PATCH 014/101] Fix pubusb tests Change-Id: I7dfe60b0f1240dc58a664968fd97ca5a8fa1109d --- samples/snippets/iam_test.py | 8 ++++++-- samples/snippets/subscriber_test.py | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/samples/snippets/iam_test.py b/samples/snippets/iam_test.py index 7adf87056..6d9f7d4d2 100644 --- a/samples/snippets/iam_test.py +++ b/samples/snippets/iam_test.py @@ -21,12 +21,16 @@ TEST_SUBSCRIPTION = 'iam-test-subscription' -@pytest.fixture +@pytest.fixture(scope='module') def test_topic(): client = pubsub.Client() topic = client.topic(TEST_TOPIC) - topic.create() + + if not topic.exists(): + topic.create() + yield topic + if topic.exists(): topic.delete() diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index 3b51d48af..e1fe2d482 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -22,12 +22,16 @@ TEST_SUBSCRIPTION = 'subscription-test-subscription' -@pytest.fixture +@pytest.fixture(scope='module') def test_topic(): client = pubsub.Client() topic = client.topic(TEST_TOPIC) - topic.create() + + if not topic.exists(): + topic.create() + yield topic + if topic.exists(): topic.delete() From f135a5b800432d741b2d1cc59ca58aaeeb3db034 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Fri, 10 Mar 2017 21:25:51 -0800 Subject: [PATCH 015/101] Auto-update dependencies. [(#825)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/825) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index b53a39956..2694ebdd9 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.22.0 +google-cloud-pubsub==0.23.0 From 101d32cc98f47408014d6623aa76e077cfd199b0 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 4 Apr 2017 09:39:33 -0700 Subject: [PATCH 016/101] Auto-update dependencies. [(#876)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/876) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 2694ebdd9..787d6c448 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.23.0 +google-cloud-pubsub==0.24.0 From 2f5d2a9c53b3c4bd029a24f0ec58cd151727dcd6 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Wed, 12 Apr 2017 15:14:35 -0700 Subject: [PATCH 017/101] Fix reference to our testing tools --- samples/snippets/publisher_test.py | 2 +- samples/snippets/subscriber_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/publisher_test.py b/samples/snippets/publisher_test.py index 6c0a9be00..d7ca67098 100644 --- a/samples/snippets/publisher_test.py +++ b/samples/snippets/publisher_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from gcp.testing import eventually_consistent +from gcp_devrel.testing import eventually_consistent from google.cloud import pubsub import pytest diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index e1fe2d482..9f7f5a1bf 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from gcp.testing import eventually_consistent +from gcp_devrel.testing import eventually_consistent from google.cloud import pubsub import pytest From 3ba6fa153951bd761121af5a335b1ab400a4ef8e Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 27 Apr 2017 09:54:41 -0700 Subject: [PATCH 018/101] Re-generate all readmes --- samples/snippets/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index 8ea32bab0..b507e8fc2 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -26,7 +26,7 @@ authentication: .. code-block:: bash - gcloud beta auth application-default login + gcloud auth application-default login #. When running on App Engine or Compute Engine, credentials are already From 9b113cf9be0acec4c92c5a592fc21ee72cdbc6eb Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 1 May 2017 10:49:29 -0700 Subject: [PATCH 019/101] Auto-update dependencies. [(#922)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/922) * Auto-update dependencies. * Fix pubsub iam samples --- samples/snippets/iam.py | 16 ++++++++++------ samples/snippets/iam_test.py | 2 +- samples/snippets/requirements.txt | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/samples/snippets/iam.py b/samples/snippets/iam.py index 1f97e4570..66fb6eb73 100644 --- a/samples/snippets/iam.py +++ b/samples/snippets/iam.py @@ -67,9 +67,11 @@ def set_topic_policy(topic_name): policy = topic.get_iam_policy() # Add all users as viewers. - policy.viewers.add(policy.all_users()) - # Add a group as editors. - policy.editors.add(policy.group('cloud-logs@google.com')) + policy['roles/pubsub.viewer'] = [policy.all_users()] + # Add a group as publisherss. + publishers = policy.get('roles/pubsub.publisher', []) + publishers.append(policy.group('cloud-logs@google.com')) + policy['roles/pubsub.publisher'] = publishers # Set the policy topic.set_iam_policy(policy) @@ -85,9 +87,11 @@ def set_subscription_policy(topic_name, subscription_name): policy = subscription.get_iam_policy() # Add all users as viewers. - policy.viewers.add(policy.all_users()) - # Add a group as editors. - policy.editors.add(policy.group('cloud-logs@google.com')) + policy['roles/viewer'] = [policy.all_users()] + # # Add a group as editors. + editors = policy.get('roles/editor', []) + editors.append(policy.group('cloud-logs@google.com')) + policy['roles/editor'] = editors # Set the policy subscription.set_iam_policy(policy) diff --git a/samples/snippets/iam_test.py b/samples/snippets/iam_test.py index 6d9f7d4d2..f36dbd214 100644 --- a/samples/snippets/iam_test.py +++ b/samples/snippets/iam_test.py @@ -67,7 +67,7 @@ def test_set_topic_policy(test_topic): policy = test_topic.get_iam_policy() assert policy.viewers - assert policy.editors + assert policy['roles/pubsub.publisher'] def test_set_subscription_policy(test_subscription): diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 787d6c448..1412eed3d 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.24.0 +google-cloud-pubsub==0.25.0 From 1ed3e69448f6a1adf0b2aa2ae15d21e60934514d Mon Sep 17 00:00:00 2001 From: Bill Prin Date: Tue, 23 May 2017 17:01:25 -0700 Subject: [PATCH 020/101] Fix README rst links [(#962)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/962) * Fix README rst links * Update all READMEs --- samples/snippets/README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index b507e8fc2..0c18e7ff6 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -192,11 +192,11 @@ This sample uses the `Google Cloud Client Library for Python`_. You can read the documentation for more details on API usage and use GitHub to `browse the source`_ and `report issues`_. -.. Google Cloud Client Library for Python: +.. _Google Cloud Client Library for Python: https://googlecloudplatform.github.io/google-cloud-python/ -.. browse the source: +.. _browse the source: https://github.com/GoogleCloudPlatform/google-cloud-python -.. report issues: +.. _report issues: https://github.com/GoogleCloudPlatform/google-cloud-python/issues From a0daf6d98946621742b98fad0df69add28316545 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 27 Jun 2017 12:41:15 -0700 Subject: [PATCH 021/101] Auto-update dependencies. [(#1004)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1004) * Auto-update dependencies. * Fix natural language samples * Fix pubsub iam samples * Fix language samples * Fix bigquery samples --- samples/snippets/iam.py | 4 ++-- samples/snippets/requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/snippets/iam.py b/samples/snippets/iam.py index 66fb6eb73..7f35f9d9f 100644 --- a/samples/snippets/iam.py +++ b/samples/snippets/iam.py @@ -70,7 +70,7 @@ def set_topic_policy(topic_name): policy['roles/pubsub.viewer'] = [policy.all_users()] # Add a group as publisherss. publishers = policy.get('roles/pubsub.publisher', []) - publishers.append(policy.group('cloud-logs@google.com')) + publishers.add(policy.group('cloud-logs@google.com')) policy['roles/pubsub.publisher'] = publishers # Set the policy @@ -90,7 +90,7 @@ def set_subscription_policy(topic_name, subscription_name): policy['roles/viewer'] = [policy.all_users()] # # Add a group as editors. editors = policy.get('roles/editor', []) - editors.append(policy.group('cloud-logs@google.com')) + editors.add(policy.group('cloud-logs@google.com')) policy['roles/editor'] = editors # Set the policy diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 1412eed3d..01606e11b 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.25.0 +google-cloud-pubsub==0.26.0 From 0f4a96bfbcf39af6535a195e059c2a2282378ebd Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 7 Aug 2017 10:04:55 -0700 Subject: [PATCH 022/101] Auto-update dependencies. [(#1055)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1055) * Auto-update dependencies. * Explicitly use latest bigtable client Change-Id: Id71e9e768f020730e4ca9514a0d7ebaa794e7d9e * Revert language update for now Change-Id: I8867f154e9a5aae00d0047c9caf880e5e8f50c53 * Remove pdb. smh Change-Id: I5ff905fadc026eebbcd45512d4e76e003e3b2b43 --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 01606e11b..65c3daf96 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.26.0 +google-cloud-pubsub==0.27.0 From 8c5312c6ff99a02a91039d0b37298121d7c7c985 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Mon, 28 Aug 2017 09:15:55 -0700 Subject: [PATCH 023/101] Update pubsub samples [(#1092)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1092) --- samples/snippets/iam.py | 138 ++++++++++++++-------------- samples/snippets/iam_test.py | 110 +++++++++++----------- samples/snippets/publisher.py | 118 ++++++++++++++++++------ samples/snippets/publisher_test.py | 76 ++++++++++----- samples/snippets/quickstart.py | 19 ++-- samples/snippets/quickstart_test.py | 30 +++--- samples/snippets/requirements.txt | 2 +- samples/snippets/subscriber.py | 113 ++++++++++++++--------- samples/snippets/subscriber_test.py | 137 +++++++++++++++++++-------- 9 files changed, 460 insertions(+), 283 deletions(-) diff --git a/samples/snippets/iam.py b/samples/snippets/iam.py index 7f35f9d9f..b46bc1147 100644 --- a/samples/snippets/iam.py +++ b/samples/snippets/iam.py @@ -23,115 +23,113 @@ import argparse -from google.cloud import pubsub +from google.cloud import pubsub_v1 -def get_topic_policy(topic_name): +def get_topic_policy(project, topic_name): """Prints the IAM policy for the given topic.""" - pubsub_client = pubsub.Client() - topic = pubsub_client.topic(topic_name) + client = pubsub_v1.PublisherClient() + topic_path = client.topic_path(project, topic_name) - policy = topic.get_iam_policy() + policy = client.get_iam_policy(topic_path) - print('Policy for topic {}:'.format(topic.name)) - print('Version: {}'.format(policy.version)) - print('Owners: {}'.format(policy.owners)) - print('Editors: {}'.format(policy.editors)) - print('Viewers: {}'.format(policy.viewers)) - print('Publishers: {}'.format(policy.publishers)) - print('Subscribers: {}'.format(policy.subscribers)) + print('Policy for topic {}:'.format(topic_path)) + for binding in policy.bindings: + print('Role: {}, Members: {}'.format(binding.role, binding.members)) -def get_subscription_policy(topic_name, subscription_name): +def get_subscription_policy(project, subscription_name): """Prints the IAM policy for the given subscription.""" - pubsub_client = pubsub.Client() - topic = pubsub_client.topic(topic_name) - subscription = topic.subscription(subscription_name) + client = pubsub_v1.SubscriberClient() + subscription_path = client.subscription_path(project, subscription_name) - policy = subscription.get_iam_policy() + policy = client.get_iam_policy(subscription_path) - print('Policy for subscription {} on topic {}:'.format( - subscription.name, topic.name)) - print('Version: {}'.format(policy.version)) - print('Owners: {}'.format(policy.owners)) - print('Editors: {}'.format(policy.editors)) - print('Viewers: {}'.format(policy.viewers)) - print('Publishers: {}'.format(policy.publishers)) - print('Subscribers: {}'.format(policy.subscribers)) + print('Policy for subscription {}:'.format(subscription_path)) + for binding in policy.bindings: + print('Role: {}, Members: {}'.format(binding.role, binding.members)) -def set_topic_policy(topic_name): +def set_topic_policy(project, topic_name): """Sets the IAM policy for a topic.""" - pubsub_client = pubsub.Client() - topic = pubsub_client.topic(topic_name) - policy = topic.get_iam_policy() + client = pubsub_v1.PublisherClient() + topic_path = client.topic_path(project, topic_name) + + policy = client.get_iam_policy(topic_path) # Add all users as viewers. - policy['roles/pubsub.viewer'] = [policy.all_users()] - # Add a group as publisherss. - publishers = policy.get('roles/pubsub.publisher', []) - publishers.add(policy.group('cloud-logs@google.com')) - policy['roles/pubsub.publisher'] = publishers + policy.bindings.add( + role='roles/pubsub.viewer', + members=['allUsers']) + + # Add a group as a publisher. + policy.bindings.add( + role='roles/pubsub.publisher', + members=['group:cloud-logs@google.com']) # Set the policy - topic.set_iam_policy(policy) + policy = client.set_iam_policy(topic_path, policy) - print('IAM policy for topic {} set.'.format(topic.name)) + print('IAM policy for topic {} set: {}'.format( + topic_name, policy)) -def set_subscription_policy(topic_name, subscription_name): +def set_subscription_policy(project, subscription_name): """Sets the IAM policy for a topic.""" - pubsub_client = pubsub.Client() - topic = pubsub_client.topic(topic_name) - subscription = topic.subscription(subscription_name) - policy = subscription.get_iam_policy() + client = pubsub_v1.SubscriberClient() + subscription_path = client.subscription_path(project, subscription_name) + + policy = client.get_iam_policy(subscription_path) # Add all users as viewers. - policy['roles/viewer'] = [policy.all_users()] - # # Add a group as editors. - editors = policy.get('roles/editor', []) - editors.add(policy.group('cloud-logs@google.com')) - policy['roles/editor'] = editors + policy.bindings.add( + role='roles/pubsub.viewer', + members=['allUsers']) + + # Add a group as an editor. + policy.bindings.add( + role='roles/editor', + members=['group:cloud-logs@google.com']) # Set the policy - subscription.set_iam_policy(policy) + policy = client.set_iam_policy(subscription_path, policy) - print('IAM policy for subscription {} on topic {} set.'.format( - topic.name, subscription.name)) + print('IAM policy for subscription {} set: {}'.format( + subscription_name, policy)) -def check_topic_permissions(topic_name): +def check_topic_permissions(project, topic_name): """Checks to which permissions are available on the given topic.""" - pubsub_client = pubsub.Client() - topic = pubsub_client.topic(topic_name) + client = pubsub_v1.PublisherClient() + topic_path = client.topic_path(project, topic_name) permissions_to_check = [ 'pubsub.topics.publish', 'pubsub.topics.update' ] - allowed_permissions = topic.check_iam_permissions(permissions_to_check) + allowed_permissions = client.test_iam_permissions( + topic_path, permissions_to_check) print('Allowed permissions for topic {}: {}'.format( - topic.name, allowed_permissions)) + topic_path, allowed_permissions)) -def check_subscription_permissions(topic_name, subscription_name): +def check_subscription_permissions(project, subscription_name): """Checks to which permissions are available on the given subscription.""" - pubsub_client = pubsub.Client() - topic = pubsub_client.topic(topic_name) - subscription = topic.subscription(subscription_name) + client = pubsub_v1.SubscriberClient() + subscription_path = client.subscription_path(project, subscription_name) permissions_to_check = [ 'pubsub.subscriptions.consume', 'pubsub.subscriptions.update' ] - allowed_permissions = subscription.check_iam_permissions( - permissions_to_check) + allowed_permissions = client.test_iam_permissions( + subscription_path, permissions_to_check) - print('Allowed permissions for subscription {} on topic {}: {}'.format( - subscription.name, topic.name, allowed_permissions)) + print('Allowed permissions for subscription {}: {}'.format( + subscription_path, allowed_permissions)) if __name__ == '__main__': @@ -139,6 +137,7 @@ def check_subscription_permissions(topic_name, subscription_name): description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) + parser.add_argument('project', help='Your Google Cloud project ID') subparsers = parser.add_subparsers(dest='command') @@ -148,7 +147,6 @@ def check_subscription_permissions(topic_name, subscription_name): get_subscription_policy_parser = subparsers.add_parser( 'get-subscription-policy', help=get_subscription_policy.__doc__) - get_subscription_policy_parser.add_argument('topic_name') get_subscription_policy_parser.add_argument('subscription_name') set_topic_policy_parser = subparsers.add_parser( @@ -157,7 +155,6 @@ def check_subscription_permissions(topic_name, subscription_name): set_subscription_policy_parser = subparsers.add_parser( 'set-subscription-policy', help=set_subscription_policy.__doc__) - set_subscription_policy_parser.add_argument('topic_name') set_subscription_policy_parser.add_argument('subscription_name') check_topic_permissions_parser = subparsers.add_parser( @@ -167,20 +164,19 @@ def check_subscription_permissions(topic_name, subscription_name): check_subscription_permissions_parser = subparsers.add_parser( 'check-subscription-permissions', help=check_subscription_permissions.__doc__) - check_subscription_permissions_parser.add_argument('topic_name') check_subscription_permissions_parser.add_argument('subscription_name') args = parser.parse_args() if args.command == 'get-topic-policy': - get_topic_policy(args.topic_name) + get_topic_policy(args.project, args.topic_name) elif args.command == 'get-subscription-policy': - get_subscription_policy(args.topic_name, args.subscription_name) + get_subscription_policy(args.project, args.subscription_name) elif args.command == 'set-topic-policy': - set_topic_policy(args.topic_name) + set_topic_policy(args.project, args.topic_name) elif args.command == 'set-subscription-policy': - set_subscription_policy(args.topic_name, args.subscription_name) + set_subscription_policy(args.project, args.subscription_name) elif args.command == 'check-topic-permissions': - check_topic_permissions(args.topic_name) + check_topic_permissions(args.project, args.topic_name) elif args.command == 'check-subscription-permissions': - check_subscription_permissions(args.topic_name, args.subscription_name) + check_subscription_permissions(args.project, args.subscription_name) diff --git a/samples/snippets/iam_test.py b/samples/snippets/iam_test.py index f36dbd214..3deaec746 100644 --- a/samples/snippets/iam_test.py +++ b/samples/snippets/iam_test.py @@ -12,94 +12,100 @@ # See the License for the specific language governing permissions and # limitations under the License. -from google.cloud import pubsub +import os + +from google.cloud import pubsub_v1 import pytest import iam -TEST_TOPIC = 'iam-test-topic' -TEST_SUBSCRIPTION = 'iam-test-subscription' +PROJECT = os.environ['GCLOUD_PROJECT'] +TOPIC = 'iam-test-topic' +SUBSCRIPTION = 'iam-test-subscription' @pytest.fixture(scope='module') -def test_topic(): - client = pubsub.Client() - topic = client.topic(TEST_TOPIC) +def publisher_client(): + yield pubsub_v1.PublisherClient() - if not topic.exists(): - topic.create() - yield topic +@pytest.fixture(scope='module') +def topic(publisher_client): + topic_path = publisher_client.topic_path(PROJECT, TOPIC) - if topic.exists(): - topic.delete() + try: + publisher_client.delete_topic(topic_path) + except: + pass + publisher_client.create_topic(topic_path) -@pytest.fixture -def test_subscription(test_topic): - subscription = test_topic.subscription(TEST_SUBSCRIPTION) - yield subscription - if subscription.exists(): - subscription.delete() + yield topic_path -def test_get_topic_policy(test_topic, capsys): - iam.get_topic_policy(test_topic.name) +@pytest.fixture(scope='module') +def subscriber_client(): + yield pubsub_v1.SubscriberClient() - out, _ = capsys.readouterr() - assert test_topic.name in out +@pytest.fixture +def subscription(subscriber_client, topic): + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION) + + try: + subscriber_client.delete_subscription(subscription_path) + except: + pass -def test_get_subscription_policy(test_subscription, capsys): - test_subscription.create() + subscriber_client.create_subscription(subscription_path, topic=topic) - iam.get_subscription_policy( - test_subscription.topic.name, - test_subscription.name) + yield subscription_path + + +def test_get_topic_policy(topic, capsys): + iam.get_topic_policy(PROJECT, TOPIC) out, _ = capsys.readouterr() - assert test_subscription.topic.name in out - assert test_subscription.name in out + assert topic in out + +def test_get_subscription_policy(subscription, capsys): + iam.get_subscription_policy(PROJECT, SUBSCRIPTION) + + out, _ = capsys.readouterr() + assert subscription in out -def test_set_topic_policy(test_topic): - iam.set_topic_policy(test_topic.name) - policy = test_topic.get_iam_policy() - assert policy.viewers - assert policy['roles/pubsub.publisher'] +def test_set_topic_policy(publisher_client, topic): + iam.set_topic_policy(PROJECT, TOPIC) + policy = publisher_client.get_iam_policy(topic) + assert 'roles/pubsub.publisher' in str(policy) + assert 'allUsers' in str(policy) -def test_set_subscription_policy(test_subscription): - test_subscription.create() - iam.set_subscription_policy( - test_subscription.topic.name, - test_subscription.name) +def test_set_subscription_policy(subscriber_client, subscription): + iam.set_subscription_policy(PROJECT, SUBSCRIPTION) - policy = test_subscription.get_iam_policy() - assert policy.viewers - assert policy.editors + policy = subscriber_client.get_iam_policy(subscription) + assert 'roles/pubsub.viewer' in str(policy) + assert 'allUsers' in str(policy) -def test_check_topic_permissions(test_topic, capsys): - iam.check_topic_permissions(test_topic.name) +def test_check_topic_permissions(topic, capsys): + iam.check_topic_permissions(PROJECT, TOPIC) out, _ = capsys.readouterr() - assert test_topic.name in out + assert topic in out assert 'pubsub.topics.publish' in out -def test_check_subscription_permissions(test_subscription, capsys): - test_subscription.create() - - iam.check_subscription_permissions( - test_subscription.topic.name, - test_subscription.name) +def test_check_subscription_permissions(subscription, capsys): + iam.check_subscription_permissions(PROJECT, SUBSCRIPTION) out, _ = capsys.readouterr() - assert test_subscription.topic.name in out - assert test_subscription.name in out + assert subscription in out assert 'pubsub.subscriptions.consume' in out diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index 465047560..4304ddf91 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -23,48 +23,92 @@ import argparse -from google.cloud import pubsub +from google.cloud import pubsub_v1 -def list_topics(): - """Lists all Pub/Sub topics in the current project.""" - pubsub_client = pubsub.Client() +def list_topics(project): + """Lists all Pub/Sub topics in the given project.""" + publisher = pubsub_v1.PublisherClient() + project_path = publisher.project_path(project) - for topic in pubsub_client.list_topics(): - print(topic.name) + for topic in publisher.list_topics(project_path): + print(topic) -def create_topic(topic_name): +def create_topic(project, topic_name): """Create a new Pub/Sub topic.""" - pubsub_client = pubsub.Client() - topic = pubsub_client.topic(topic_name) + publisher = pubsub_v1.PublisherClient() + topic_path = publisher.topic_path(project, topic_name) - topic.create() + topic = publisher.create_topic(topic_path) - print('Topic {} created.'.format(topic.name)) + print('Topic created: {}'.format(topic)) -def delete_topic(topic_name): +def delete_topic(project, topic_name): """Deletes an existing Pub/Sub topic.""" - pubsub_client = pubsub.Client() - topic = pubsub_client.topic(topic_name) + publisher = pubsub_v1.PublisherClient() + topic_path = publisher.topic_path(project, topic_name) - topic.delete() + publisher.delete_topic(topic_path) - print('Topic {} deleted.'.format(topic.name)) + print('Topic deleted: {}'.format(topic_path)) -def publish_message(topic_name, data): - """Publishes a message to a Pub/Sub topic with the given data.""" - pubsub_client = pubsub.Client() - topic = pubsub_client.topic(topic_name) +def publish_messages(project, topic_name): + """Publishes multiple messages to a Pub/Sub topic.""" + publisher = pubsub_v1.PublisherClient() + topic_path = publisher.topic_path(project, topic_name) - # Data must be a bytestring - data = data.encode('utf-8') + for n in range(1, 10): + data = u'Message number {}'.format(n) + # Data must be a bytestring + data = data.encode('utf-8') + publisher.publish(topic_path, data=data) - message_id = topic.publish(data) + print('Published messages.') - print('Message {} published.'.format(message_id)) + +def publish_messages_with_futures(project, topic_name): + """Publishes multiple messages to a Pub/Sub topic and prints their + message IDs.""" + publisher = pubsub_v1.PublisherClient() + topic_path = publisher.topic_path(project, topic_name) + + # When you publish a message, the client returns a Future. This Future + # can be used to track when the message is published. + futures = [] + + for n in range(1, 10): + data = u'Message number {}'.format(n) + # Data must be a bytestring + data = data.encode('utf-8') + message_future = publisher.publish(topic_path, data=data) + futures.append(message_future) + + print('Published message IDs:') + for future in futures: + print(future.result()) + + +def publish_messages_with_batch_settings(project, topic_name): + """Publishes multiple messages to a Pub/Sub topic with batch settings.""" + # Configure the batch to publish once there is one kilobyte of data or + # 1 second has passed. + batch_settings = pubsub_v1.types.BatchSettings( + max_bytes=1024, # One kilobyte + max_latency=1, # One second + ) + publisher = pubsub_v1.PublisherClient(batch_settings) + topic_path = publisher.topic_path(project, topic_name) + + for n in range(1, 10): + data = u'Message number {}'.format(n) + # Data must be a bytestring + data = data.encode('utf-8') + publisher.publish(topic_path, data=data) + + print('Published messages.') if __name__ == '__main__': @@ -72,6 +116,7 @@ def publish_message(topic_name, data): description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) + parser.add_argument('project', help='Your Google Cloud project ID') subparsers = parser.add_subparsers(dest='command') subparsers.add_parser('list', help=list_topics.__doc__) @@ -83,17 +128,30 @@ def publish_message(topic_name, data): delete_parser.add_argument('topic_name') publish_parser = subparsers.add_parser( - 'publish', help=publish_message.__doc__) + 'publish', help=publish_messages.__doc__) publish_parser.add_argument('topic_name') - publish_parser.add_argument('data') + + publish_with_futures_parser = subparsers.add_parser( + 'publish-with-futures', + help=publish_messages_with_futures.__doc__) + publish_with_futures_parser.add_argument('topic_name') + + publish_with_batch_settings_parser = subparsers.add_parser( + 'publish-with-batch-settings', + help=publish_messages_with_batch_settings.__doc__) + publish_with_batch_settings_parser.add_argument('topic_name') args = parser.parse_args() if args.command == 'list': - list_topics() + list_topics(args.project) elif args.command == 'create': - create_topic(args.topic_name) + create_topic(args.project, args.topic_name) elif args.command == 'delete': - delete_topic(args.topic_name) + delete_topic(args.project, args.topic_name) elif args.command == 'publish': - publish_message(args.topic_name, args.data) + publish_messages(args.project, args.topic_name) + elif args.command == 'publish-with-futures': + publish_messages_with_futures(args.project, args.topic_name) + elif args.command == 'publish-with-batch-settings': + publish_messages_with_batch_settings(args.project, args.topic_name) diff --git a/samples/snippets/publisher_test.py b/samples/snippets/publisher_test.py index d7ca67098..b400c9f24 100644 --- a/samples/snippets/publisher_test.py +++ b/samples/snippets/publisher_test.py @@ -12,56 +12,84 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + from gcp_devrel.testing import eventually_consistent -from google.cloud import pubsub +from google.cloud import pubsub_v1 import pytest import publisher -TEST_TOPIC = 'publisher-test-topic' +PROJECT = os.environ['GCLOUD_PROJECT'] +TOPIC = 'publisher-test-topic' + + +@pytest.fixture +def client(): + yield pubsub_v1.PublisherClient() @pytest.fixture -def test_topic(): - client = pubsub.Client() - topic = client.topic(TEST_TOPIC) - yield topic - if topic.exists(): - topic.delete() +def topic(client): + topic_path = client.topic_path(PROJECT, TOPIC) + try: + client.delete_topic(topic_path) + except: + pass -def test_list(test_topic, capsys): - test_topic.create() + client.create_topic(topic_path) + yield topic_path + + +def test_list(client, topic, capsys): @eventually_consistent.call def _(): - publisher.list_topics() + publisher.list_topics(PROJECT) out, _ = capsys.readouterr() - assert test_topic.name in out + assert topic in out -def test_create(test_topic): - publisher.create_topic(test_topic.name) +def test_create(client): + topic_path = client.topic_path(PROJECT, TOPIC) + try: + client.delete_topic(topic_path) + except: + pass + + publisher.create_topic(PROJECT, TOPIC) @eventually_consistent.call def _(): - assert test_topic.exists() - + assert client.get_topic(topic_path) -def test_delete(test_topic): - test_topic.create() - publisher.delete_topic(test_topic.name) +def test_delete(client, topic): + publisher.delete_topic(PROJECT, TOPIC) @eventually_consistent.call def _(): - assert not test_topic.exists() + with pytest.raises(Exception): + client.get_topic(client.topic_path(PROJECT, TOPIC)) -def test_publish(test_topic, capsys): - test_topic.create() +def test_publish(topic, capsys): + publisher.publish_messages(PROJECT, TOPIC) + + out, _ = capsys.readouterr() + assert 'Published' in out + + +def test_publish_with_batch_settings(topic, capsys): + publisher.publish_messages_with_batch_settings(PROJECT, TOPIC) + + out, _ = capsys.readouterr() + assert 'Published' in out + - publisher.publish_message(test_topic.name, 'hello') +def test_publish_with_futures(topic, capsys): + publisher.publish_messages_with_futures(PROJECT, TOPIC) out, _ = capsys.readouterr() - assert 'published' in out + assert 'Published' in out diff --git a/samples/snippets/quickstart.py b/samples/snippets/quickstart.py index fdcb45003..c9823d789 100644 --- a/samples/snippets/quickstart.py +++ b/samples/snippets/quickstart.py @@ -18,21 +18,20 @@ def run_quickstart(): # [START pubsub_quickstart] # Imports the Google Cloud client library - from google.cloud import pubsub + from google.cloud import pubsub_v1 # Instantiates a client - pubsub_client = pubsub.Client() + publisher = pubsub_v1.PublisherClient() - # The name for the new topic - topic_name = 'my-new-topic' + # The resource path for the new topic contains the project ID + # and the topic name. + topic_path = publisher.topic_path( + 'my-project', 'my-new-topic') - # Prepares the new topic - topic = pubsub_client.topic(topic_name) + # Create the topic. + topic = publisher.create_topic(topic_path) - # Creates the new topic - topic.create() - - print('Topic {} created.'.format(topic.name)) + print('Topic created: {}'.format(topic)) # [END pubsub_quickstart] diff --git a/samples/snippets/quickstart_test.py b/samples/snippets/quickstart_test.py index bbb3bd75f..71e157d48 100644 --- a/samples/snippets/quickstart_test.py +++ b/samples/snippets/quickstart_test.py @@ -12,34 +12,36 @@ # See the License for the specific language governing permissions and # limitations under the License. -from google.cloud import pubsub +import os + +from google.cloud import pubsub_v1 +import mock import pytest import quickstart - -# Must match the dataset listed in quickstart.py (there's no easy way to -# extract this). +PROJECT = os.environ['GCLOUD_PROJECT'] +# Must match the dataset listed in quickstart.py TOPIC_NAME = 'my-new-topic' +TOPIC_PATH = 'projects/{}/topics/{}'.format(PROJECT, TOPIC_NAME) @pytest.fixture def temporary_topic(): - """Fixture that ensures the test dataset does not exist before or - after a test.""" - pubsub_client = pubsub.Client() - topic = pubsub_client.topic(TOPIC_NAME) + """Fixture that ensures the test topic does not exist before the test.""" + publisher = pubsub_v1.PublisherClient() - if topic.exists(): - topic.delete() + try: + publisher.delete_topic(TOPIC_PATH) + except: + pass yield - if topic.exists(): - topic.delete() - -def test_quickstart(capsys, temporary_topic): +@mock.patch.object( + pubsub_v1.PublisherClient, 'topic_path', return_value=TOPIC_PATH) +def test_quickstart(unused_topic_path, temporary_topic, capsys): quickstart.run_quickstart() out, _ = capsys.readouterr() assert TOPIC_NAME in out diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 65c3daf96..6b4c47db9 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.27.0 +google-cloud-pubsub==0.28.2 diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 9a564496e..aef2ab679 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -22,63 +22,83 @@ """ import argparse +import time -from google.cloud import pubsub +from google.cloud import pubsub_v1 -def list_subscriptions(topic_name): +def list_subscriptions(project, topic_name): """Lists all subscriptions for a given topic.""" - pubsub_client = pubsub.Client() - topic = pubsub_client.topic(topic_name) + subscriber = pubsub_v1.SubscriberClient() + topic_path = subscriber.topic_path(project, topic_name) - for subscription in topic.list_subscriptions(): + for subscription in subscriber.list_subscriptions(topic_path): print(subscription.name) -def create_subscription(topic_name, subscription_name): +def create_subscription(project, topic_name, subscription_name): """Create a new pull subscription on the given topic.""" - pubsub_client = pubsub.Client() - topic = pubsub_client.topic(topic_name) + subscriber = pubsub_v1.SubscriberClient() + topic_path = subscriber.topic_path(project, topic_name) + subscription_path = subscriber.subscription_path( + project, subscription_name) - subscription = topic.subscription(subscription_name) - subscription.create() + subscription = subscriber.create_subscription( + subscription_path, topic_path) - print('Subscription {} created on topic {}.'.format( - subscription.name, topic.name)) + print('Subscription created: {}'.format(subscription)) -def delete_subscription(topic_name, subscription_name): +def delete_subscription(project, subscription_name): """Deletes an existing Pub/Sub topic.""" - pubsub_client = pubsub.Client() - topic = pubsub_client.topic(topic_name) - subscription = topic.subscription(subscription_name) + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path( + project, subscription_name) - subscription.delete() + subscriber.delete_subscription(subscription_path) - print('Subscription {} deleted on topic {}.'.format( - subscription.name, topic.name)) + print('Subscription deleted: {}'.format(subscription_path)) -def receive_message(topic_name, subscription_name): - """Receives a message from a pull subscription.""" - pubsub_client = pubsub.Client() - topic = pubsub_client.topic(topic_name) - subscription = topic.subscription(subscription_name) +def receive_messages(project, subscription_name): + """Receives messages from a pull subscription.""" + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path( + project, subscription_name) - # Change return_immediately=False to block until messages are - # received. - results = subscription.pull(return_immediately=True) + def callback(message): + print('Received message: {}'.format(message)) + message.ack() - print('Received {} messages.'.format(len(results))) + subscriber.subscribe(subscription_path, callback=callback) - for ack_id, message in results: - print('* {}: {}, {}'.format( - message.message_id, message.data, message.attributes)) + # The subscriber is non-blocking, so we must keep the main thread from + # exiting to allow it to process messages in the background. + print('Listening for messages on {}'.format(subscription_path)) + while True: + time.sleep(60) - # Acknowledge received messages. If you do not acknowledge, Pub/Sub will - # redeliver the message. - if results: - subscription.acknowledge([ack_id for ack_id, message in results]) + +def receive_messages_with_flow_control(project, subscription_name): + """Receives messages from a pull subscription with flow control.""" + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path( + project, subscription_name) + + def callback(message): + print('Received message: {}'.format(message)) + message.ack() + + # Limit the subscriber to only have ten outstanding messages at a time. + flow_control = pubsub_v1.types.FlowControl(max_messages=10) + subscriber.subscribe( + subscription_path, callback=callback, flow_control=flow_control) + + # The subscriber is non-blocking, so we must keep the main thread from + # exiting to allow it to process messages in the background. + print('Listening for messages on {}'.format(subscription_path)) + while True: + time.sleep(60) if __name__ == '__main__': @@ -86,6 +106,7 @@ def receive_message(topic_name, subscription_name): description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) + parser.add_argument('project', help='Your Google Cloud project ID') subparsers = parser.add_subparsers(dest='command') list_parser = subparsers.add_parser( @@ -99,21 +120,29 @@ def receive_message(topic_name, subscription_name): delete_parser = subparsers.add_parser( 'delete', help=delete_subscription.__doc__) - delete_parser.add_argument('topic_name') delete_parser.add_argument('subscription_name') receive_parser = subparsers.add_parser( - 'receive', help=receive_message.__doc__) - receive_parser.add_argument('topic_name') + 'receive', help=receive_messages.__doc__) receive_parser.add_argument('subscription_name') + receive_with_flow_control_parser = subparsers.add_parser( + 'receive-flow-control', + help=receive_messages_with_flow_control.__doc__) + receive_with_flow_control_parser.add_argument('subscription_name') + args = parser.parse_args() if args.command == 'list': - list_subscriptions(args.topic_name) + list_subscriptions(args.project, args.topic_name) elif args.command == 'create': - create_subscription(args.topic_name, args.subscription_name) + create_subscription( + args.project, args.topic_name, args.subscription_name) elif args.command == 'delete': - delete_subscription(args.topic_name, args.subscription_name) + delete_subscription( + args.project, args.subscription_name) elif args.command == 'receive': - receive_message(args.topic_name, args.subscription_name) + receive_messages(args.project, args.topic_name, args.subscription_name) + elif args.command == 'receive-flow-control': + receive_messages_with_flow_control( + args.project, args.topic_name, args.subscription_name) diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index 9f7f5a1bf..0acadf437 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -12,76 +12,135 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +import time + from gcp_devrel.testing import eventually_consistent -from google.cloud import pubsub +from google.cloud import pubsub_v1 +import mock import pytest import subscriber -TEST_TOPIC = 'subscription-test-topic' -TEST_SUBSCRIPTION = 'subscription-test-subscription' +PROJECT = os.environ['GCLOUD_PROJECT'] +TOPIC = 'subscription-test-topic' +SUBSCRIPTION = 'subscription-test-subscription' + + +@pytest.fixture(scope='module') +def publisher_client(): + yield pubsub_v1.PublisherClient() @pytest.fixture(scope='module') -def test_topic(): - client = pubsub.Client() - topic = client.topic(TEST_TOPIC) +def topic(publisher_client): + topic_path = publisher_client.topic_path(PROJECT, TOPIC) + + try: + publisher_client.delete_topic(topic_path) + except: + pass + + publisher_client.create_topic(topic_path) - if not topic.exists(): - topic.create() + yield topic_path - yield topic - if topic.exists(): - topic.delete() +@pytest.fixture(scope='module') +def subscriber_client(): + yield pubsub_v1.SubscriberClient() @pytest.fixture -def test_subscription(test_topic): - subscription = test_topic.subscription(TEST_SUBSCRIPTION) - yield subscription - if subscription.exists(): - subscription.delete() +def subscription(subscriber_client, topic): + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION) + + try: + subscriber_client.delete_subscription(subscription_path) + except: + pass + + subscriber_client.create_subscription(subscription_path, topic=topic) + yield subscription_path -def test_list(test_subscription, capsys): - test_subscription.create() +def test_list(subscription, capsys): @eventually_consistent.call def _(): - subscriber.list_subscriptions(test_subscription.topic.name) + subscriber.list_subscriptions(PROJECT, TOPIC) out, _ = capsys.readouterr() - assert test_subscription.name in out + assert subscription in out -def test_create(test_subscription): - subscriber.create_subscription( - test_subscription.topic.name, test_subscription.name) +def test_create(subscriber_client): + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION) + try: + subscriber_client.delete_subscription(subscription_path) + except: + pass + + subscriber.create_subscription(PROJECT, TOPIC, SUBSCRIPTION) @eventually_consistent.call def _(): - assert test_subscription.exists() - + assert subscriber_client.get_subscription(subscription_path) -def test_delete(test_subscription): - test_subscription.create() - subscriber.delete_subscription( - test_subscription.topic.name, test_subscription.name) +def test_delete(subscriber_client, subscription): + subscriber.delete_subscription(PROJECT, SUBSCRIPTION) @eventually_consistent.call def _(): - assert not test_subscription.exists() + with pytest.raises(Exception): + subscriber_client.get_subscription(subscription) -def test_receive(test_subscription, capsys): - topic = test_subscription.topic - test_subscription.create() +def _publish_messages(publisher_client, topic): + for n in range(5): + data = u'Message {}'.format(n).encode('utf-8') + publisher_client.publish( + topic, data=data) - topic.publish('hello'.encode('utf-8')) - @eventually_consistent.call - def _(): - subscriber.receive_message(topic.name, test_subscription.name) - out, _ = capsys.readouterr() - assert 'hello' in out +def _make_sleep_patch(): + real_sleep = time.sleep + + def new_sleep(period): + if period == 60: + real_sleep(5) + raise RuntimeError('sigil') + else: + real_sleep(period) + + return mock.patch('time.sleep', new=new_sleep) + + +def test_receive(publisher_client, topic, subscription, capsys): + _publish_messages(publisher_client, topic) + + with _make_sleep_patch(): + with pytest.raises(RuntimeError, match='sigil'): + subscriber.receive_messages(PROJECT, SUBSCRIPTION) + + out, _ = capsys.readouterr() + assert 'Listening' in out + assert subscription in out + assert 'Message 1' in out + + +def test_receive_with_flow_control( + publisher_client, topic, subscription, capsys): + _publish_messages(publisher_client, topic) + + with _make_sleep_patch(): + with pytest.raises(RuntimeError, match='sigil'): + subscriber.receive_messages_with_flow_control( + PROJECT, SUBSCRIPTION) + + out, _ = capsys.readouterr() + assert 'Listening' in out + assert subscription in out + assert 'Message 1' in out From 1343f98e7343914108e256d9d15436041a769e26 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Mon, 28 Aug 2017 11:23:40 -0700 Subject: [PATCH 024/101] Fix argpraser for pubsub subscriber Change-Id: I776863091846ee8ff8a70078c8b8d5498cf81ed6 --- samples/snippets/subscriber.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index aef2ab679..401faaab3 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -142,7 +142,7 @@ def callback(message): delete_subscription( args.project, args.subscription_name) elif args.command == 'receive': - receive_messages(args.project, args.topic_name, args.subscription_name) + receive_messages(args.project, args.subscription_name) elif args.command == 'receive-flow-control': receive_messages_with_flow_control( - args.project, args.topic_name, args.subscription_name) + args.project, args.subscription_name) From bd92f8b1be56c98f9b5e828f4d4fc8017de57845 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Mon, 28 Aug 2017 15:28:13 -0700 Subject: [PATCH 025/101] Add comment about result blocking in pubsub samples Change-Id: I149fc1242ceb6b2cff8eae7ef18b364dd5c26566 --- samples/snippets/publisher.py | 1 + 1 file changed, 1 insertion(+) diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index 4304ddf91..cf12b2626 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -88,6 +88,7 @@ def publish_messages_with_futures(project, topic_name): print('Published message IDs:') for future in futures: + # result() blocks until the message is published. print(future.result()) From 4e3eb071ff9fc5e928a199ad144834dcde02377b Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 12 Sep 2017 12:26:44 -0700 Subject: [PATCH 026/101] Auto-update dependencies. [(#1097)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1097) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 6b4c47db9..17edefddd 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.28.2 +google-cloud-pubsub==0.28.3 From 4485e7c0ae134a851880eb6a72d3f1bbc9e63ff2 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Mon, 18 Sep 2017 11:04:05 -0700 Subject: [PATCH 027/101] Update all generated readme auth instructions [(#1121)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1121) Change-Id: I03b5eaef8b17ac3dc3c0339fd2c7447bd3e11bd2 --- samples/snippets/README.rst | 65 +++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 35 deletions(-) diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index 0c18e7ff6..a0d39bd61 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -17,34 +17,12 @@ Setup Authentication ++++++++++++++ -Authentication is typically done through `Application Default Credentials`_, -which means you do not have to change the code to authenticate as long as -your environment has credentials. You have a few options for setting up -authentication: +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. -#. When running locally, use the `Google Cloud SDK`_ - - .. code-block:: bash - - gcloud auth application-default login - - -#. When running on App Engine or Compute Engine, credentials are already - set-up. However, you may need to configure your Compute Engine instance - with `additional scopes`_. - -#. You can create a `Service Account key file`_. This file can be used to - authenticate to Google Cloud Platform services from any environment. To use - the file, set the ``GOOGLE_APPLICATION_CREDENTIALS`` environment variable to - the path to the key file, for example: - - .. code-block:: bash - - export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service_account.json - -.. _Application Default Credentials: https://cloud.google.com/docs/authentication#getting_credentials_for_server-centric_flow -.. _additional scopes: https://cloud.google.com/compute/docs/authentication#using -.. _Service Account key file: https://developers.google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started Install Dependencies ++++++++++++++++++++ @@ -93,7 +71,10 @@ To run this sample: $ python publisher.py - usage: publisher.py [-h] {list,create,delete,publish} ... + usage: publisher.py [-h] + project + {list,create,delete,publish,publish-with-futures,publish-with-batch-settings} + ... This application demonstrates how to perform basic operations on topics with the Cloud Pub/Sub API. @@ -102,12 +83,18 @@ To run this sample: at https://cloud.google.com/pubsub/docs. positional arguments: - {list,create,delete,publish} - list Lists all Pub/Sub topics in the current project. + project Your Google Cloud project ID + {list,create,delete,publish,publish-with-futures,publish-with-batch-settings} + list Lists all Pub/Sub topics in the given project. create Create a new Pub/Sub topic. delete Deletes an existing Pub/Sub topic. - publish Publishes a message to a Pub/Sub topic with the given - data. + publish Publishes multiple messages to a Pub/Sub topic. + publish-with-futures + Publishes multiple messages to a Pub/Sub topic and + prints their message IDs. + publish-with-batch-settings + Publishes multiple messages to a Pub/Sub topic with + batch settings. optional arguments: -h, --help show this help message and exit @@ -124,7 +111,9 @@ To run this sample: $ python subscriber.py - usage: subscriber.py [-h] {list,create,delete,receive} ... + usage: subscriber.py [-h] + project {list,create,delete,receive,receive-flow-control} + ... This application demonstrates how to perform basic operations on subscriptions with the Cloud Pub/Sub API. @@ -133,11 +122,15 @@ To run this sample: at https://cloud.google.com/pubsub/docs. positional arguments: - {list,create,delete,receive} + project Your Google Cloud project ID + {list,create,delete,receive,receive-flow-control} list Lists all subscriptions for a given topic. create Create a new pull subscription on the given topic. delete Deletes an existing Pub/Sub topic. - receive Receives a message from a pull subscription. + receive Receives messages from a pull subscription. + receive-flow-control + Receives messages from a pull subscription with flow + control. optional arguments: -h, --help show this help message and exit @@ -155,6 +148,7 @@ To run this sample: $ python iam.py usage: iam.py [-h] + project {get-topic-policy,get-subscription-policy,set-topic-policy,set-subscription-policy,check-topic-permissions,check-subscription-permissions} ... @@ -165,6 +159,7 @@ To run this sample: at https://cloud.google.com/pubsub/docs. positional arguments: + project Your Google Cloud project ID {get-topic-policy,get-subscription-policy,set-topic-policy,set-subscription-policy,check-topic-permissions,check-subscription-permissions} get-topic-policy Prints the IAM policy for the given topic. get-subscription-policy From e138d7d885a97d04265ec3850b25078e546e6f27 Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Thu, 12 Oct 2017 10:16:11 -0700 Subject: [PATCH 028/101] Added Link to Python Setup Guide [(#1158)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1158) * Update Readme.rst to add Python setup guide As requested in b/64770713. This sample is linked in documentation https://cloud.google.com/bigtable/docs/scaling, and it would make more sense to update the guide here than in the documentation. * Update README.rst * Update README.rst * Update README.rst * Update README.rst * Update README.rst * Update install_deps.tmpl.rst * Updated readmegen scripts and re-generated related README files * Fixed the lint error --- samples/snippets/README.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index a0d39bd61..df1dbf7c6 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -27,7 +27,10 @@ credentials for applications. Install Dependencies ++++++++++++++++++++ -#. Install `pip`_ and `virtualenv`_ if you do not already have them. +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup #. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. From 9adce6818e5d54225b189018799fe7185e3c03d9 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 23 Oct 2017 14:23:30 -0700 Subject: [PATCH 029/101] Auto-update dependencies. [(#1138)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1138) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 17edefddd..8738f6887 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.28.3 +google-cloud-pubsub==0.28.4 From 671d7873d8fc92b7a3bcb050357e12f27a306e12 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 24 Oct 2017 12:14:35 -0700 Subject: [PATCH 030/101] Fix a few more lint issues Change-Id: I0d420f3053f391fa225e4b8179e45fd1138f5c65 --- samples/snippets/iam_test.py | 4 ++-- samples/snippets/publisher_test.py | 4 ++-- samples/snippets/quickstart_test.py | 2 +- samples/snippets/subscriber_test.py | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/samples/snippets/iam_test.py b/samples/snippets/iam_test.py index 3deaec746..8a524c35a 100644 --- a/samples/snippets/iam_test.py +++ b/samples/snippets/iam_test.py @@ -35,7 +35,7 @@ def topic(publisher_client): try: publisher_client.delete_topic(topic_path) - except: + except Exception: pass publisher_client.create_topic(topic_path) @@ -55,7 +55,7 @@ def subscription(subscriber_client, topic): try: subscriber_client.delete_subscription(subscription_path) - except: + except Exception: pass subscriber_client.create_subscription(subscription_path, topic=topic) diff --git a/samples/snippets/publisher_test.py b/samples/snippets/publisher_test.py index b400c9f24..120148c0a 100644 --- a/samples/snippets/publisher_test.py +++ b/samples/snippets/publisher_test.py @@ -35,7 +35,7 @@ def topic(client): try: client.delete_topic(topic_path) - except: + except Exception: pass client.create_topic(topic_path) @@ -55,7 +55,7 @@ def test_create(client): topic_path = client.topic_path(PROJECT, TOPIC) try: client.delete_topic(topic_path) - except: + except Exception: pass publisher.create_topic(PROJECT, TOPIC) diff --git a/samples/snippets/quickstart_test.py b/samples/snippets/quickstart_test.py index 71e157d48..520213bcf 100644 --- a/samples/snippets/quickstart_test.py +++ b/samples/snippets/quickstart_test.py @@ -33,7 +33,7 @@ def temporary_topic(): try: publisher.delete_topic(TOPIC_PATH) - except: + except Exception: pass yield diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index 0acadf437..2cc955d9d 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -38,7 +38,7 @@ def topic(publisher_client): try: publisher_client.delete_topic(topic_path) - except: + except Exception: pass publisher_client.create_topic(topic_path) @@ -58,7 +58,7 @@ def subscription(subscriber_client, topic): try: subscriber_client.delete_subscription(subscription_path) - except: + except Exception: pass subscriber_client.create_subscription(subscription_path, topic=topic) @@ -79,7 +79,7 @@ def test_create(subscriber_client): PROJECT, SUBSCRIPTION) try: subscriber_client.delete_subscription(subscription_path) - except: + except Exception: pass subscriber.create_subscription(PROJECT, TOPIC, SUBSCRIPTION) From 76b7463f23170529d173aa3cc4bb6ccc83163711 Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Tue, 31 Oct 2017 10:44:51 -0700 Subject: [PATCH 031/101] Add Snippet for Listing All Subscriptions in a Project [(#1169)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1169) --- samples/snippets/subscriber.py | 26 ++++++++++++++++++++------ samples/snippets/subscriber_test.py | 12 ++++++++++-- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 401faaab3..9dded25b1 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -27,7 +27,7 @@ from google.cloud import pubsub_v1 -def list_subscriptions(project, topic_name): +def list_subscriptions_in_topic(project, topic_name): """Lists all subscriptions for a given topic.""" subscriber = pubsub_v1.SubscriberClient() topic_path = subscriber.topic_path(project, topic_name) @@ -36,6 +36,15 @@ def list_subscriptions(project, topic_name): print(subscription.name) +def list_subscriptions_in_project(project): + """Lists all subscriptions in the current project.""" + subscriber = pubsub_v1.SubscriberClient() + project_path = subscriber.project_path(project) + + for subscription in subscriber.list_subscriptions(project_path): + print(subscription.name) + + def create_subscription(project, topic_name, subscription_name): """Create a new pull subscription on the given topic.""" subscriber = pubsub_v1.SubscriberClient() @@ -109,9 +118,12 @@ def callback(message): parser.add_argument('project', help='Your Google Cloud project ID') subparsers = parser.add_subparsers(dest='command') - list_parser = subparsers.add_parser( - 'list', help=list_subscriptions.__doc__) - list_parser.add_argument('topic_name') + list_in_topic_parser = subparsers.add_parser( + 'list_in_topic', help=list_subscriptions_in_topic.__doc__) + list_in_topic_parser.add_argument('topic_name') + + list_in_project_parser = subparsers.add_parser( + 'list_in_project', help=list_subscriptions_in_project.__doc__) create_parser = subparsers.add_parser( 'create', help=create_subscription.__doc__) @@ -133,8 +145,10 @@ def callback(message): args = parser.parse_args() - if args.command == 'list': - list_subscriptions(args.project, args.topic_name) + if args.command == 'list_in_topic': + list_subscriptions_in_topic(args.project, args.topic_name) + elif args.command == 'list_in_project': + list_subscriptions_in_project(args.project) elif args.command == 'create': create_subscription( args.project, args.topic_name, args.subscription_name) diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index 2cc955d9d..8b5e97ac4 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -66,10 +66,18 @@ def subscription(subscriber_client, topic): yield subscription_path -def test_list(subscription, capsys): +def test_list_in_topic(subscription, capsys): @eventually_consistent.call def _(): - subscriber.list_subscriptions(PROJECT, TOPIC) + subscriber.list_subscriptions_in_topic(PROJECT, TOPIC) + out, _ = capsys.readouterr() + assert subscription in out + + +def test_list_in_project(subscription, capsys): + @eventually_consistent.call + def _(): + subscriber.list_subscriptions_in_project(PROJECT) out, _ = capsys.readouterr() assert subscription in out From 229569ab816617bef5407cb7788bc529a0dc46e4 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Wed, 1 Nov 2017 12:30:10 -0700 Subject: [PATCH 032/101] Auto-update dependencies. [(#1186)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1186) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 8738f6887..fd3aa4fe8 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.28.4 +google-cloud-pubsub==0.29.0 From 7e31f49c5763d78ea3a4f906bf265644058d9d91 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 28 Nov 2017 09:52:33 -0800 Subject: [PATCH 033/101] Auto-update dependencies. [(#1234)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1234) * Auto-update dependencies. * Drop pytest-logcapture as it's no longer needed Change-Id: Ia8b9e8aaf248e9770db6bc4842a4532df8383893 --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index fd3aa4fe8..815e7d0e4 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.29.0 +google-cloud-pubsub==0.29.1 From e0e31c80ea4cabbd8bfa3a512e6a275b422ee073 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Thu, 30 Nov 2017 10:25:03 -0800 Subject: [PATCH 034/101] Auto-update dependencies. [(#1239)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1239) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 815e7d0e4..b6977877d 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.29.1 +google-cloud-pubsub==0.29.2 From 31ffa0e72423d579095495b153381cd160e37a0d Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Thu, 7 Dec 2017 10:34:29 -0800 Subject: [PATCH 035/101] Added "Open in Cloud Shell" buttons to README files [(#1254)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1254) --- samples/snippets/README.rst | 55 ++++++++++++++++++++++++---------- samples/snippets/README.rst.in | 2 ++ 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index df1dbf7c6..0ecf4b193 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -3,6 +3,10 @@ Google Cloud Pub/Sub Python Samples =============================================================================== +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/README.rst + + This directory contains samples for Google Cloud Pub/Sub. `Google Cloud Pub/Sub`_ is a fully-managed real-time messaging service that allows you to send and receive messages between independent applications. @@ -54,6 +58,10 @@ Samples Quickstart +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/quickstart.py;pubsub/cloud-client/README.rst + + To run this sample: @@ -66,6 +74,10 @@ To run this sample: Publisher +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/publisher.py;pubsub/cloud-client/README.rst + + To run this sample: @@ -78,13 +90,13 @@ To run this sample: project {list,create,delete,publish,publish-with-futures,publish-with-batch-settings} ... - + This application demonstrates how to perform basic operations on topics with the Cloud Pub/Sub API. - + For more information, see the README.md under /pubsub and the documentation at https://cloud.google.com/pubsub/docs. - + positional arguments: project Your Google Cloud project ID {list,create,delete,publish,publish-with-futures,publish-with-batch-settings} @@ -98,14 +110,19 @@ To run this sample: publish-with-batch-settings Publishes multiple messages to a Pub/Sub topic with batch settings. - + optional arguments: -h, --help show this help message and exit + Subscribers +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/subscriber.py;pubsub/cloud-client/README.rst + + To run this sample: @@ -115,33 +132,40 @@ To run this sample: $ python subscriber.py usage: subscriber.py [-h] - project {list,create,delete,receive,receive-flow-control} + project + {list_in_topic,list_in_project,create,delete,receive,receive-flow-control} ... - + This application demonstrates how to perform basic operations on subscriptions with the Cloud Pub/Sub API. - + For more information, see the README.md under /pubsub and the documentation at https://cloud.google.com/pubsub/docs. - + positional arguments: project Your Google Cloud project ID - {list,create,delete,receive,receive-flow-control} - list Lists all subscriptions for a given topic. + {list_in_topic,list_in_project,create,delete,receive,receive-flow-control} + list_in_topic Lists all subscriptions for a given topic. + list_in_project Lists all subscriptions in the current project. create Create a new pull subscription on the given topic. delete Deletes an existing Pub/Sub topic. receive Receives messages from a pull subscription. receive-flow-control Receives messages from a pull subscription with flow control. - + optional arguments: -h, --help show this help message and exit + Identity and Access Management +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/iam.py;pubsub/cloud-client/README.rst + + To run this sample: @@ -154,13 +178,13 @@ To run this sample: project {get-topic-policy,get-subscription-policy,set-topic-policy,set-subscription-policy,check-topic-permissions,check-subscription-permissions} ... - + This application demonstrates how to perform basic operations on IAM policies with the Cloud Pub/Sub API. - + For more information, see the README.md under /pubsub and the documentation at https://cloud.google.com/pubsub/docs. - + positional arguments: project Your Google Cloud project ID {get-topic-policy,get-subscription-policy,set-topic-policy,set-subscription-policy,check-topic-permissions,check-subscription-permissions} @@ -176,13 +200,14 @@ To run this sample: check-subscription-permissions Checks to which permissions are available on the given subscription. - + optional arguments: -h, --help show this help message and exit + The client library ------------------------------------------------------------------------------- diff --git a/samples/snippets/README.rst.in b/samples/snippets/README.rst.in index 6a9fd00c7..ddbc64712 100644 --- a/samples/snippets/README.rst.in +++ b/samples/snippets/README.rst.in @@ -26,3 +26,5 @@ samples: show_help: true cloud_client_library: true + +folder: pubsub/cloud-client \ No newline at end of file From d099168666c7656293fca602f8680d5f3da8134f Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 11 Dec 2017 09:45:02 -0800 Subject: [PATCH 036/101] Auto-update dependencies. [(#1263)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1263) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index b6977877d..bf0da6a02 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.29.2 +google-cloud-pubsub==0.29.3 From 70ec3ec4c4419f9c51164e6136c63fe29b022d94 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 12 Dec 2017 09:26:42 -0800 Subject: [PATCH 037/101] Auto-update dependencies. [(#1272)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1272) * Auto-update dependencies. * Update requirements.txt --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index bf0da6a02..23b9f7a6c 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.29.3 +google-cloud-pubsub==0.29.4 From e4181e18f085139519ae6c2327902aa620869122 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 2 Jan 2018 14:02:47 -0800 Subject: [PATCH 038/101] Auto-update dependencies. [(#1282)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1282) * Auto-update dependencies. * Fix storage acl sample Change-Id: I413bea899fdde4c4859e4070a9da25845b81f7cf --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 23b9f7a6c..8c845ba26 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.29.4 +google-cloud-pubsub==0.30.1 From 489b3813b4c1aa3b454abe8a4cd8b785b88d382d Mon Sep 17 00:00:00 2001 From: noerog <32459203+noerog@users.noreply.github.com> Date: Mon, 8 Jan 2018 17:16:12 -0500 Subject: [PATCH 039/101] Add listen for errors sample. [(#1306)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1306) * Add listen for errors sample. * Update subscriber.py * Update subscriber.py --- samples/snippets/subscriber.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 9dded25b1..577e77cb9 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -110,6 +110,30 @@ def callback(message): time.sleep(60) +def listen_for_errors(project, subscription_name): + """Receives messages and catches errors from a pull subscription.""" + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path( + project, subscription_name) + + def callback(message): + print('Received message: {}'.format(message)) + message.ack() + + subscription = subscriber.subscribe(subscription_path, callback=callback) + + # Blocks the thread while messages are coming in through the stream. Any + # exceptions that crop up on the thread will be set on the future. + future = subscription.open(callback) + try: + future.result() + except Exception as e: + print( + 'Listening for messages on {} threw an Exception: {}.'.format( + subscription_name, e)) + raise + + if __name__ == '__main__': parser = argparse.ArgumentParser( description=__doc__, @@ -143,6 +167,10 @@ def callback(message): help=receive_messages_with_flow_control.__doc__) receive_with_flow_control_parser.add_argument('subscription_name') + listen_for_errors_parser = subparsers.add_parser( + 'listen_for_errors', help=listen_for_errors.__doc__) + listen_for_errors_parser.add_argument('subscription_name') + args = parser.parse_args() if args.command == 'list_in_topic': @@ -160,3 +188,5 @@ def callback(message): elif args.command == 'receive-flow-control': receive_messages_with_flow_control( args.project, args.subscription_name) + elif args.command == 'listen_for_errors': + listen_for_errors(args.project, args.subscription_name) From 010ad9dced46a04de127ba3a9e21063a44d2a166 Mon Sep 17 00:00:00 2001 From: L J Date: Wed, 17 Jan 2018 13:27:44 -0500 Subject: [PATCH 040/101] Fix subscription.open get called twice in the client libraries [(#1321)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1321) --- samples/snippets/subscriber.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 577e77cb9..1fd480e59 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -124,9 +124,8 @@ def callback(message): # Blocks the thread while messages are coming in through the stream. Any # exceptions that crop up on the thread will be set on the future. - future = subscription.open(callback) try: - future.result() + subscription.future.result() except Exception as e: print( 'Listening for messages on {} threw an Exception: {}.'.format( From 0f83623c346a8ef56a6d9069a21cf9570f591fd0 Mon Sep 17 00:00:00 2001 From: noerog <32459203+noerog@users.noreply.github.com> Date: Thu, 25 Jan 2018 13:41:26 -0500 Subject: [PATCH 041/101] Add tests for creating push subscription. [(#1332)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1332) This is a separate PR from actually adding the sample, which is in https://github.com/GoogleCloudPlatform/python-docs-samples/pull/1331. --- samples/snippets/subscriber_test.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index 8b5e97ac4..0999e1218 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -25,6 +25,7 @@ PROJECT = os.environ['GCLOUD_PROJECT'] TOPIC = 'subscription-test-topic' SUBSCRIPTION = 'subscription-test-subscription' +ENDPOINT = 'https://{}.appspot.com/push'.format(PROJECT) @pytest.fixture(scope='module') @@ -97,6 +98,21 @@ def _(): assert subscriber_client.get_subscription(subscription_path) +def test_create_push(subscriber_client): + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION) + try: + subscriber_client.delete_subscription(subscription_path) + except Exception: + pass + + subscriber.create_push_subscription(PROJECT, TOPIC, SUBSCRIPTION, ENDPOINT) + + @eventually_consistent.call + def _(): + assert subscriber_client.get_subscription(subscription_path) + + def test_delete(subscriber_client, subscription): subscriber.delete_subscription(PROJECT, SUBSCRIPTION) From 3d5cedc6a1993b8d74331e0c5636efccc9e1e4e5 Mon Sep 17 00:00:00 2001 From: noerog <32459203+noerog@users.noreply.github.com> Date: Thu, 25 Jan 2018 13:41:43 -0500 Subject: [PATCH 042/101] Add create push subscription sample. [(#1331)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1331) --- samples/snippets/subscriber.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 1fd480e59..f8558e39f 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -58,6 +58,26 @@ def create_subscription(project, topic_name, subscription_name): print('Subscription created: {}'.format(subscription)) +def create_push_subscription(project, + topic_name, + subscription_name, + endpoint): + """Create a new push subscription on the given topic.""" + subscriber = pubsub_v1.SubscriberClient() + topic_path = subscriber.topic_path(project, topic_name) + subscription_path = subscriber.subscription_path( + project, subscription_name) + + push_config = pubsub_v1.types.PushConfig( + push_endpoint=endpoint) + + subscription = subscriber.create_subscription( + subscription_path, topic_path, push_config) + + print('Push subscription created: {}'.format(subscription)) + print('Endpoint for subscription is: {}'.format(endpoint)) + + def delete_subscription(project, subscription_name): """Deletes an existing Pub/Sub topic.""" subscriber = pubsub_v1.SubscriberClient() @@ -153,6 +173,12 @@ def callback(message): create_parser.add_argument('topic_name') create_parser.add_argument('subscription_name') + create_push_parser = subparsers.add_parser( + 'create-push', help=create_push_subscription.__doc__) + create_push_parser.add_argument('topic_name') + create_push_parser.add_argument('subscription_name') + create_push_parser.add_argument('endpoint') + delete_parser = subparsers.add_parser( 'delete', help=delete_subscription.__doc__) delete_parser.add_argument('subscription_name') @@ -179,6 +205,12 @@ def callback(message): elif args.command == 'create': create_subscription( args.project, args.topic_name, args.subscription_name) + elif args.command == 'create-push': + create_push_subscription( + args.project, + args.topic_name, + args.subscription_name, + args.endpoint) elif args.command == 'delete': delete_subscription( args.project, args.subscription_name) From 79d2279e21e3b20b5d061fbd7ddb9168e14dc072 Mon Sep 17 00:00:00 2001 From: noerog <32459203+noerog@users.noreply.github.com> Date: Fri, 2 Feb 2018 01:15:27 -0500 Subject: [PATCH 043/101] Add sample for updating a subscription. [(#1335)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1335) --- samples/snippets/subscriber.py | 36 ++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index f8558e39f..b1a793272 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -89,6 +89,34 @@ def delete_subscription(project, subscription_name): print('Subscription deleted: {}'.format(subscription_path)) +def update_subscription(project, subscription_name, ack_deadline_seconds): + """ + Updates an existing Pub/Sub subscription's ackDeadlineSeconds + from 10 seconds (default). Note that certain properties of a + subscription, such as its topic, are not modifiable. + """ + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path( + project, subscription_name) + + subscription = pubsub_v1.types.Subscription( + name=subscription_path, + ack_deadline_seconds=ack_deadline_seconds) + + update_mask = { + 'paths': { + 'ack_deadline_seconds', + } + } + + subscriber.update_subscription(subscription, update_mask) + result = subscriber.get_subscription(subscription_path) + + print('Subscription updated: {}'.format(subscription_path)) + print('New ack_deadline_seconds value is: {}'.format( + result.ack_deadline_seconds)) + + def receive_messages(project, subscription_name): """Receives messages from a pull subscription.""" subscriber = pubsub_v1.SubscriberClient() @@ -183,6 +211,11 @@ def callback(message): 'delete', help=delete_subscription.__doc__) delete_parser.add_argument('subscription_name') + update_parser = subparsers.add_parser( + 'update', help=update_subscription.__doc__) + update_parser.add_argument('subscription_name') + update_parser.add_argument('ack_deadline_seconds', type=int) + receive_parser = subparsers.add_parser( 'receive', help=receive_messages.__doc__) receive_parser.add_argument('subscription_name') @@ -214,6 +247,9 @@ def callback(message): elif args.command == 'delete': delete_subscription( args.project, args.subscription_name) + elif args.command == 'update': + update_subscription( + args.project, args.subscription_name, args.ack_deadline_seconds) elif args.command == 'receive': receive_messages(args.project, args.subscription_name) elif args.command == 'receive-flow-control': From 68e80ff11840f37fcd39a20682d2920502892361 Mon Sep 17 00:00:00 2001 From: noerog <32459203+noerog@users.noreply.github.com> Date: Mon, 5 Feb 2018 11:55:13 -0500 Subject: [PATCH 044/101] Change update_subscription to change endpoint URL. [(#1344)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1344) The documentation specifies that the update subscription commands show how to update an endpoint URL: https://cloud.google.com/pubsub/docs/admin#update_a_subscription. --- samples/snippets/subscriber.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index b1a793272..f03039bbc 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -62,7 +62,10 @@ def create_push_subscription(project, topic_name, subscription_name, endpoint): - """Create a new push subscription on the given topic.""" + """Create a new push subscription on the given topic. + For example, endpoint is + "https://my-test-project.appspot.com/push". + """ subscriber = pubsub_v1.SubscriberClient() topic_path = subscriber.topic_path(project, topic_name) subscription_path = subscriber.subscription_path( @@ -89,23 +92,27 @@ def delete_subscription(project, subscription_name): print('Subscription deleted: {}'.format(subscription_path)) -def update_subscription(project, subscription_name, ack_deadline_seconds): +def update_subscription(project, subscription_name, endpoint): """ - Updates an existing Pub/Sub subscription's ackDeadlineSeconds - from 10 seconds (default). Note that certain properties of a - subscription, such as its topic, are not modifiable. + Updates an existing Pub/Sub subscription's push endpoint URL. + Note that certain properties of a subscription, such as + its topic, are not modifiable. For example, endpoint is + "https://my-test-project.appspot.com/push". """ subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( project, subscription_name) + push_config = pubsub_v1.types.PushConfig( + push_endpoint=endpoint) + subscription = pubsub_v1.types.Subscription( name=subscription_path, - ack_deadline_seconds=ack_deadline_seconds) + push_config=push_config) update_mask = { 'paths': { - 'ack_deadline_seconds', + 'push_config', } } @@ -113,8 +120,8 @@ def update_subscription(project, subscription_name, ack_deadline_seconds): result = subscriber.get_subscription(subscription_path) print('Subscription updated: {}'.format(subscription_path)) - print('New ack_deadline_seconds value is: {}'.format( - result.ack_deadline_seconds)) + print('New endpoint for subscription is: {}'.format( + result.push_config)) def receive_messages(project, subscription_name): @@ -214,7 +221,7 @@ def callback(message): update_parser = subparsers.add_parser( 'update', help=update_subscription.__doc__) update_parser.add_argument('subscription_name') - update_parser.add_argument('ack_deadline_seconds', type=int) + update_parser.add_argument('endpoint') receive_parser = subparsers.add_parser( 'receive', help=receive_messages.__doc__) @@ -249,7 +256,7 @@ def callback(message): args.project, args.subscription_name) elif args.command == 'update': update_subscription( - args.project, args.subscription_name, args.ack_deadline_seconds) + args.project, args.subscription_name, args.endpoint) elif args.command == 'receive': receive_messages(args.project, args.subscription_name) elif args.command == 'receive-flow-control': From 4b4a2051305fe1a59831b294b10f4a59bac9c44a Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 26 Feb 2018 09:03:37 -0800 Subject: [PATCH 045/101] Auto-update dependencies. [(#1359)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1359) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 8c845ba26..d300b96a8 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.30.1 +google-cloud-pubsub==0.32.0 From b9e39a31f3e1314185fcc7f9fa54f1863c86b63f Mon Sep 17 00:00:00 2001 From: DPE bot Date: Thu, 8 Mar 2018 13:33:57 -0800 Subject: [PATCH 046/101] Auto-update dependencies. [(#1389)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1389) --- samples/snippets/requirements.txt | 2 +- samples/snippets/subscriber.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index d300b96a8..fdea342db 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.32.0 +google-cloud-pubsub==0.32.1 diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index f03039bbc..827dcd313 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -29,11 +29,11 @@ def list_subscriptions_in_topic(project, topic_name): """Lists all subscriptions for a given topic.""" - subscriber = pubsub_v1.SubscriberClient() + subscriber = pubsub_v1.PublisherClient() topic_path = subscriber.topic_path(project, topic_name) - for subscription in subscriber.list_subscriptions(topic_path): - print(subscription.name) + for subscription in subscriber.list_topic_subscriptions(topic_path): + print(subscription) def list_subscriptions_in_project(project): From d92b5108f52b6de47980b112b0f1909836aea0f6 Mon Sep 17 00:00:00 2001 From: chenyumic Date: Fri, 16 Mar 2018 16:41:13 -0700 Subject: [PATCH 047/101] Added sample for publishing/receiving messages with custom attributes [(#1409)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1409) --- samples/snippets/publisher.py | 24 ++++++++++++++++++++++ samples/snippets/publisher_test.py | 7 +++++++ samples/snippets/subscriber.py | 32 +++++++++++++++++++++++++++++ samples/snippets/subscriber_test.py | 21 +++++++++++++++++++ 4 files changed, 84 insertions(+) diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index cf12b2626..76d058993 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -69,6 +69,23 @@ def publish_messages(project, topic_name): print('Published messages.') +def publish_messages_with_custom_attributes(project, topic_name): + """Publishes multiple messages with custom attributes + to a Pub/Sub topic.""" + publisher = pubsub_v1.PublisherClient() + topic_path = publisher.topic_path(project, topic_name) + + for n in range(1, 10): + data = u'Message number {}'.format(n) + # Data must be a bytestring + data = data.encode('utf-8') + # Add two attributes, origin and username, to the message + publisher.publish( + topic_path, data, origin='python-sample', username='gcp') + + print('Published messages with custom attributes.') + + def publish_messages_with_futures(project, topic_name): """Publishes multiple messages to a Pub/Sub topic and prints their message IDs.""" @@ -132,6 +149,11 @@ def publish_messages_with_batch_settings(project, topic_name): 'publish', help=publish_messages.__doc__) publish_parser.add_argument('topic_name') + publish_with_custom_attributes_parser = subparsers.add_parser( + 'publish-with-custom-attributes', + help=publish_messages_with_custom_attributes.__doc__) + publish_with_custom_attributes_parser.add_argument('topic_name') + publish_with_futures_parser = subparsers.add_parser( 'publish-with-futures', help=publish_messages_with_futures.__doc__) @@ -152,6 +174,8 @@ def publish_messages_with_batch_settings(project, topic_name): delete_topic(args.project, args.topic_name) elif args.command == 'publish': publish_messages(args.project, args.topic_name) + elif args.command == 'publish-with-custom-attributes': + publish_messages_with_custom_attributes(args.project, args.topic_name) elif args.command == 'publish-with-futures': publish_messages_with_futures(args.project, args.topic_name) elif args.command == 'publish-with-batch-settings': diff --git a/samples/snippets/publisher_test.py b/samples/snippets/publisher_test.py index 120148c0a..a008a0610 100644 --- a/samples/snippets/publisher_test.py +++ b/samples/snippets/publisher_test.py @@ -81,6 +81,13 @@ def test_publish(topic, capsys): assert 'Published' in out +def test_publish_with_custom_attributes(topic, capsys): + publisher.publish_messages_with_custom_attributes(PROJECT, TOPIC) + + out, _ = capsys.readouterr() + assert 'Published' in out + + def test_publish_with_batch_settings(topic, capsys): publisher.publish_messages_with_batch_settings(PROJECT, TOPIC) diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 827dcd313..f68c06c95 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -143,6 +143,30 @@ def callback(message): time.sleep(60) +def receive_messages_with_custom_attributes(project, subscription_name): + """Receives messages from a pull subscription.""" + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path( + project, subscription_name) + + def callback(message): + print('Received message: {}'.format(message.data)) + if message.attributes: + print('Attributes:') + for key in message.attributes: + value = message.attributes.get(key) + print('{}: {}'.format(key, value)) + message.ack() + + subscriber.subscribe(subscription_path, callback=callback) + + # The subscriber is non-blocking, so we must keep the main thread from + # exiting to allow it to process messages in the background. + print('Listening for messages on {}'.format(subscription_path)) + while True: + time.sleep(60) + + def receive_messages_with_flow_control(project, subscription_name): """Receives messages from a pull subscription with flow control.""" subscriber = pubsub_v1.SubscriberClient() @@ -227,6 +251,11 @@ def callback(message): 'receive', help=receive_messages.__doc__) receive_parser.add_argument('subscription_name') + receive_with_custom_attributes_parser = subparsers.add_parser( + 'receive-custom-attributes', + help=receive_messages_with_custom_attributes.__doc__) + receive_with_custom_attributes_parser.add_argument('subscription_name') + receive_with_flow_control_parser = subparsers.add_parser( 'receive-flow-control', help=receive_messages_with_flow_control.__doc__) @@ -259,6 +288,9 @@ def callback(message): args.project, args.subscription_name, args.endpoint) elif args.command == 'receive': receive_messages(args.project, args.subscription_name) + elif args.command == 'receive-custom-attributes': + receive_messages_with_custom_attributes( + args.project, args.subscription_name) elif args.command == 'receive-flow-control': receive_messages_with_flow_control( args.project, args.subscription_name) diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index 0999e1218..f04373ae8 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -129,6 +129,11 @@ def _publish_messages(publisher_client, topic): topic, data=data) +def _publish_messages_with_custom_attributes(publisher_client, topic): + data = u'Test message'.encode('utf-8') + publisher_client.publish(topic, data=data, origin='python-sample') + + def _make_sleep_patch(): real_sleep = time.sleep @@ -155,6 +160,22 @@ def test_receive(publisher_client, topic, subscription, capsys): assert 'Message 1' in out +def test_receive_with_custom_attributes( + publisher_client, topic, subscription, capsys): + _publish_messages_with_custom_attributes(publisher_client, topic) + + with _make_sleep_patch(): + with pytest.raises(RuntimeError, match='sigil'): + subscriber.receive_messages_with_custom_attributes( + PROJECT, SUBSCRIPTION) + + out, _ = capsys.readouterr() + assert 'Test message' in out + assert 'Attributes' in out + assert 'origin' in out + assert 'python-sample' in out + + def test_receive_with_flow_control( publisher_client, topic, subscription, capsys): _publish_messages(publisher_client, topic) From ca14620f94f722a1442dc30c52a83f2df3efb724 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 2 Apr 2018 02:51:10 -0700 Subject: [PATCH 048/101] Auto-update dependencies. --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index fdea342db..c74eb4e3c 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.32.1 +google-cloud-pubsub==0.33.0 From a4b8f306041ec568b8947735d637dd812023f369 Mon Sep 17 00:00:00 2001 From: chenyumic Date: Fri, 6 Apr 2018 22:57:36 -0700 Subject: [PATCH 049/101] Regenerate the README files and fix the Open in Cloud Shell link for some samples [(#1441)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1441) --- samples/snippets/README.rst | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index 0ecf4b193..209ab5d7f 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -12,7 +12,7 @@ This directory contains samples for Google Cloud Pub/Sub. `Google Cloud Pub/Sub` -.. _Google Cloud Pub/Sub: https://cloud.google.com/pubsub/docs +.. _Google Cloud Pub/Sub: https://cloud.google.com/pubsub/docs Setup ------------------------------------------------------------------------------- @@ -59,7 +59,7 @@ Quickstart +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/quickstart.py;pubsub/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/quickstart.py,pubsub/cloud-client/README.rst @@ -75,7 +75,7 @@ Publisher +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/publisher.py;pubsub/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/publisher.py,pubsub/cloud-client/README.rst @@ -88,7 +88,7 @@ To run this sample: usage: publisher.py [-h] project - {list,create,delete,publish,publish-with-futures,publish-with-batch-settings} + {list,create,delete,publish,publish-with-custom-attributes,publish-with-futures,publish-with-batch-settings} ... This application demonstrates how to perform basic operations on topics @@ -99,11 +99,14 @@ To run this sample: positional arguments: project Your Google Cloud project ID - {list,create,delete,publish,publish-with-futures,publish-with-batch-settings} + {list,create,delete,publish,publish-with-custom-attributes,publish-with-futures,publish-with-batch-settings} list Lists all Pub/Sub topics in the given project. create Create a new Pub/Sub topic. delete Deletes an existing Pub/Sub topic. publish Publishes multiple messages to a Pub/Sub topic. + publish-with-custom-attributes + Publishes multiple messages with custom attributes to + a Pub/Sub topic. publish-with-futures Publishes multiple messages to a Pub/Sub topic and prints their message IDs. @@ -120,7 +123,7 @@ Subscribers +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/subscriber.py;pubsub/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/subscriber.py,pubsub/cloud-client/README.rst @@ -133,7 +136,7 @@ To run this sample: usage: subscriber.py [-h] project - {list_in_topic,list_in_project,create,delete,receive,receive-flow-control} + {list_in_topic,list_in_project,create,create-push,delete,update,receive,receive-custom-attributes,receive-flow-control,listen_for_errors} ... This application demonstrates how to perform basic operations on @@ -144,15 +147,27 @@ To run this sample: positional arguments: project Your Google Cloud project ID - {list_in_topic,list_in_project,create,delete,receive,receive-flow-control} + {list_in_topic,list_in_project,create,create-push,delete,update,receive,receive-custom-attributes,receive-flow-control,listen_for_errors} list_in_topic Lists all subscriptions for a given topic. list_in_project Lists all subscriptions in the current project. create Create a new pull subscription on the given topic. + create-push Create a new push subscription on the given topic. For + example, endpoint is "https://my-test- + project.appspot.com/push". delete Deletes an existing Pub/Sub topic. + update Updates an existing Pub/Sub subscription's push + endpoint URL. Note that certain properties of a + subscription, such as its topic, are not modifiable. + For example, endpoint is "https://my-test- + project.appspot.com/push". receive Receives messages from a pull subscription. + receive-custom-attributes + Receives messages from a pull subscription. receive-flow-control Receives messages from a pull subscription with flow control. + listen_for_errors Receives messages and catches errors from a pull + subscription. optional arguments: -h, --help show this help message and exit @@ -163,7 +178,7 @@ Identity and Access Management +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/iam.py;pubsub/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/iam.py,pubsub/cloud-client/README.rst From 912307938b6de77fe585ca07d48e4c1e5e82ef14 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Thu, 26 Apr 2018 10:26:41 -0700 Subject: [PATCH 050/101] Update READMEs to fix numbering and add git clone [(#1464)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1464) --- samples/snippets/README.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index 209ab5d7f..e0e265f8d 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -31,10 +31,16 @@ credentials for applications. Install Dependencies ++++++++++++++++++++ +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + #. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. - .. _Python Development Environment Setup Guide: - https://cloud.google.com/python/setup + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup #. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. From 6f6c60cc0de639c8e90c6d7b3dc7b4c9c2e554b0 Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Mon, 21 May 2018 15:05:29 -0700 Subject: [PATCH 051/101] PubSub: adds region tags and updates existing to standard [(#1491)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1491) --- samples/snippets/iam.py | 12 ++++++++++++ samples/snippets/publisher.py | 14 ++++++++++++++ samples/snippets/quickstart.py | 4 ++-- samples/snippets/subscriber.py | 32 ++++++++++++++++++++++++++------ 4 files changed, 54 insertions(+), 8 deletions(-) diff --git a/samples/snippets/iam.py b/samples/snippets/iam.py index b46bc1147..bd44f1ab6 100644 --- a/samples/snippets/iam.py +++ b/samples/snippets/iam.py @@ -28,6 +28,7 @@ def get_topic_policy(project, topic_name): """Prints the IAM policy for the given topic.""" + # [START pubsub_get_topic_policy] client = pubsub_v1.PublisherClient() topic_path = client.topic_path(project, topic_name) @@ -36,10 +37,12 @@ def get_topic_policy(project, topic_name): print('Policy for topic {}:'.format(topic_path)) for binding in policy.bindings: print('Role: {}, Members: {}'.format(binding.role, binding.members)) + # [END pubsub_get_topic_policy] def get_subscription_policy(project, subscription_name): """Prints the IAM policy for the given subscription.""" + # [START pubsub_get_subscription_policy] client = pubsub_v1.SubscriberClient() subscription_path = client.subscription_path(project, subscription_name) @@ -48,10 +51,12 @@ def get_subscription_policy(project, subscription_name): print('Policy for subscription {}:'.format(subscription_path)) for binding in policy.bindings: print('Role: {}, Members: {}'.format(binding.role, binding.members)) + # [END pubsub_get_subscription_policy] def set_topic_policy(project, topic_name): """Sets the IAM policy for a topic.""" + # [START pubsub_set_topic_policy] client = pubsub_v1.PublisherClient() topic_path = client.topic_path(project, topic_name) @@ -72,10 +77,12 @@ def set_topic_policy(project, topic_name): print('IAM policy for topic {} set: {}'.format( topic_name, policy)) + # [END pubsub_set_topic_policy] def set_subscription_policy(project, subscription_name): """Sets the IAM policy for a topic.""" + # [START pubsub_set_subscription_policy] client = pubsub_v1.SubscriberClient() subscription_path = client.subscription_path(project, subscription_name) @@ -96,10 +103,12 @@ def set_subscription_policy(project, subscription_name): print('IAM policy for subscription {} set: {}'.format( subscription_name, policy)) + # [END pubsub_set_subscription_policy] def check_topic_permissions(project, topic_name): """Checks to which permissions are available on the given topic.""" + # [START pubsub_test_topic_permissions] client = pubsub_v1.PublisherClient() topic_path = client.topic_path(project, topic_name) @@ -113,10 +122,12 @@ def check_topic_permissions(project, topic_name): print('Allowed permissions for topic {}: {}'.format( topic_path, allowed_permissions)) + # [END pubsub_test_topic_permissions] def check_subscription_permissions(project, subscription_name): """Checks to which permissions are available on the given subscription.""" + # [START pubsub_test_subscription_permissions] client = pubsub_v1.SubscriberClient() subscription_path = client.subscription_path(project, subscription_name) @@ -130,6 +141,7 @@ def check_subscription_permissions(project, subscription_name): print('Allowed permissions for subscription {}: {}'.format( subscription_path, allowed_permissions)) + # [END pubsub_test_subscription_permissions] if __name__ == '__main__': diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index 76d058993..f2f5e3ac6 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -28,35 +28,42 @@ def list_topics(project): """Lists all Pub/Sub topics in the given project.""" + # [START pubsub_list_topics] publisher = pubsub_v1.PublisherClient() project_path = publisher.project_path(project) for topic in publisher.list_topics(project_path): print(topic) + # [END pubsub_list_topics] def create_topic(project, topic_name): """Create a new Pub/Sub topic.""" + # [START pubsub_create_topic] publisher = pubsub_v1.PublisherClient() topic_path = publisher.topic_path(project, topic_name) topic = publisher.create_topic(topic_path) print('Topic created: {}'.format(topic)) + # [END pubsub_create_topic] def delete_topic(project, topic_name): """Deletes an existing Pub/Sub topic.""" + # [START pubsub_delete_topic] publisher = pubsub_v1.PublisherClient() topic_path = publisher.topic_path(project, topic_name) publisher.delete_topic(topic_path) print('Topic deleted: {}'.format(topic_path)) + # [END pubsub_delete_topic] def publish_messages(project, topic_name): """Publishes multiple messages to a Pub/Sub topic.""" + # [START pubsub_quickstart_publisher] publisher = pubsub_v1.PublisherClient() topic_path = publisher.topic_path(project, topic_name) @@ -67,11 +74,13 @@ def publish_messages(project, topic_name): publisher.publish(topic_path, data=data) print('Published messages.') + # [END pubsub_quickstart_publisher] def publish_messages_with_custom_attributes(project, topic_name): """Publishes multiple messages with custom attributes to a Pub/Sub topic.""" + # [START pubsub_publish_custom_attributes] publisher = pubsub_v1.PublisherClient() topic_path = publisher.topic_path(project, topic_name) @@ -84,11 +93,13 @@ def publish_messages_with_custom_attributes(project, topic_name): topic_path, data, origin='python-sample', username='gcp') print('Published messages with custom attributes.') + # [END pubsub_publish_custom_attributes] def publish_messages_with_futures(project, topic_name): """Publishes multiple messages to a Pub/Sub topic and prints their message IDs.""" + # [START pubsub_publisher_concurrency_control] publisher = pubsub_v1.PublisherClient() topic_path = publisher.topic_path(project, topic_name) @@ -107,10 +118,12 @@ def publish_messages_with_futures(project, topic_name): for future in futures: # result() blocks until the message is published. print(future.result()) + # [END pubsub_publisher_concurrency_control] def publish_messages_with_batch_settings(project, topic_name): """Publishes multiple messages to a Pub/Sub topic with batch settings.""" + # [START pubsub_publisher_batch_settings] # Configure the batch to publish once there is one kilobyte of data or # 1 second has passed. batch_settings = pubsub_v1.types.BatchSettings( @@ -127,6 +140,7 @@ def publish_messages_with_batch_settings(project, topic_name): publisher.publish(topic_path, data=data) print('Published messages.') + # [END pubsub_publisher_batch_settings] if __name__ == '__main__': diff --git a/samples/snippets/quickstart.py b/samples/snippets/quickstart.py index c9823d789..1ff2efed3 100644 --- a/samples/snippets/quickstart.py +++ b/samples/snippets/quickstart.py @@ -16,7 +16,7 @@ def run_quickstart(): - # [START pubsub_quickstart] + # [START pubsub_quickstart_create_topic] # Imports the Google Cloud client library from google.cloud import pubsub_v1 @@ -32,7 +32,7 @@ def run_quickstart(): topic = publisher.create_topic(topic_path) print('Topic created: {}'.format(topic)) - # [END pubsub_quickstart] + # [END pubsub_quickstart_create_topic] if __name__ == '__main__': diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index f68c06c95..34f2301d8 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -29,24 +29,29 @@ def list_subscriptions_in_topic(project, topic_name): """Lists all subscriptions for a given topic.""" + # [START pubsub_list_topic_subscriptions] subscriber = pubsub_v1.PublisherClient() topic_path = subscriber.topic_path(project, topic_name) for subscription in subscriber.list_topic_subscriptions(topic_path): print(subscription) + # [END pubsub_list_topic_subscriptions] def list_subscriptions_in_project(project): """Lists all subscriptions in the current project.""" + # [START pubsub_list_subscriptions] subscriber = pubsub_v1.SubscriberClient() project_path = subscriber.project_path(project) for subscription in subscriber.list_subscriptions(project_path): print(subscription.name) + # [END pubsub_list_subscriptions] def create_subscription(project, topic_name, subscription_name): """Create a new pull subscription on the given topic.""" + # [START pubsub_create_pull_subscription] subscriber = pubsub_v1.SubscriberClient() topic_path = subscriber.topic_path(project, topic_name) subscription_path = subscriber.subscription_path( @@ -56,16 +61,16 @@ def create_subscription(project, topic_name, subscription_name): subscription_path, topic_path) print('Subscription created: {}'.format(subscription)) + # [END pubsub_create_pull_subscription] def create_push_subscription(project, topic_name, subscription_name, endpoint): - """Create a new push subscription on the given topic. - For example, endpoint is - "https://my-test-project.appspot.com/push". - """ + """Create a new push subscription on the given topic.""" + # [START pubsub_create_push_subscription] + # endpoint = "https://my-test-project.appspot.com/push" subscriber = pubsub_v1.SubscriberClient() topic_path = subscriber.topic_path(project, topic_name) subscription_path = subscriber.subscription_path( @@ -79,10 +84,12 @@ def create_push_subscription(project, print('Push subscription created: {}'.format(subscription)) print('Endpoint for subscription is: {}'.format(endpoint)) + # [END pubsub_create_push_subscription] def delete_subscription(project, subscription_name): """Deletes an existing Pub/Sub topic.""" + # [START pubsub_delete_subscription] subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( project, subscription_name) @@ -90,15 +97,17 @@ def delete_subscription(project, subscription_name): subscriber.delete_subscription(subscription_path) print('Subscription deleted: {}'.format(subscription_path)) + # [END pubsub_delete_subscription] def update_subscription(project, subscription_name, endpoint): """ Updates an existing Pub/Sub subscription's push endpoint URL. Note that certain properties of a subscription, such as - its topic, are not modifiable. For example, endpoint is - "https://my-test-project.appspot.com/push". + its topic, are not modifiable. """ + # [START pubsub_update_push_configuration] + # endpoint = "https://my-test-project.appspot.com/push" subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( project, subscription_name) @@ -122,10 +131,13 @@ def update_subscription(project, subscription_name, endpoint): print('Subscription updated: {}'.format(subscription_path)) print('New endpoint for subscription is: {}'.format( result.push_config)) + # [END pubsub_update_push_configuration] def receive_messages(project, subscription_name): """Receives messages from a pull subscription.""" + # [START pubsub_subscriber_async_pull] + # [START pubsub_quickstart_subscriber] subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( project, subscription_name) @@ -141,10 +153,13 @@ def callback(message): print('Listening for messages on {}'.format(subscription_path)) while True: time.sleep(60) + # [END pubsub_subscriber_async_pull] + # [END pubsub_quickstart_subscriber] def receive_messages_with_custom_attributes(project, subscription_name): """Receives messages from a pull subscription.""" + # [START pubsub_subscriber_sync_pull_custom_attributes] subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( project, subscription_name) @@ -165,10 +180,12 @@ def callback(message): print('Listening for messages on {}'.format(subscription_path)) while True: time.sleep(60) + # [END pubsub_subscriber_sync_pull_custom_attributes] def receive_messages_with_flow_control(project, subscription_name): """Receives messages from a pull subscription with flow control.""" + # [START pubsub_subscriber_flow_settings] subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( project, subscription_name) @@ -187,10 +204,12 @@ def callback(message): print('Listening for messages on {}'.format(subscription_path)) while True: time.sleep(60) + # [END pubsub_subscriber_flow_settings] def listen_for_errors(project, subscription_name): """Receives messages and catches errors from a pull subscription.""" + # [START pubsub_subscriber_error_listener] subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( project, subscription_name) @@ -210,6 +229,7 @@ def callback(message): 'Listening for messages on {} threw an Exception: {}.'.format( subscription_name, e)) raise + # [END pubsub_subscriber_error_listener] if __name__ == '__main__': From 31b0415d7e8b3812099e807e3d81c5f8a1223083 Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Tue, 22 May 2018 09:14:19 -0700 Subject: [PATCH 052/101] Pubsub: Add missing region tag [(#1498)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1498) --- samples/snippets/publisher.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index f2f5e3ac6..f77f6ba78 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -64,6 +64,7 @@ def delete_topic(project, topic_name): def publish_messages(project, topic_name): """Publishes multiple messages to a Pub/Sub topic.""" # [START pubsub_quickstart_publisher] + # [START pubsub_publish] publisher = pubsub_v1.PublisherClient() topic_path = publisher.topic_path(project, topic_name) @@ -75,6 +76,7 @@ def publish_messages(project, topic_name): print('Published messages.') # [END pubsub_quickstart_publisher] + # [END pubsub_publish] def publish_messages_with_custom_attributes(project, topic_name): From 618bf114b1c8bf82a3f82d2b371eff0d9dbbc0d5 Mon Sep 17 00:00:00 2001 From: chenyumic Date: Thu, 21 Jun 2018 14:20:44 -0700 Subject: [PATCH 053/101] Add the Pub/Sub handle_publisher_error sample [(#1440)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1440) * Add the Pub/Sub handle_publisher_error sample * Update requirements.txt * Update publisher.py * Update publisher.py * Added region tag --- samples/snippets/publisher.py | 33 +++++++++++++++++++++++++++++++ samples/snippets/requirements.txt | 3 ++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index f77f6ba78..c24dddca9 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -22,6 +22,7 @@ """ import argparse +import concurrent.futures from google.cloud import pubsub_v1 @@ -123,6 +124,38 @@ def publish_messages_with_futures(project, topic_name): # [END pubsub_publisher_concurrency_control] +def publish_messages_with_error_handler(project, topic_name): + """Publishes multiple messages to a Pub/Sub topic with an error handler.""" + # [START pubsub_publish_messages_error_handler] + publisher = pubsub_v1.PublisherClient() + topic_path = publisher.topic_path(project, topic_name) + + # When you publish a message, the client returns a Future. This Future + # can be used to track if an error has occurred. + futures = [] + + def callback(f): + exc = f.exception() + if exc: + print('Publishing message on {} threw an Exception {}.'.format( + topic_name, exc)) + + for n in range(1, 10): + data = u'Message number {}'.format(n) + # Data must be a bytestring + data = data.encode('utf-8') + message_future = publisher.publish(topic_path, data=data) + message_future.add_done_callback(callback) + futures.append(message_future) + + # We must keep the main thread from exiting to allow it to process + # messages in the background. + concurrent.futures.wait(futures) + + print('Published messages.') + # [END pubsub_publish_messages_error_handler] + + def publish_messages_with_batch_settings(project, topic_name): """Publishes multiple messages to a Pub/Sub topic with batch settings.""" # [START pubsub_publisher_batch_settings] diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index c74eb4e3c..81f06995b 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1,2 @@ -google-cloud-pubsub==0.33.0 +google-cloud-pubsub==0.32.1 +futures==3.1.1; python_version < '3' From 59de9b82fc12cfb6920b96debba78d7187dcce23 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Thu, 19 Jul 2018 13:31:04 -0700 Subject: [PATCH 054/101] Modified publisher with error handling [(#1568)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1568) --- samples/snippets/publisher.py | 31 +++++++++++++++++------------- samples/snippets/publisher_test.py | 7 +++++++ samples/snippets/requirements.txt | 3 +-- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index c24dddca9..96caba9f4 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -22,7 +22,7 @@ """ import argparse -import concurrent.futures +import time from google.cloud import pubsub_v1 @@ -130,29 +130,27 @@ def publish_messages_with_error_handler(project, topic_name): publisher = pubsub_v1.PublisherClient() topic_path = publisher.topic_path(project, topic_name) - # When you publish a message, the client returns a Future. This Future - # can be used to track if an error has occurred. - futures = [] - - def callback(f): - exc = f.exception() - if exc: + def callback(message_future): + if message_future.exception(): print('Publishing message on {} threw an Exception {}.'.format( - topic_name, exc)) + topic_name, message_future.exception())) + else: + print(message_future.result()) for n in range(1, 10): data = u'Message number {}'.format(n) # Data must be a bytestring data = data.encode('utf-8') + # When you publish a message, the client returns a Future. message_future = publisher.publish(topic_path, data=data) message_future.add_done_callback(callback) - futures.append(message_future) + + print('Published message IDs:') # We must keep the main thread from exiting to allow it to process # messages in the background. - concurrent.futures.wait(futures) - - print('Published messages.') + while True: + time.sleep(60) # [END pubsub_publish_messages_error_handler] @@ -208,6 +206,11 @@ def publish_messages_with_batch_settings(project, topic_name): help=publish_messages_with_futures.__doc__) publish_with_futures_parser.add_argument('topic_name') + publish_with_error_handler_parser = subparsers.add_parser( + 'publish-with-error-handler', + help=publish_messages_with_error_handler.__doc__) + publish_with_error_handler_parser.add_argument('topic_name') + publish_with_batch_settings_parser = subparsers.add_parser( 'publish-with-batch-settings', help=publish_messages_with_batch_settings.__doc__) @@ -227,5 +230,7 @@ def publish_messages_with_batch_settings(project, topic_name): publish_messages_with_custom_attributes(args.project, args.topic_name) elif args.command == 'publish-with-futures': publish_messages_with_futures(args.project, args.topic_name) + elif args.command == 'publish-with-error-handler': + publish_messages_with_error_handler(args.project, args.topic_name) elif args.command == 'publish-with-batch-settings': publish_messages_with_batch_settings(args.project, args.topic_name) diff --git a/samples/snippets/publisher_test.py b/samples/snippets/publisher_test.py index a008a0610..2eda09c6c 100644 --- a/samples/snippets/publisher_test.py +++ b/samples/snippets/publisher_test.py @@ -95,6 +95,13 @@ def test_publish_with_batch_settings(topic, capsys): assert 'Published' in out +def test_publish_with_error_handler(topic, capsys): + publisher.publish_messages_with_error_handler(PROJECT, TOPIC) + + out, _ = capsys.readouterr() + assert 'Published' in out + + def test_publish_with_futures(topic, capsys): publisher.publish_messages_with_futures(PROJECT, TOPIC) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 81f06995b..c74eb4e3c 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,2 +1 @@ -google-cloud-pubsub==0.32.1 -futures==3.1.1; python_version < '3' +google-cloud-pubsub==0.33.0 From 3b41ec38dd6748cfc4670739a62187c47b444e44 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Fri, 10 Aug 2018 16:01:02 -0700 Subject: [PATCH 055/101] Updated google-cloud-pubsub to version 0.35 [(#1624)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1624) * Updated library version * Rewrote test for publish with error handler * Custom _publish function in test prints no 'Attributes' --- samples/snippets/publisher_test.py | 21 ++++++++++++++++++++- samples/snippets/requirements.txt | 2 +- samples/snippets/subscriber.py | 2 +- samples/snippets/subscriber_test.py | 1 - 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/samples/snippets/publisher_test.py b/samples/snippets/publisher_test.py index 2eda09c6c..cdb4d0e0e 100644 --- a/samples/snippets/publisher_test.py +++ b/samples/snippets/publisher_test.py @@ -13,9 +13,11 @@ # limitations under the License. import os +import time from gcp_devrel.testing import eventually_consistent from google.cloud import pubsub_v1 +import mock import pytest import publisher @@ -43,6 +45,19 @@ def topic(client): yield topic_path +def _make_sleep_patch(): + real_sleep = time.sleep + + def new_sleep(period): + if period == 60: + real_sleep(5) + raise RuntimeError('sigil') + else: + real_sleep(period) + + return mock.patch('time.sleep', new=new_sleep) + + def test_list(client, topic, capsys): @eventually_consistent.call def _(): @@ -96,7 +111,11 @@ def test_publish_with_batch_settings(topic, capsys): def test_publish_with_error_handler(topic, capsys): - publisher.publish_messages_with_error_handler(PROJECT, TOPIC) + + with _make_sleep_patch(): + with pytest.raises(RuntimeError, match='sigil'): + publisher.publish_messages_with_error_handler( + PROJECT, TOPIC) out, _ = capsys.readouterr() assert 'Published' in out diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index c74eb4e3c..23ed91dd2 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.33.0 +google-cloud-pubsub==0.35.0 diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 34f2301d8..46bb51188 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -191,7 +191,7 @@ def receive_messages_with_flow_control(project, subscription_name): project, subscription_name) def callback(message): - print('Received message: {}'.format(message)) + print('Received message: {}'.format(message.data)) message.ack() # Limit the subscriber to only have ten outstanding messages at a time. diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index f04373ae8..adbc44e84 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -171,7 +171,6 @@ def test_receive_with_custom_attributes( out, _ = capsys.readouterr() assert 'Test message' in out - assert 'Attributes' in out assert 'origin' in out assert 'python-sample' in out From 28d35a1ee6ac53c5f4e2363c4048fca8466f036d Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Wed, 15 Aug 2018 15:59:01 -0700 Subject: [PATCH 056/101] Added timeout in error handling [(#1636)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1636) --- samples/snippets/publisher.py | 3 ++- samples/snippets/subscriber.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index 96caba9f4..a577abc63 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -131,7 +131,8 @@ def publish_messages_with_error_handler(project, topic_name): topic_path = publisher.topic_path(project, topic_name) def callback(message_future): - if message_future.exception(): + # When timeout is unspecified, the exception method waits indefinitely. + if message_future.exception(timeout=30): print('Publishing message on {} threw an Exception {}.'.format( topic_name, message_future.exception())) else: diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 46bb51188..51fa96b86 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -223,12 +223,12 @@ def callback(message): # Blocks the thread while messages are coming in through the stream. Any # exceptions that crop up on the thread will be set on the future. try: - subscription.future.result() + # When timeout is unspecified, the result method waits indefinitely. + subscription.future.result(timeout=30) except Exception as e: print( 'Listening for messages on {} threw an Exception: {}.'.format( subscription_name, e)) - raise # [END pubsub_subscriber_error_listener] From 37feb9d4c96cb626272b4d89218d37cc331803de Mon Sep 17 00:00:00 2001 From: DPE bot Date: Tue, 28 Aug 2018 11:17:45 -0700 Subject: [PATCH 057/101] Auto-update dependencies. [(#1658)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1658) * Auto-update dependencies. * Rollback appengine/standard/bigquery/. * Rollback appengine/standard/iap/. * Rollback bigtable/metricscaler. * Rolledback appengine/flexible/datastore. * Rollback dataproc/ * Rollback jobs/api_client * Rollback vision/cloud-client. * Rollback functions/ocr/app. * Rollback iot/api-client/end_to_end_example. * Rollback storage/cloud-client. * Rollback kms/api-client. * Rollback dlp/ * Rollback bigquery/cloud-client. * Rollback iot/api-client/manager. * Rollback appengine/flexible/cloudsql_postgresql. --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 23ed91dd2..936a9f0ed 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.35.0 +google-cloud-pubsub==0.37.2 From d264c75ed8a20645d6d10d5803cb2abf91f3439b Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Thu, 30 Aug 2018 14:02:11 -0700 Subject: [PATCH 058/101] Added sample for Pub/Sub synchronous pull subscriber [(#1673)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1673) * Added sample for synchronous pull --- samples/snippets/subscriber.py | 45 +++++++++++++++++++++++++++++ samples/snippets/subscriber_test.py | 15 ++++++++++ 2 files changed, 60 insertions(+) diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 51fa96b86..83e5700dd 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -90,6 +90,8 @@ def create_push_subscription(project, def delete_subscription(project, subscription_name): """Deletes an existing Pub/Sub topic.""" # [START pubsub_delete_subscription] + # project = "Your Google Cloud Project ID" + # subscription_name = "Your Pubsub subscription name" subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( project, subscription_name) @@ -138,6 +140,8 @@ def receive_messages(project, subscription_name): """Receives messages from a pull subscription.""" # [START pubsub_subscriber_async_pull] # [START pubsub_quickstart_subscriber] + # project = "Your Google Cloud Project ID" + # subscription_name = "Your Pubsub subscription name" subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( project, subscription_name) @@ -160,6 +164,8 @@ def callback(message): def receive_messages_with_custom_attributes(project, subscription_name): """Receives messages from a pull subscription.""" # [START pubsub_subscriber_sync_pull_custom_attributes] + # project = "Your Google Cloud Project ID" + # subscription_name = "Your Pubsub subscription name" subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( project, subscription_name) @@ -186,6 +192,8 @@ def callback(message): def receive_messages_with_flow_control(project, subscription_name): """Receives messages from a pull subscription with flow control.""" # [START pubsub_subscriber_flow_settings] + # project = "Your Google Cloud Project ID" + # subscription_name = "Your Pubsub subscription name" subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( project, subscription_name) @@ -207,9 +215,38 @@ def callback(message): # [END pubsub_subscriber_flow_settings] +def receive_messages_synchronously(project, subscription_name): + """Pulling messages synchronously.""" + # [START pubsub_subscriber_sync_pull] + # project = "Your Google Cloud Project ID" + # subscription_name = "Your Pubsub subscription name" + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path( + project, subscription_name) + + # Builds a pull request with a specific number of messages to return. + # `return_immediately` is set to False so that the system waits (for a + # bounded amount of time) until at lease one message is available. + response = subscriber.pull( + subscription_path, + max_messages=3, + return_immediately=False) + + ack_ids = [] + for received_message in response.received_messages: + print("Received: {}".format(received_message.message.data)) + ack_ids.append(received_message.ack_id) + + # Acknowledges the received messages so they will not be sent again. + subscriber.acknowledge(subscription_path, ack_ids) + # [END pubsub_subscriber_sync_pull] + + def listen_for_errors(project, subscription_name): """Receives messages and catches errors from a pull subscription.""" # [START pubsub_subscriber_error_listener] + # project = "Your Google Cloud Project ID" + # subscription_name = "Your Pubsub subscription name" subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( project, subscription_name) @@ -281,6 +318,11 @@ def callback(message): help=receive_messages_with_flow_control.__doc__) receive_with_flow_control_parser.add_argument('subscription_name') + receive_messages_synchronously_parser = subparsers.add_parser( + 'receive-synchronously', + help=receive_messages_synchronously.__doc__) + receive_messages_synchronously_parser.add_argument('subscription_name') + listen_for_errors_parser = subparsers.add_parser( 'listen_for_errors', help=listen_for_errors.__doc__) listen_for_errors_parser.add_argument('subscription_name') @@ -314,5 +356,8 @@ def callback(message): elif args.command == 'receive-flow-control': receive_messages_with_flow_control( args.project, args.subscription_name) + elif args.command == 'receive-synchronously': + receive_messages_synchronously( + args.project, args.subscription_name) elif args.command == 'listen_for_errors': listen_for_errors(args.project, args.subscription_name) diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index adbc44e84..728f971f2 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -188,3 +188,18 @@ def test_receive_with_flow_control( assert 'Listening' in out assert subscription in out assert 'Message 1' in out + + +def test_receive_synchronously( + publisher_client, topic, subscription, capsys): + _publish_messages(publisher_client, topic) + + with _make_sleep_patch(): + with pytest.raises(RuntimeError, match='sigil'): + subscriber.receive_messages_with_flow_control( + PROJECT, SUBSCRIPTION) + + out, _ = capsys.readouterr() + assert 'Message 1' in out + assert 'Message 2' in out + assert 'Message 3' in out From 8e689f722614036b94d4a27e8944e0a544ff9dec Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Fri, 31 Aug 2018 14:49:04 -0700 Subject: [PATCH 059/101] Updated variable name [(#1680)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1680) --- samples/snippets/subscriber.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 83e5700dd..38f1e527f 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -30,10 +30,10 @@ def list_subscriptions_in_topic(project, topic_name): """Lists all subscriptions for a given topic.""" # [START pubsub_list_topic_subscriptions] - subscriber = pubsub_v1.PublisherClient() - topic_path = subscriber.topic_path(project, topic_name) + publisher = pubsub_v1.PublisherClient() + topic_path = publisher.topic_path(project, topic_name) - for subscription in subscriber.list_topic_subscriptions(topic_path): + for subscription in publisher.list_topic_subscriptions(topic_path): print(subscription) # [END pubsub_list_topic_subscriptions] From 319b370bd9cf854ea473b5feba5df58c3baa9fe0 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Thu, 6 Sep 2018 17:28:27 -0700 Subject: [PATCH 060/101] Fixed return object from `subscriber.subscribe()` [(#1685)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1685) --- samples/snippets/subscriber.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 38f1e527f..e31560722 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -255,13 +255,13 @@ def callback(message): print('Received message: {}'.format(message)) message.ack() - subscription = subscriber.subscribe(subscription_path, callback=callback) + future = subscriber.subscribe(subscription_path, callback=callback) # Blocks the thread while messages are coming in through the stream. Any # exceptions that crop up on the thread will be set on the future. try: # When timeout is unspecified, the result method waits indefinitely. - subscription.future.result(timeout=30) + future.result(timeout=30) except Exception as e: print( 'Listening for messages on {} threw an Exception: {}.'.format( From a88c9f0d76139903dcf2cbc4571b75352b6d6f69 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Fri, 14 Sep 2018 16:11:33 -0700 Subject: [PATCH 061/101] Pub/Sub: synchronous pull with lease management [(#1701)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1701) * Synchronous pull with lease management * Updated library version --- samples/snippets/requirements.txt | 2 +- samples/snippets/subscriber.py | 95 +++++++++++++++++++++++++---- samples/snippets/subscriber_test.py | 68 ++++++++++++++++----- 3 files changed, 137 insertions(+), 28 deletions(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 936a9f0ed..81a62427c 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.37.2 +google-cloud-pubsub==0.38.0 diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index e31560722..3915a82ef 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -23,6 +23,9 @@ import argparse import time +import logging +import random +import multiprocessing from google.cloud import pubsub_v1 @@ -215,7 +218,7 @@ def callback(message): # [END pubsub_subscriber_flow_settings] -def receive_messages_synchronously(project, subscription_name): +def synchronous_pull(project, subscription_name): """Pulling messages synchronously.""" # [START pubsub_subscriber_sync_pull] # project = "Your Google Cloud Project ID" @@ -224,13 +227,10 @@ def receive_messages_synchronously(project, subscription_name): subscription_path = subscriber.subscription_path( project, subscription_name) - # Builds a pull request with a specific number of messages to return. - # `return_immediately` is set to False so that the system waits (for a - # bounded amount of time) until at lease one message is available. - response = subscriber.pull( - subscription_path, - max_messages=3, - return_immediately=False) + NUM_MESSAGES=3 + + # The subscriber pulls a specific number of messages. + response = subscriber.pull(subscription_path, max_messages=NUM_MESSAGES) ack_ids = [] for received_message in response.received_messages: @@ -239,9 +239,72 @@ def receive_messages_synchronously(project, subscription_name): # Acknowledges the received messages so they will not be sent again. subscriber.acknowledge(subscription_path, ack_ids) + + print("Received and acknowledged {} messages. Done.".format(NUM_MESSAGES)) # [END pubsub_subscriber_sync_pull] +def synchronous_pull_with_lease_management(project, subscription_name): + """Pulling messages synchronously with lease management""" + # [START pubsub_subscriber_sync_pull_with_lease] + # project = "Your Google Cloud Project ID" + # subscription_name = "Your Pubsub subscription name" + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path( + project, subscription_name) + + NUM_MESSAGES=2 + ACK_DEADLINE=30 + SLEEP_TIME=10 + + # The subscriber pulls a specific number of messages. + response = subscriber.pull(subscription_path, max_messages=NUM_MESSAGES) + + multiprocessing.log_to_stderr() + logger = multiprocessing.get_logger() + logger.setLevel(logging.INFO) + + def worker(msg): + """Simulates a long-running process.""" + RUN_TIME = random.randint(1,60) + logger.info('{}: Running {} for {}s'.format( + time.strftime("%X", time.gmtime()), msg.message.data, RUN_TIME)) + time.sleep(RUN_TIME) + + # `processes` stores process as key and ack id and message as values. + processes = dict() + for message in response.received_messages: + process = multiprocessing.Process(target=worker, args=(message,)) + processes[process] = (message.ack_id, message.message.data) + process.start() + + while processes: + for process, (ack_id, msg_data) in processes.items(): + # If the process is still running, reset the ack deadline as + # specified by ACK_DEADLINE once every while as specified + # by SLEEP_TIME. + if process.is_alive(): + # `ack_deadline_seconds` must be between 10 to 600. + subscriber.modify_ack_deadline(subscription_path, + [ack_id], ack_deadline_seconds=ACK_DEADLINE) + logger.info('{}: Reset ack deadline for {} for {}s'.format( + time.strftime("%X", time.gmtime()), msg_data, ACK_DEADLINE)) + + # If the processs is finished, acknowledges using `ack_id`. + else: + subscriber.acknowledge(subscription_path, [ack_id]) + logger.info("{}: Acknowledged {}".format( + time.strftime("%X", time.gmtime()), msg_data)) + processes.pop(process) + + # If there are still processes running, sleeps the thread. + if processes: + time.sleep(SLEEP_TIME) + + print("Received and acknowledged {} messages. Done.".format(NUM_MESSAGES)) + # [END pubsub_subscriber_sync_pull_with_lease] + + def listen_for_errors(project, subscription_name): """Receives messages and catches errors from a pull subscription.""" # [START pubsub_subscriber_error_listener] @@ -318,10 +381,15 @@ def callback(message): help=receive_messages_with_flow_control.__doc__) receive_with_flow_control_parser.add_argument('subscription_name') - receive_messages_synchronously_parser = subparsers.add_parser( + synchronous_pull_parser = subparsers.add_parser( 'receive-synchronously', - help=receive_messages_synchronously.__doc__) - receive_messages_synchronously_parser.add_argument('subscription_name') + help=synchronous_pull.__doc__) + synchronous_pull_parser.add_argument('subscription_name') + + synchronous_pull_with_lease_management_parser = subparsers.add_parser( + 'receive-synchronously-with-lease', + help=synchronous_pull_with_lease_management.__doc__) + synchronous_pull_with_lease_management_parser.add_argument('subscription_name') listen_for_errors_parser = subparsers.add_parser( 'listen_for_errors', help=listen_for_errors.__doc__) @@ -357,7 +425,10 @@ def callback(message): receive_messages_with_flow_control( args.project, args.subscription_name) elif args.command == 'receive-synchronously': - receive_messages_synchronously( + synchronous_pull( + args.project, args.subscription_name) + elif args.command == 'receive-synchronously-with-lease': + synchronous_pull_with_lease_management( args.project, args.subscription_name) elif args.command == 'listen_for_errors': listen_for_errors(args.project, args.subscription_name) diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index 728f971f2..9f554398e 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -25,6 +25,8 @@ PROJECT = os.environ['GCLOUD_PROJECT'] TOPIC = 'subscription-test-topic' SUBSCRIPTION = 'subscription-test-subscription' +SUBSCRIPTION_SYNC1 = 'subscription-test-subscription-sync1' +SUBSCRIPTION_SYNC2 = 'subscription-test-subscription-sync2' ENDPOINT = 'https://{}.appspot.com/push'.format(PROJECT) @@ -67,6 +69,36 @@ def subscription(subscriber_client, topic): yield subscription_path +@pytest.fixture +def subscription_sync1(subscriber_client, topic): + subscription_sync_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION_SYNC1) + + try: + subscriber_client.delete_subscription(subscription_sync_path) + except Exception: + pass + + subscriber_client.create_subscription(subscription_sync_path, topic=topic) + + yield subscription_sync_path + + +@pytest.fixture +def subscription_sync2(subscriber_client, topic): + subscription_sync_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION_SYNC2) + + try: + subscriber_client.delete_subscription(subscription_sync_path) + except Exception: + pass + + subscriber_client.create_subscription(subscription_sync_path, topic=topic) + + yield subscription_sync_path + + def test_list_in_topic(subscription, capsys): @eventually_consistent.call def _(): @@ -160,6 +192,27 @@ def test_receive(publisher_client, topic, subscription, capsys): assert 'Message 1' in out +def test_receive_synchronously( + publisher_client, topic, subscription_sync1, capsys): + _publish_messages(publisher_client, topic) + + subscriber.synchronous_pull(PROJECT, SUBSCRIPTION_SYNC1) + + out, _ = capsys.readouterr() + assert 'Done.' in out + + +def test_receive_synchronously_with_lease( + publisher_client, topic, subscription_sync2, capsys): + _publish_messages(publisher_client, topic) + + subscriber.synchronous_pull_with_lease_management( + PROJECT, SUBSCRIPTION_SYNC2) + + out, _ = capsys.readouterr() + assert 'Done.' in out + + def test_receive_with_custom_attributes( publisher_client, topic, subscription, capsys): _publish_messages_with_custom_attributes(publisher_client, topic) @@ -188,18 +241,3 @@ def test_receive_with_flow_control( assert 'Listening' in out assert subscription in out assert 'Message 1' in out - - -def test_receive_synchronously( - publisher_client, topic, subscription, capsys): - _publish_messages(publisher_client, topic) - - with _make_sleep_patch(): - with pytest.raises(RuntimeError, match='sigil'): - subscriber.receive_messages_with_flow_control( - PROJECT, SUBSCRIPTION) - - out, _ = capsys.readouterr() - assert 'Message 1' in out - assert 'Message 2' in out - assert 'Message 3' in out From d28750bf9947de008ab4ee0851afb8bf300138b7 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Fri, 12 Oct 2018 10:39:44 -0700 Subject: [PATCH 062/101] Pub/Sub: moved import statements inside region tags [(#1753)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1753) * Moved import stataments inside region tags * Explained topic and subscription path methods --- samples/snippets/publisher.py | 105 ++++++++++++----- samples/snippets/subscriber.py | 206 +++++++++++++++++++++------------ 2 files changed, 207 insertions(+), 104 deletions(-) diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index a577abc63..b7e574e04 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -22,27 +22,33 @@ """ import argparse -import time -from google.cloud import pubsub_v1 - -def list_topics(project): +def list_topics(project_id): """Lists all Pub/Sub topics in the given project.""" # [START pubsub_list_topics] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + publisher = pubsub_v1.PublisherClient() - project_path = publisher.project_path(project) + project_path = publisher.project_path(project_id) for topic in publisher.list_topics(project_path): print(topic) # [END pubsub_list_topics] -def create_topic(project, topic_name): +def create_topic(project_id, topic_name): """Create a new Pub/Sub topic.""" # [START pubsub_create_topic] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project, topic_name) + topic_path = publisher.topic_path(project_id, topic_name) topic = publisher.create_topic(topic_path) @@ -50,11 +56,16 @@ def create_topic(project, topic_name): # [END pubsub_create_topic] -def delete_topic(project, topic_name): +def delete_topic(project_id, topic_name): """Deletes an existing Pub/Sub topic.""" # [START pubsub_delete_topic] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project, topic_name) + topic_path = publisher.topic_path(project_id, topic_name) publisher.delete_topic(topic_path) @@ -62,30 +73,44 @@ def delete_topic(project, topic_name): # [END pubsub_delete_topic] -def publish_messages(project, topic_name): +def publish_messages(project_id, topic_name): """Publishes multiple messages to a Pub/Sub topic.""" # [START pubsub_quickstart_publisher] # [START pubsub_publish] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project, topic_name) + # The `topic_path` method creates a fully qualified identifier + # in the form `projects/{project_id}/topics/{topic_name}` + topic_path = publisher.topic_path(project_id, topic_name) for n in range(1, 10): data = u'Message number {}'.format(n) # Data must be a bytestring data = data.encode('utf-8') - publisher.publish(topic_path, data=data) + # When you publish a message, the client returns a future. + future = publisher.publish(topic_path, data=data) + print('Published {} of message ID {}.'.format(data, future.result())) print('Published messages.') # [END pubsub_quickstart_publisher] # [END pubsub_publish] -def publish_messages_with_custom_attributes(project, topic_name): +def publish_messages_with_custom_attributes(project_id, topic_name): """Publishes multiple messages with custom attributes to a Pub/Sub topic.""" # [START pubsub_publish_custom_attributes] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project, topic_name) + topic_path = publisher.topic_path(project_id, topic_name) for n in range(1, 10): data = u'Message number {}'.format(n) @@ -99,12 +124,17 @@ def publish_messages_with_custom_attributes(project, topic_name): # [END pubsub_publish_custom_attributes] -def publish_messages_with_futures(project, topic_name): +def publish_messages_with_futures(project_id, topic_name): """Publishes multiple messages to a Pub/Sub topic and prints their message IDs.""" # [START pubsub_publisher_concurrency_control] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project, topic_name) + topic_path = publisher.topic_path(project_id, topic_name) # When you publish a message, the client returns a Future. This Future # can be used to track when the message is published. @@ -124,11 +154,18 @@ def publish_messages_with_futures(project, topic_name): # [END pubsub_publisher_concurrency_control] -def publish_messages_with_error_handler(project, topic_name): +def publish_messages_with_error_handler(project_id, topic_name): """Publishes multiple messages to a Pub/Sub topic with an error handler.""" # [START pubsub_publish_messages_error_handler] + import time + + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project, topic_name) + topic_path = publisher.topic_path(project_id, topic_name) def callback(message_future): # When timeout is unspecified, the exception method waits indefinitely. @@ -155,17 +192,22 @@ def callback(message_future): # [END pubsub_publish_messages_error_handler] -def publish_messages_with_batch_settings(project, topic_name): +def publish_messages_with_batch_settings(project_id, topic_name): """Publishes multiple messages to a Pub/Sub topic with batch settings.""" # [START pubsub_publisher_batch_settings] - # Configure the batch to publish once there is one kilobyte of data or - # 1 second has passed. + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + + # Configure the batch to publish as soon as there is one kilobyte + # of data or one second has passed. batch_settings = pubsub_v1.types.BatchSettings( max_bytes=1024, # One kilobyte max_latency=1, # One second ) publisher = pubsub_v1.PublisherClient(batch_settings) - topic_path = publisher.topic_path(project, topic_name) + topic_path = publisher.topic_path(project_id, topic_name) for n in range(1, 10): data = u'Message number {}'.format(n) @@ -182,7 +224,7 @@ def publish_messages_with_batch_settings(project, topic_name): description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) - parser.add_argument('project', help='Your Google Cloud project ID') + parser.add_argument('project_id', help='Your Google Cloud project ID') subparsers = parser.add_subparsers(dest='command') subparsers.add_parser('list', help=list_topics.__doc__) @@ -220,18 +262,19 @@ def publish_messages_with_batch_settings(project, topic_name): args = parser.parse_args() if args.command == 'list': - list_topics(args.project) + list_topics(args.project_id) elif args.command == 'create': - create_topic(args.project, args.topic_name) + create_topic(args.project_id, args.topic_name) elif args.command == 'delete': - delete_topic(args.project, args.topic_name) + delete_topic(args.project_id, args.topic_name) elif args.command == 'publish': - publish_messages(args.project, args.topic_name) + publish_messages(args.project_id, args.topic_name) elif args.command == 'publish-with-custom-attributes': - publish_messages_with_custom_attributes(args.project, args.topic_name) + publish_messages_with_custom_attributes( + args.project_id, args.topic_name) elif args.command == 'publish-with-futures': - publish_messages_with_futures(args.project, args.topic_name) + publish_messages_with_futures(args.project_id, args.topic_name) elif args.command == 'publish-with-error-handler': - publish_messages_with_error_handler(args.project, args.topic_name) + publish_messages_with_error_handler(args.project_id, args.topic_name) elif args.command == 'publish-with-batch-settings': - publish_messages_with_batch_settings(args.project, args.topic_name) + publish_messages_with_batch_settings(args.project_id, args.topic_name) diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 3915a82ef..5802218b4 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -22,43 +22,52 @@ """ import argparse -import time -import logging -import random -import multiprocessing -from google.cloud import pubsub_v1 - -def list_subscriptions_in_topic(project, topic_name): +def list_subscriptions_in_topic(project_id, topic_name): """Lists all subscriptions for a given topic.""" # [START pubsub_list_topic_subscriptions] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project, topic_name) + topic_path = publisher.topic_path(project_id, topic_name) for subscription in publisher.list_topic_subscriptions(topic_path): print(subscription) # [END pubsub_list_topic_subscriptions] -def list_subscriptions_in_project(project): +def list_subscriptions_in_project(project_id): """Lists all subscriptions in the current project.""" # [START pubsub_list_subscriptions] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + subscriber = pubsub_v1.SubscriberClient() - project_path = subscriber.project_path(project) + project_path = subscriber.project_path(project_id) for subscription in subscriber.list_subscriptions(project_path): print(subscription.name) # [END pubsub_list_subscriptions] -def create_subscription(project, topic_name, subscription_name): +def create_subscription(project_id, topic_name, subscription_name): """Create a new pull subscription on the given topic.""" # [START pubsub_create_pull_subscription] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + # TODO subscription_name = "Your Pub/Sub subscription name" + subscriber = pubsub_v1.SubscriberClient() - topic_path = subscriber.topic_path(project, topic_name) + topic_path = subscriber.topic_path(project_id, topic_name) subscription_path = subscriber.subscription_path( - project, subscription_name) + project_id, subscription_name) subscription = subscriber.create_subscription( subscription_path, topic_path) @@ -67,17 +76,23 @@ def create_subscription(project, topic_name, subscription_name): # [END pubsub_create_pull_subscription] -def create_push_subscription(project, +def create_push_subscription(project_id, topic_name, subscription_name, endpoint): """Create a new push subscription on the given topic.""" # [START pubsub_create_push_subscription] - # endpoint = "https://my-test-project.appspot.com/push" + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + # TODO subscription_name = "Your Pub/Sub subscription name" + # TODO endpoint = "https://my-test-project.appspot.com/push" + subscriber = pubsub_v1.SubscriberClient() - topic_path = subscriber.topic_path(project, topic_name) + topic_path = subscriber.topic_path(project_id, topic_name) subscription_path = subscriber.subscription_path( - project, subscription_name) + project_id, subscription_name) push_config = pubsub_v1.types.PushConfig( push_endpoint=endpoint) @@ -90,14 +105,17 @@ def create_push_subscription(project, # [END pubsub_create_push_subscription] -def delete_subscription(project, subscription_name): +def delete_subscription(project_id, subscription_name): """Deletes an existing Pub/Sub topic.""" # [START pubsub_delete_subscription] - # project = "Your Google Cloud Project ID" - # subscription_name = "Your Pubsub subscription name" + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO subscription_name = "Your Pub/Sub subscription name" + subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( - project, subscription_name) + project_id, subscription_name) subscriber.delete_subscription(subscription_path) @@ -105,17 +123,23 @@ def delete_subscription(project, subscription_name): # [END pubsub_delete_subscription] -def update_subscription(project, subscription_name, endpoint): +def update_subscription(project_id, subscription_name, endpoint): """ Updates an existing Pub/Sub subscription's push endpoint URL. Note that certain properties of a subscription, such as its topic, are not modifiable. """ # [START pubsub_update_push_configuration] - # endpoint = "https://my-test-project.appspot.com/push" + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + # TODO subscription_name = "Your Pub/Sub subscription name" + # TODO endpoint = "https://my-test-project.appspot.com/push" + subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( - project, subscription_name) + project_id, subscription_name) push_config = pubsub_v1.types.PushConfig( push_endpoint=endpoint) @@ -139,15 +163,22 @@ def update_subscription(project, subscription_name, endpoint): # [END pubsub_update_push_configuration] -def receive_messages(project, subscription_name): +def receive_messages(project_id, subscription_name): """Receives messages from a pull subscription.""" # [START pubsub_subscriber_async_pull] # [START pubsub_quickstart_subscriber] - # project = "Your Google Cloud Project ID" - # subscription_name = "Your Pubsub subscription name" + import time + + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO subscription_name = "Your Pub/Sub subscription name" + subscriber = pubsub_v1.SubscriberClient() + # The `subscription_path` method creates a fully qualified identifier + # in the form `projects/{project_id}/subscriptions/{subscription_name}` subscription_path = subscriber.subscription_path( - project, subscription_name) + project_id, subscription_name) def callback(message): print('Received message: {}'.format(message)) @@ -155,8 +186,8 @@ def callback(message): subscriber.subscribe(subscription_path, callback=callback) - # The subscriber is non-blocking, so we must keep the main thread from - # exiting to allow it to process messages in the background. + # The subscriber is non-blocking. We must keep the main thread from + # exiting to allow it to process messages asynchronously in the background. print('Listening for messages on {}'.format(subscription_path)) while True: time.sleep(60) @@ -164,14 +195,19 @@ def callback(message): # [END pubsub_quickstart_subscriber] -def receive_messages_with_custom_attributes(project, subscription_name): +def receive_messages_with_custom_attributes(project_id, subscription_name): """Receives messages from a pull subscription.""" # [START pubsub_subscriber_sync_pull_custom_attributes] - # project = "Your Google Cloud Project ID" - # subscription_name = "Your Pubsub subscription name" + import time + + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO subscription_name = "Your Pub/Sub subscription name" + subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( - project, subscription_name) + project_id, subscription_name) def callback(message): print('Received message: {}'.format(message.data)) @@ -192,14 +228,19 @@ def callback(message): # [END pubsub_subscriber_sync_pull_custom_attributes] -def receive_messages_with_flow_control(project, subscription_name): +def receive_messages_with_flow_control(project_id, subscription_name): """Receives messages from a pull subscription with flow control.""" # [START pubsub_subscriber_flow_settings] - # project = "Your Google Cloud Project ID" - # subscription_name = "Your Pubsub subscription name" + import time + + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO subscription_name = "Your Pub/Sub subscription name" + subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( - project, subscription_name) + project_id, subscription_name) def callback(message): print('Received message: {}'.format(message.data)) @@ -218,16 +259,19 @@ def callback(message): # [END pubsub_subscriber_flow_settings] -def synchronous_pull(project, subscription_name): +def synchronous_pull(project_id, subscription_name): """Pulling messages synchronously.""" # [START pubsub_subscriber_sync_pull] - # project = "Your Google Cloud Project ID" - # subscription_name = "Your Pubsub subscription name" + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO subscription_name = "Your Pub/Sub subscription name" + subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( - project, subscription_name) + project_id, subscription_name) - NUM_MESSAGES=3 + NUM_MESSAGES = 3 # The subscriber pulls a specific number of messages. response = subscriber.pull(subscription_path, max_messages=NUM_MESSAGES) @@ -244,18 +288,26 @@ def synchronous_pull(project, subscription_name): # [END pubsub_subscriber_sync_pull] -def synchronous_pull_with_lease_management(project, subscription_name): +def synchronous_pull_with_lease_management(project_id, subscription_name): """Pulling messages synchronously with lease management""" # [START pubsub_subscriber_sync_pull_with_lease] - # project = "Your Google Cloud Project ID" - # subscription_name = "Your Pubsub subscription name" + import logging + import multiprocessing + import random + import time + + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO subscription_name = "Your Pub/Sub subscription name" + subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( - project, subscription_name) + project_id, subscription_name) - NUM_MESSAGES=2 - ACK_DEADLINE=30 - SLEEP_TIME=10 + NUM_MESSAGES = 2 + ACK_DEADLINE = 30 + SLEEP_TIME = 10 # The subscriber pulls a specific number of messages. response = subscriber.pull(subscription_path, max_messages=NUM_MESSAGES) @@ -266,7 +318,7 @@ def synchronous_pull_with_lease_management(project, subscription_name): def worker(msg): """Simulates a long-running process.""" - RUN_TIME = random.randint(1,60) + RUN_TIME = random.randint(1, 60) logger.info('{}: Running {} for {}s'.format( time.strftime("%X", time.gmtime()), msg.message.data, RUN_TIME)) time.sleep(RUN_TIME) @@ -279,16 +331,20 @@ def worker(msg): process.start() while processes: - for process, (ack_id, msg_data) in processes.items(): + for process in list(processes): + ack_id, msg_data = processes[process] # If the process is still running, reset the ack deadline as # specified by ACK_DEADLINE once every while as specified # by SLEEP_TIME. if process.is_alive(): # `ack_deadline_seconds` must be between 10 to 600. - subscriber.modify_ack_deadline(subscription_path, - [ack_id], ack_deadline_seconds=ACK_DEADLINE) + subscriber.modify_ack_deadline( + subscription_path, + [ack_id], + ack_deadline_seconds=ACK_DEADLINE) logger.info('{}: Reset ack deadline for {} for {}s'.format( - time.strftime("%X", time.gmtime()), msg_data, ACK_DEADLINE)) + time.strftime("%X", time.gmtime()), + msg_data, ACK_DEADLINE)) # If the processs is finished, acknowledges using `ack_id`. else: @@ -305,14 +361,17 @@ def worker(msg): # [END pubsub_subscriber_sync_pull_with_lease] -def listen_for_errors(project, subscription_name): +def listen_for_errors(project_id, subscription_name): """Receives messages and catches errors from a pull subscription.""" # [START pubsub_subscriber_error_listener] - # project = "Your Google Cloud Project ID" - # subscription_name = "Your Pubsub subscription name" + from google.cloud import pubsub_v1 + + # TODO project = "Your Google Cloud Project ID" + # TODO subscription_name = "Your Pubsub subscription name" + subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( - project, subscription_name) + project_id, subscription_name) def callback(message): print('Received message: {}'.format(message)) @@ -337,7 +396,7 @@ def callback(message): description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) - parser.add_argument('project', help='Your Google Cloud project ID') + parser.add_argument('project_id', help='Your Google Cloud project ID') subparsers = parser.add_subparsers(dest='command') list_in_topic_parser = subparsers.add_parser( @@ -389,7 +448,8 @@ def callback(message): synchronous_pull_with_lease_management_parser = subparsers.add_parser( 'receive-synchronously-with-lease', help=synchronous_pull_with_lease_management.__doc__) - synchronous_pull_with_lease_management_parser.add_argument('subscription_name') + synchronous_pull_with_lease_management_parser.add_argument( + 'subscription_name') listen_for_errors_parser = subparsers.add_parser( 'listen_for_errors', help=listen_for_errors.__doc__) @@ -398,37 +458,37 @@ def callback(message): args = parser.parse_args() if args.command == 'list_in_topic': - list_subscriptions_in_topic(args.project, args.topic_name) + list_subscriptions_in_topic(args.project_id, args.topic_name) elif args.command == 'list_in_project': - list_subscriptions_in_project(args.project) + list_subscriptions_in_project(args.project_id) elif args.command == 'create': create_subscription( - args.project, args.topic_name, args.subscription_name) + args.project_id, args.topic_name, args.subscription_name) elif args.command == 'create-push': create_push_subscription( - args.project, + args.project_id, args.topic_name, args.subscription_name, args.endpoint) elif args.command == 'delete': delete_subscription( - args.project, args.subscription_name) + args.project_id, args.subscription_name) elif args.command == 'update': update_subscription( - args.project, args.subscription_name, args.endpoint) + args.project_id, args.subscription_name, args.endpoint) elif args.command == 'receive': - receive_messages(args.project, args.subscription_name) + receive_messages(args.project_id, args.subscription_name) elif args.command == 'receive-custom-attributes': receive_messages_with_custom_attributes( - args.project, args.subscription_name) + args.project_id, args.subscription_name) elif args.command == 'receive-flow-control': receive_messages_with_flow_control( - args.project, args.subscription_name) + args.project_id, args.subscription_name) elif args.command == 'receive-synchronously': synchronous_pull( - args.project, args.subscription_name) + args.project_id, args.subscription_name) elif args.command == 'receive-synchronously-with-lease': synchronous_pull_with_lease_management( - args.project, args.subscription_name) + args.project_id, args.subscription_name) elif args.command == 'listen_for_errors': - listen_for_errors(args.project, args.subscription_name) + listen_for_errors(args.project_id, args.subscription_name) From bb3e1354a46079c86708b63ea72eef6ea2317994 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Mon, 29 Oct 2018 17:24:09 -0700 Subject: [PATCH 063/101] Pub/Sub end-to-end sample [(#1800)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1800) * Created new end-to-end sample, moved old sample * Add space around operator --- samples/snippets/publisher.py | 2 + samples/snippets/quickstart.py | 91 +++++++++++++++++++++++++---- samples/snippets/quickstart_test.py | 58 ++++++++++++------ 3 files changed, 123 insertions(+), 28 deletions(-) diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index b7e574e04..fcb0d9b0f 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -41,6 +41,7 @@ def list_topics(project_id): def create_topic(project_id, topic_name): """Create a new Pub/Sub topic.""" + # [START pubsub_quickstart_create_topic] # [START pubsub_create_topic] from google.cloud import pubsub_v1 @@ -53,6 +54,7 @@ def create_topic(project_id, topic_name): topic = publisher.create_topic(topic_path) print('Topic created: {}'.format(topic)) + # [END pubsub_quickstart_create_topic] # [END pubsub_create_topic] diff --git a/samples/snippets/quickstart.py b/samples/snippets/quickstart.py index 1ff2efed3..f48d085e0 100644 --- a/samples/snippets/quickstart.py +++ b/samples/snippets/quickstart.py @@ -15,25 +15,94 @@ # limitations under the License. -def run_quickstart(): - # [START pubsub_quickstart_create_topic] - # Imports the Google Cloud client library +import argparse + + +def end_to_end(project_id, topic_name, subscription_name, num_messages): + # [START pubsub_end_to_end] + import time + from google.cloud import pubsub_v1 - # Instantiates a client + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + # TODO num_messages = number of messages to test end-to-end + + # Instantiates a publisher and subscriber client publisher = pubsub_v1.PublisherClient() + subscriber = pubsub_v1.SubscriberClient() + + # The `topic_path` method creates a fully qualified identifier + # in the form `projects/{project_id}/topics/{topic_name}` + topic_path = subscriber.topic_path(project_id, topic_name) - # The resource path for the new topic contains the project ID - # and the topic name. - topic_path = publisher.topic_path( - 'my-project', 'my-new-topic') + # The `subscription_path` method creates a fully qualified identifier + # in the form `projects/{project_id}/subscriptions/{subscription_name}` + subscription_path = subscriber.subscription_path( + project_id, subscription_name) # Create the topic. topic = publisher.create_topic(topic_path) + print('\nTopic created: {}'.format(topic.name)) + + # Create a subscription. + subscription = subscriber.create_subscription( + subscription_path, topic_path) + print('\nSubscription created: {}\n'.format(subscription.name)) + + publish_begin = time.time() + + # Publish messages. + for n in range(num_messages): + data = u'Message number {}'.format(n) + # Data must be a bytestring + data = data.encode('utf-8') + # When you publish a message, the client returns a future. + future = publisher.publish(topic_path, data=data) + print('Published {} of message ID {}.'.format(data, future.result())) + + publish_time = time.time() - publish_begin - print('Topic created: {}'.format(topic)) - # [END pubsub_quickstart_create_topic] + messages = set() + + def callback(message): + print('Received message: {}'.format(message)) + # Unacknowledged messages will be sent again. + message.ack() + messages.add(message) + + subscribe_begin = time.time() + + # Receive messages. The subscriber is nonblocking. + subscriber.subscribe(subscription_path, callback=callback) + + print('\nListening for messages on {}...\n'.format(subscription_path)) + + while True: + if len(messages) == num_messages: + subscribe_time = time.time() - subscribe_begin + print("\nReceived all messages.") + print("Publish time lapsed: {:.2f}s.".format(publish_time)) + print("Subscribe time lapsed: {:.2f}s.".format(subscribe_time)) + break + else: + # Sleeps the thread at 50Hz to save on resources. + time.sleep(1. / 50) + # [END pubsub_end_to_end] if __name__ == '__main__': - run_quickstart() + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('project_id', help='Your Google Cloud project ID') + parser.add_argument('topic_name', help='Your topic name') + parser.add_argument('subscription_name', help='Your subscription name') + parser.add_argument('num_msgs', type=int, help='Number of test messages') + + args = parser.parse_args() + + end_to_end(args.project_id, args.topic_name, args.subscription_name, + args.num_msgs) diff --git a/samples/snippets/quickstart_test.py b/samples/snippets/quickstart_test.py index 520213bcf..ee6f7d4b2 100644 --- a/samples/snippets/quickstart_test.py +++ b/samples/snippets/quickstart_test.py @@ -1,4 +1,6 @@ -# Copyright 2016 Google Inc. All Rights Reserved. +#!/usr/bin/env python + +# Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,33 +17,55 @@ import os from google.cloud import pubsub_v1 -import mock import pytest - import quickstart PROJECT = os.environ['GCLOUD_PROJECT'] -# Must match the dataset listed in quickstart.py -TOPIC_NAME = 'my-new-topic' -TOPIC_PATH = 'projects/{}/topics/{}'.format(PROJECT, TOPIC_NAME) +TOPIC = 'end-to-end-test-topic' +SUBSCRIPTION = 'end-to-end-test-topic-sub' +N = 10 + + +@pytest.fixture(scope='module') +def publisher_client(): + yield pubsub_v1.PublisherClient() -@pytest.fixture -def temporary_topic(): - """Fixture that ensures the test topic does not exist before the test.""" - publisher = pubsub_v1.PublisherClient() +@pytest.fixture(scope='module') +def topic(publisher_client): + topic_path = publisher_client.topic_path(PROJECT, TOPIC) try: - publisher.delete_topic(TOPIC_PATH) + publisher_client.delete_topic(topic_path) except Exception: pass - yield + yield TOPIC -@mock.patch.object( - pubsub_v1.PublisherClient, 'topic_path', return_value=TOPIC_PATH) -def test_quickstart(unused_topic_path, temporary_topic, capsys): - quickstart.run_quickstart() +@pytest.fixture(scope='module') +def subscriber_client(): + yield pubsub_v1.SubscriberClient() + + +@pytest.fixture(scope='module') +def subscription(subscriber_client, topic): + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION) + + try: + subscriber_client.delete_subscription(subscription_path) + except Exception: + pass + + yield SUBSCRIPTION + + +def test_end_to_end(topic, subscription, capsys): + + quickstart.end_to_end(PROJECT, topic, subscription, N) out, _ = capsys.readouterr() - assert TOPIC_NAME in out + + assert "Received all messages" in out + assert "Publish time lapsed" in out + assert "Subscribe time lapsed" in out From 31d8762fbdf273501b91ed5b1227867318053b81 Mon Sep 17 00:00:00 2001 From: noerog <32459203+noerog@users.noreply.github.com> Date: Thu, 6 Dec 2018 20:58:32 -0500 Subject: [PATCH 064/101] Add test for updating a subscription. [(#1336)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1336) Tests for https://github.com/GoogleCloudPlatform/python-docs-samples/pull/1335. Using ack_deadline_seconds as the example. --- samples/snippets/subscriber_test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index 9f554398e..3f5de61de 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -154,6 +154,16 @@ def _(): subscriber_client.get_subscription(subscription) +def test_update(subscriber_client, subscription, capsys): + ACK_DEADLINE_SECONDS = 100 + + subscriber.update_subscription(PROJECT, SUBSCRIPTION, ACK_DEADLINE_SECONDS) + + out, _ = capsys.readouterr() + assert subscription in out + assert '100' in out + + def _publish_messages(publisher_client, topic): for n in range(5): data = u'Message {}'.format(n).encode('utf-8') From aa05874e67e3b52bbb9bc071b61beeb59c7c29e1 Mon Sep 17 00:00:00 2001 From: Charles Engelke Date: Wed, 12 Dec 2018 19:44:07 -0800 Subject: [PATCH 065/101] Fix update test to use new endpoint [(#1925)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1925) * Fix update test to use new endpoint * Handle subscription already exists Previous deletions don't always succeed * Use a new endpoint for update --- samples/snippets/subscriber_test.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index 3f5de61de..df5b1092b 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -17,6 +17,7 @@ from gcp_devrel.testing import eventually_consistent from google.cloud import pubsub_v1 +import google.api_core.exceptions import mock import pytest @@ -28,6 +29,7 @@ SUBSCRIPTION_SYNC1 = 'subscription-test-subscription-sync1' SUBSCRIPTION_SYNC2 = 'subscription-test-subscription-sync2' ENDPOINT = 'https://{}.appspot.com/push'.format(PROJECT) +NEW_ENDPOINT = 'https://{}.appspot.com/push2'.format(PROJECT) @pytest.fixture(scope='module') @@ -64,7 +66,10 @@ def subscription(subscriber_client, topic): except Exception: pass - subscriber_client.create_subscription(subscription_path, topic=topic) + try: + subscriber_client.create_subscription(subscription_path, topic=topic) + except google.api_core.exceptions.AlreadyExists: + pass yield subscription_path @@ -155,13 +160,10 @@ def _(): def test_update(subscriber_client, subscription, capsys): - ACK_DEADLINE_SECONDS = 100 - - subscriber.update_subscription(PROJECT, SUBSCRIPTION, ACK_DEADLINE_SECONDS) + subscriber.update_subscription(PROJECT, SUBSCRIPTION, NEW_ENDPOINT) out, _ = capsys.readouterr() - assert subscription in out - assert '100' in out + assert 'Subscription updated' in out def _publish_messages(publisher_client, topic): From f16c5ee2d722d5f896cbd387f142890b4b5b49e1 Mon Sep 17 00:00:00 2001 From: DPEBot Date: Wed, 6 Feb 2019 12:06:35 -0800 Subject: [PATCH 066/101] Auto-update dependencies. [(#1980)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1980) * Auto-update dependencies. * Update requirements.txt * Update requirements.txt --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 81a62427c..d8470ecf9 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.38.0 +google-cloud-pubsub==0.39.1 From aad1e18d6ccc68816252fad04b589b995c504540 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Tue, 12 Feb 2019 13:04:59 -0800 Subject: [PATCH 067/101] Cloud Pub/Sub Quickstart V2 [(#2004)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2004) * Quickstart V2 * Adopts Kir's suggestions * Adopted Tim's suggestions * proper resource deletion during teardown --- samples/snippets/quickstart/pub.py | 73 +++++++++++++++ samples/snippets/quickstart/pub_test.py | 61 +++++++++++++ samples/snippets/quickstart/sub.py | 64 ++++++++++++++ samples/snippets/quickstart/sub_test.py | 113 ++++++++++++++++++++++++ 4 files changed, 311 insertions(+) create mode 100644 samples/snippets/quickstart/pub.py create mode 100644 samples/snippets/quickstart/pub_test.py create mode 100644 samples/snippets/quickstart/sub.py create mode 100644 samples/snippets/quickstart/sub_test.py diff --git a/samples/snippets/quickstart/pub.py b/samples/snippets/quickstart/pub.py new file mode 100644 index 000000000..9617b34ea --- /dev/null +++ b/samples/snippets/quickstart/pub.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START pubsub_quickstart_pub_all] +import argparse +import time +# [START pubsub_quickstart_pub_deps] +from google.cloud import pubsub_v1 +# [END pubsub_quickstart_pub_deps] + + +def get_callback(api_future, data): + """Wrap message data in the context of the callback function.""" + + def callback(api_future): + try: + print("Published message {} now has message ID {}".format( + data, api_future.result())) + except Exception: + print("A problem occurred when publishing {}: {}\n".format( + data, api_future.exception())) + raise + return callback + + +def pub(project_id, topic_name): + """Publishes a message to a Pub/Sub topic.""" + # [START pubsub_quickstart_pub_client] + # Initialize a Publisher client + client = pubsub_v1.PublisherClient() + # [END pubsub_quickstart_pub_client] + # Create a fully qualified identifier in the form of + # `projects/{project_id}/topics/{topic_name}` + topic_path = client.topic_path(project_id, topic_name) + + # Data sent to Cloud Pub/Sub must be a bytestring + data = b"Hello, World!" + + # When you publish a message, the client returns a future. + api_future = client.publish(topic_path, data=data) + api_future.add_done_callback(get_callback(api_future, data)) + + # Keep the main thread from exiting until background message + # is processed. + while api_future.running(): + time.sleep(0.1) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('project_id', help='Google Cloud project ID') + parser.add_argument('topic_name', help='Pub/Sub topic name') + + args = parser.parse_args() + + pub(args.project_id, args.topic_name) +# [END pubsub_quickstart_pub_all] diff --git a/samples/snippets/quickstart/pub_test.py b/samples/snippets/quickstart/pub_test.py new file mode 100644 index 000000000..09443364a --- /dev/null +++ b/samples/snippets/quickstart/pub_test.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest + +from google.api_core.exceptions import AlreadyExists +from google.cloud import pubsub_v1 + +import pub + +PROJECT = os.environ['GCLOUD_PROJECT'] +TOPIC = 'quickstart-pub-test-topic' + + +@pytest.fixture(scope='module') +def publisher_client(): + yield pubsub_v1.PublisherClient() + + +@pytest.fixture(scope='module') +def topic(publisher_client): + topic_path = publisher_client.topic_path(PROJECT, TOPIC) + + try: + publisher_client.create_topic(topic_path) + except AlreadyExists: + pass + + yield TOPIC + + +@pytest.fixture +def to_delete(publisher_client): + doomed = [] + yield doomed + for item in doomed: + publisher_client.delete_topic(item) + + +def test_pub(publisher_client, topic, to_delete, capsys): + pub.pub(PROJECT, topic) + + to_delete.append('projects/{}/topics/{}'.format(PROJECT, TOPIC)) + + out, _ = capsys.readouterr() + + assert "Published message b'Hello, World!'" in out diff --git a/samples/snippets/quickstart/sub.py b/samples/snippets/quickstart/sub.py new file mode 100644 index 000000000..520803d70 --- /dev/null +++ b/samples/snippets/quickstart/sub.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START pubsub_quickstart_sub_all] +import argparse +import time +# [START pubsub_quickstart_sub_deps] +from google.cloud import pubsub_v1 +# [END pubsub_quickstart_sub_deps] + + +def sub(project_id, subscription_name): + """Receives messages from a Pub/Sub subscription.""" + # [START pubsub_quickstart_sub_client] + # Initialize a Subscriber client + client = pubsub_v1.SubscriberClient() + # [END pubsub_quickstart_sub_client] + # Create a fully qualified identifier in the form of + # `projects/{project_id}/subscriptions/{subscription_name}` + subscription_path = client.subscription_path( + project_id, subscription_name) + + def callback(message): + print('Received message {} of message ID {}'.format( + message, message.message_id)) + # Acknowledge the message. Unack'ed messages will be redelivered. + message.ack() + print('Acknowledged message of message ID {}\n'.format( + message.message_id)) + + client.subscribe(subscription_path, callback=callback) + print('Listening for messages on {}..\n'.format(subscription_path)) + + # Keep the main thread from exiting so the subscriber can + # process messages in the background. + while True: + time.sleep(60) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('project_id', help='Google Cloud project ID') + parser.add_argument('subscription_name', help='Pub/Sub subscription name') + + args = parser.parse_args() + + sub(args.project_id, args.subscription_name) +# [END pubsub_quickstart_sub_all] diff --git a/samples/snippets/quickstart/sub_test.py b/samples/snippets/quickstart/sub_test.py new file mode 100644 index 000000000..9c70384ed --- /dev/null +++ b/samples/snippets/quickstart/sub_test.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import os +import pytest +import time + +from google.api_core.exceptions import AlreadyExists +from google.cloud import pubsub_v1 + +import sub + + +PROJECT = os.environ['GCLOUD_PROJECT'] +TOPIC = 'quickstart-sub-test-topic' +SUBSCRIPTION = 'quickstart-sub-test-topic-sub' + + +@pytest.fixture(scope='module') +def publisher_client(): + yield pubsub_v1.PublisherClient() + + +@pytest.fixture(scope='module') +def topic_path(publisher_client): + topic_path = publisher_client.topic_path(PROJECT, TOPIC) + + try: + publisher_client.create_topic(topic_path) + except AlreadyExists: + pass + + yield topic_path + + +@pytest.fixture(scope='module') +def subscriber_client(): + yield pubsub_v1.SubscriberClient() + + +@pytest.fixture(scope='module') +def subscription(subscriber_client, topic_path): + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION) + + try: + subscriber_client.create_subscription(subscription_path, topic_path) + except AlreadyExists: + pass + + yield SUBSCRIPTION + + +@pytest.fixture +def to_delete(publisher_client, subscriber_client): + doomed = [] + yield doomed + for client, item in doomed: + if 'topics' in item: + publisher_client.delete_topic(item) + if 'subscriptions' in item: + subscriber_client.delete_subscription(item) + + +def _make_sleep_patch(): + real_sleep = time.sleep + + def new_sleep(period): + if period == 60: + real_sleep(10) + raise RuntimeError('sigil') + else: + real_sleep(period) + + return mock.patch('time.sleep', new=new_sleep) + + +def test_sub(publisher_client, + topic_path, + subscriber_client, + subscription, + to_delete, + capsys): + + publisher_client.publish(topic_path, data=b'Hello, World!') + + to_delete.append((publisher_client, topic_path)) + + with _make_sleep_patch(): + with pytest.raises(RuntimeError, match='sigil'): + sub.sub(PROJECT, subscription) + + to_delete.append((subscriber_client, + 'projects/{}/subscriptions/{}'.format(PROJECT, + SUBSCRIPTION))) + + out, _ = capsys.readouterr() + assert "Received message" in out + assert "Acknowledged message" in out From 4650d3aa97d7e1ce9937fba860e06c05582078a8 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Fri, 14 Jun 2019 14:54:22 -0700 Subject: [PATCH 068/101] Pub/Sub: publish with error-handling comments [(#2222)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2222) --- samples/snippets/publisher.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index fcb0d9b0f..d7e51c2d8 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright 2016 Google Inc. All Rights Reserved. +# Copyright 2019 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -157,8 +157,8 @@ def publish_messages_with_futures(project_id, topic_name): def publish_messages_with_error_handler(project_id, topic_name): - """Publishes multiple messages to a Pub/Sub topic with an error handler.""" # [START pubsub_publish_messages_error_handler] + """Publishes multiple messages to a Pub/Sub topic with an error handler.""" import time from google.cloud import pubsub_v1 @@ -170,10 +170,8 @@ def publish_messages_with_error_handler(project_id, topic_name): topic_path = publisher.topic_path(project_id, topic_name) def callback(message_future): - # When timeout is unspecified, the exception method waits indefinitely. - if message_future.exception(timeout=30): - print('Publishing message on {} threw an Exception {}.'.format( - topic_name, message_future.exception())) + if message_future.exception(): + print('{} needs handling.'.format(message_future.exception())) else: print(message_future.result()) @@ -183,12 +181,14 @@ def callback(message_future): data = data.encode('utf-8') # When you publish a message, the client returns a Future. message_future = publisher.publish(topic_path, data=data) + # If you wish to handle publish failures, do it in the callback. + # Otherwise, it's okay to call `message_future.result()` directly. message_future.add_done_callback(callback) print('Published message IDs:') - # We must keep the main thread from exiting to allow it to process - # messages in the background. + # We keep the main thread from exiting so message futures can be + # resolved in the background. while True: time.sleep(60) # [END pubsub_publish_messages_error_handler] From 51439fe5b7ccf101dc0a6405e48dbdcc0952ef14 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Thu, 20 Jun 2019 16:57:11 -0700 Subject: [PATCH 069/101] Resolve all futures [(#2231)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2231) --- samples/snippets/publisher.py | 72 +++++++++++++++-------------- samples/snippets/publisher_test.py | 6 +-- samples/snippets/subscriber_test.py | 6 ++- 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index d7e51c2d8..51edd7bd8 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -95,7 +95,7 @@ def publish_messages(project_id, topic_name): data = data.encode('utf-8') # When you publish a message, the client returns a future. future = publisher.publish(topic_path, data=data) - print('Published {} of message ID {}.'.format(data, future.result())) + print(future.result()) print('Published messages.') # [END pubsub_quickstart_publisher] @@ -119,8 +119,9 @@ def publish_messages_with_custom_attributes(project_id, topic_name): # Data must be a bytestring data = data.encode('utf-8') # Add two attributes, origin and username, to the message - publisher.publish( + future = publisher.publish( topic_path, data, origin='python-sample', username='gcp') + print(future.result()) print('Published messages with custom attributes.') # [END pubsub_publish_custom_attributes] @@ -138,21 +139,15 @@ def publish_messages_with_futures(project_id, topic_name): publisher = pubsub_v1.PublisherClient() topic_path = publisher.topic_path(project_id, topic_name) - # When you publish a message, the client returns a Future. This Future - # can be used to track when the message is published. - futures = [] - for n in range(1, 10): data = u'Message number {}'.format(n) # Data must be a bytestring data = data.encode('utf-8') - message_future = publisher.publish(topic_path, data=data) - futures.append(message_future) - - print('Published message IDs:') - for future in futures: - # result() blocks until the message is published. + # When you publish a message, the client returns a future. + future = publisher.publish(topic_path, data=data) print(future.result()) + + print("Published messages with futures.") # [END pubsub_publisher_concurrency_control] @@ -169,28 +164,34 @@ def publish_messages_with_error_handler(project_id, topic_name): publisher = pubsub_v1.PublisherClient() topic_path = publisher.topic_path(project_id, topic_name) - def callback(message_future): - if message_future.exception(): - print('{} needs handling.'.format(message_future.exception())) - else: - print(message_future.result()) + futures = dict() - for n in range(1, 10): - data = u'Message number {}'.format(n) - # Data must be a bytestring - data = data.encode('utf-8') - # When you publish a message, the client returns a Future. - message_future = publisher.publish(topic_path, data=data) - # If you wish to handle publish failures, do it in the callback. - # Otherwise, it's okay to call `message_future.result()` directly. - message_future.add_done_callback(callback) - - print('Published message IDs:') - - # We keep the main thread from exiting so message futures can be - # resolved in the background. - while True: - time.sleep(60) + def get_callback(f, data): + def callback(f): + try: + print(f.result()) + futures.pop(data) + except: # noqa + print("Please handle {} for {}.".format(f.exception(), data)) + return callback + + for i in range(10): + data = str(i) + futures.update({data: None}) + # When you publish a message, the client returns a future. + future = publisher.publish( + topic_path, + data=data.encode("utf-8"), # data must be a bytestring. + ) + futures[data] = future + # Publish failures shall be handled in the callback function. + future.add_done_callback(get_callback(future, data)) + + # Wait for all the publish futures to resolve before exiting. + while futures: + time.sleep(5) + + print("Published message with error handler.") # [END pubsub_publish_messages_error_handler] @@ -215,9 +216,10 @@ def publish_messages_with_batch_settings(project_id, topic_name): data = u'Message number {}'.format(n) # Data must be a bytestring data = data.encode('utf-8') - publisher.publish(topic_path, data=data) + future = publisher.publish(topic_path, data=data) + print(future.result()) - print('Published messages.') + print('Published messages with batch settings.') # [END pubsub_publisher_batch_settings] diff --git a/samples/snippets/publisher_test.py b/samples/snippets/publisher_test.py index cdb4d0e0e..c2908d746 100644 --- a/samples/snippets/publisher_test.py +++ b/samples/snippets/publisher_test.py @@ -111,11 +111,7 @@ def test_publish_with_batch_settings(topic, capsys): def test_publish_with_error_handler(topic, capsys): - - with _make_sleep_patch(): - with pytest.raises(RuntimeError, match='sigil'): - publisher.publish_messages_with_error_handler( - PROJECT, TOPIC) + publisher.publish_messages_with_error_handler(PROJECT, TOPIC) out, _ = capsys.readouterr() assert 'Published' in out diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index df5b1092b..f91007a6d 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -169,13 +169,15 @@ def test_update(subscriber_client, subscription, capsys): def _publish_messages(publisher_client, topic): for n in range(5): data = u'Message {}'.format(n).encode('utf-8') - publisher_client.publish( + future = publisher_client.publish( topic, data=data) + future.result() def _publish_messages_with_custom_attributes(publisher_client, topic): data = u'Test message'.encode('utf-8') - publisher_client.publish(topic, data=data, origin='python-sample') + future = publisher_client.publish(topic, data=data, origin='python-sample') + future.result() def _make_sleep_patch(): From bd1a4ca6b85016ffeff291b32cf0b4b2d23d250e Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Mon, 22 Jul 2019 10:23:03 -0700 Subject: [PATCH 070/101] Pub/Sub: add publish retry sample [(#2273)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2273) * Publish retry sample * double to single quotes * double to single quotes * license year --- samples/snippets/iam.py | 2 +- samples/snippets/iam_test.py | 2 +- samples/snippets/publisher.py | 112 +++++++++++++++++++++++----- samples/snippets/publisher_test.py | 9 ++- samples/snippets/quickstart.py | 2 +- samples/snippets/quickstart_test.py | 2 +- samples/snippets/subscriber.py | 2 +- samples/snippets/subscriber_test.py | 2 +- 8 files changed, 106 insertions(+), 27 deletions(-) diff --git a/samples/snippets/iam.py b/samples/snippets/iam.py index bd44f1ab6..f9865ed39 100644 --- a/samples/snippets/iam.py +++ b/samples/snippets/iam.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright 2016 Google Inc. All Rights Reserved. +# Copyright 2019 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/samples/snippets/iam_test.py b/samples/snippets/iam_test.py index 8a524c35a..cfae98ffd 100644 --- a/samples/snippets/iam_test.py +++ b/samples/snippets/iam_test.py @@ -1,4 +1,4 @@ -# Copyright 2016 Google Inc. All Rights Reserved. +# Copyright 2019 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index 51edd7bd8..2a32dc1c7 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -120,7 +120,8 @@ def publish_messages_with_custom_attributes(project_id, topic_name): data = data.encode('utf-8') # Add two attributes, origin and username, to the message future = publisher.publish( - topic_path, data, origin='python-sample', username='gcp') + topic_path, data, origin='python-sample', username='gcp' + ) print(future.result()) print('Published messages with custom attributes.') @@ -147,7 +148,7 @@ def publish_messages_with_futures(project_id, topic_name): future = publisher.publish(topic_path, data=data) print(future.result()) - print("Published messages with futures.") + print('Published messages with futures.') # [END pubsub_publisher_concurrency_control] @@ -171,8 +172,9 @@ def callback(f): try: print(f.result()) futures.pop(data) - except: # noqa - print("Please handle {} for {}.".format(f.exception(), data)) + except: # noqa + print('Please handle {} for {}.'.format(f.exception(), data)) + return callback for i in range(10): @@ -180,8 +182,7 @@ def callback(f): futures.update({data: None}) # When you publish a message, the client returns a future. future = publisher.publish( - topic_path, - data=data.encode("utf-8"), # data must be a bytestring. + topic_path, data=data.encode('utf-8') # data must be a bytestring. ) futures[data] = future # Publish failures shall be handled in the callback function. @@ -191,7 +192,7 @@ def callback(f): while futures: time.sleep(5) - print("Published message with error handler.") + print('Published message with error handler.') # [END pubsub_publish_messages_error_handler] @@ -207,7 +208,7 @@ def publish_messages_with_batch_settings(project_id, topic_name): # of data or one second has passed. batch_settings = pubsub_v1.types.BatchSettings( max_bytes=1024, # One kilobyte - max_latency=1, # One second + max_latency=1, # One second ) publisher = pubsub_v1.PublisherClient(batch_settings) topic_path = publisher.topic_path(project_id, topic_name) @@ -223,7 +224,65 @@ def publish_messages_with_batch_settings(project_id, topic_name): # [END pubsub_publisher_batch_settings] -if __name__ == '__main__': +def publish_messages_with_retry_settings(project_id, topic_name): + """Publishes messages with custom retry settings.""" + # [START pubsub_publisher_retry_settings] + from google.cloud import pubsub_v1 + + # TODO project_id = "Your Google Cloud Project ID" + # TODO topic_name = "Your Pub/Sub topic name" + + # Configure the retry settings. Defaults will be overwritten. + retry_settings = { + 'interfaces': { + 'google.pubsub.v1.Publisher': { + 'retry_codes': { + 'publish': [ + 'ABORTED', + 'CANCELLED', + 'DEADLINE_EXCEEDED', + 'INTERNAL', + 'RESOURCE_EXHAUSTED', + 'UNAVAILABLE', + 'UNKNOWN', + ] + }, + 'retry_params': { + 'messaging': { + 'initial_retry_delay_millis': 150, # default: 100 + 'retry_delay_multiplier': 1.5, # default: 1.3 + 'max_retry_delay_millis': 65000, # default: 60000 + 'initial_rpc_timeout_millis': 25000, # default: 25000 + 'rpc_timeout_multiplier': 1.0, # default: 1.0 + 'max_rpc_timeout_millis': 35000, # default: 30000 + 'total_timeout_millis': 650000, # default: 600000 + } + }, + 'methods': { + 'Publish': { + 'retry_codes_name': 'publish', + 'retry_params_name': 'messaging', + } + }, + } + } + } + + publisher = pubsub_v1.PublisherClient(client_config=retry_settings) + topic_path = publisher.topic_path(project_id, topic_name) + + for n in range(1, 10): + data = u'Message number {}'.format(n) + # Data must be a bytestring + data = data.encode('utf-8') + future = publisher.publish(topic_path, data=data) + print(future.result()) + + print('Published messages with retry settings.') + # [END pubsub_publisher_retry_settings] + + +if __name__ == "__main__": parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter @@ -233,36 +292,47 @@ def publish_messages_with_batch_settings(project_id, topic_name): subparsers = parser.add_subparsers(dest='command') subparsers.add_parser('list', help=list_topics.__doc__) - create_parser = subparsers.add_parser('create', help=create_topic.__doc__) + create_parser = subparsers.add_parser('create', + help=create_topic.__doc__) create_parser.add_argument('topic_name') - delete_parser = subparsers.add_parser('delete', help=delete_topic.__doc__) + delete_parser = subparsers.add_parser('delete', + help=delete_topic.__doc__) delete_parser.add_argument('topic_name') - publish_parser = subparsers.add_parser( - 'publish', help=publish_messages.__doc__) + publish_parser = subparsers.add_parser('publish', + help=publish_messages.__doc__) publish_parser.add_argument('topic_name') publish_with_custom_attributes_parser = subparsers.add_parser( 'publish-with-custom-attributes', - help=publish_messages_with_custom_attributes.__doc__) + help=publish_messages_with_custom_attributes.__doc__, + ) publish_with_custom_attributes_parser.add_argument('topic_name') publish_with_futures_parser = subparsers.add_parser( - 'publish-with-futures', - help=publish_messages_with_futures.__doc__) + 'publish-with-futures', help=publish_messages_with_futures.__doc__ + ) publish_with_futures_parser.add_argument('topic_name') publish_with_error_handler_parser = subparsers.add_parser( 'publish-with-error-handler', - help=publish_messages_with_error_handler.__doc__) + help=publish_messages_with_error_handler.__doc__ + ) publish_with_error_handler_parser.add_argument('topic_name') publish_with_batch_settings_parser = subparsers.add_parser( 'publish-with-batch-settings', - help=publish_messages_with_batch_settings.__doc__) + help=publish_messages_with_batch_settings.__doc__ + ) publish_with_batch_settings_parser.add_argument('topic_name') + publish_with_retry_settings_parser = subparsers.add_parser( + 'publish-with-retry-settings', + help=publish_messages_with_retry_settings.__doc__ + ) + publish_with_retry_settings_parser.add_argument('topic_name') + args = parser.parse_args() if args.command == 'list': @@ -274,11 +344,13 @@ def publish_messages_with_batch_settings(project_id, topic_name): elif args.command == 'publish': publish_messages(args.project_id, args.topic_name) elif args.command == 'publish-with-custom-attributes': - publish_messages_with_custom_attributes( - args.project_id, args.topic_name) + publish_messages_with_custom_attributes(args.project_id, + args.topic_name) elif args.command == 'publish-with-futures': publish_messages_with_futures(args.project_id, args.topic_name) elif args.command == 'publish-with-error-handler': publish_messages_with_error_handler(args.project_id, args.topic_name) elif args.command == 'publish-with-batch-settings': publish_messages_with_batch_settings(args.project_id, args.topic_name) + elif args.command == 'publish-with-retry-settings': + publish_messages_with_retry_settings(args.project_id, args.topic_name) diff --git a/samples/snippets/publisher_test.py b/samples/snippets/publisher_test.py index c2908d746..b364553c2 100644 --- a/samples/snippets/publisher_test.py +++ b/samples/snippets/publisher_test.py @@ -1,4 +1,4 @@ -# Copyright 2016 Google Inc. All Rights Reserved. +# Copyright 2019 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -110,6 +110,13 @@ def test_publish_with_batch_settings(topic, capsys): assert 'Published' in out +def test_publish_with_retry_settings(topic, capsys): + publisher.publish_messages_with_retry_settings(PROJECT, TOPIC) + + out, _ = capsys.readouterr() + assert 'Published' in out + + def test_publish_with_error_handler(topic, capsys): publisher.publish_messages_with_error_handler(PROJECT, TOPIC) diff --git a/samples/snippets/quickstart.py b/samples/snippets/quickstart.py index f48d085e0..10ff76f9b 100644 --- a/samples/snippets/quickstart.py +++ b/samples/snippets/quickstart.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright 2016 Google Inc. All Rights Reserved. +# Copyright 2019 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/samples/snippets/quickstart_test.py b/samples/snippets/quickstart_test.py index ee6f7d4b2..3fce09dc8 100644 --- a/samples/snippets/quickstart_test.py +++ b/samples/snippets/quickstart_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright 2018 Google Inc. All Rights Reserved. +# Copyright 2019 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 5802218b4..92d779135 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright 2016 Google Inc. All Rights Reserved. +# Copyright 2019 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index f91007a6d..2dcfb33e2 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -1,4 +1,4 @@ -# Copyright 2016 Google Inc. All Rights Reserved. +# Copyright 2019 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 678a1eba30d38bc602ee959bea4bc41b349e4fc2 Mon Sep 17 00:00:00 2001 From: Keiji Yoshida Date: Wed, 31 Jul 2019 05:52:46 +0900 Subject: [PATCH 071/101] Fix a TODO comment on pubsub/cloud-client/subscriber.py [(#2302)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2302) --- samples/snippets/subscriber.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 92d779135..64e939515 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -366,7 +366,7 @@ def listen_for_errors(project_id, subscription_name): # [START pubsub_subscriber_error_listener] from google.cloud import pubsub_v1 - # TODO project = "Your Google Cloud Project ID" + # TODO project_id = "Your Google Cloud Project ID" # TODO subscription_name = "Your Pubsub subscription name" subscriber = pubsub_v1.SubscriberClient() From c328d2624c32361aaedd471770f8563d42e70969 Mon Sep 17 00:00:00 2001 From: oli Date: Tue, 13 Aug 2019 06:02:18 +1000 Subject: [PATCH 072/101] Print actual number of messages pulled [(#2078)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2078) * Print actual number of messages pulled --- samples/snippets/subscriber.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 64e939515..181e8235b 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -284,7 +284,8 @@ def synchronous_pull(project_id, subscription_name): # Acknowledges the received messages so they will not be sent again. subscriber.acknowledge(subscription_path, ack_ids) - print("Received and acknowledged {} messages. Done.".format(NUM_MESSAGES)) + print('Received and acknowledged {} messages. Done.'.format( + len(response.received_messages))) # [END pubsub_subscriber_sync_pull] @@ -357,7 +358,8 @@ def worker(msg): if processes: time.sleep(SLEEP_TIME) - print("Received and acknowledged {} messages. Done.".format(NUM_MESSAGES)) + print('Received and acknowledged {} messages. Done.'.format( + len(response.received_messages))) # [END pubsub_subscriber_sync_pull_with_lease] From 96d642dbe3834caec93c354faeb52fce9f428727 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Tue, 13 Aug 2019 10:07:22 -0700 Subject: [PATCH 073/101] Pub/Sub: fix subscriber async region tag mistake [(#2334)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2334) --- samples/snippets/subscriber.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 181e8235b..dbaa396cd 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -198,6 +198,7 @@ def callback(message): def receive_messages_with_custom_attributes(project_id, subscription_name): """Receives messages from a pull subscription.""" # [START pubsub_subscriber_sync_pull_custom_attributes] + # [START pubsub_subscriber_async_pull_custom_attributes] import time from google.cloud import pubsub_v1 @@ -225,6 +226,7 @@ def callback(message): print('Listening for messages on {}'.format(subscription_path)) while True: time.sleep(60) + # [END pubsub_subscriber_async_pull_custom_attributes] # [END pubsub_subscriber_sync_pull_custom_attributes] From 2387979a9fedbe2514936b746a7e4afab2de846e Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Thu, 12 Sep 2019 14:01:54 -0700 Subject: [PATCH 074/101] Pub/Sub: update retry settings in sample [(#2395)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2395) --- samples/snippets/publisher.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index 2a32dc1c7..76554d025 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -249,13 +249,13 @@ def publish_messages_with_retry_settings(project_id, topic_name): }, 'retry_params': { 'messaging': { - 'initial_retry_delay_millis': 150, # default: 100 - 'retry_delay_multiplier': 1.5, # default: 1.3 - 'max_retry_delay_millis': 65000, # default: 60000 - 'initial_rpc_timeout_millis': 25000, # default: 25000 + 'initial_retry_delay_millis': 100, # default: 100 + 'retry_delay_multiplier': 1.3, # default: 1.3 + 'max_retry_delay_millis': 60000, # default: 60000 + 'initial_rpc_timeout_millis': 5000, # default: 25000 'rpc_timeout_multiplier': 1.0, # default: 1.0 - 'max_rpc_timeout_millis': 35000, # default: 30000 - 'total_timeout_millis': 650000, # default: 600000 + 'max_rpc_timeout_millis': 600000, # default: 30000 + 'total_timeout_millis': 600000, # default: 600000 } }, 'methods': { From 65c8ff620776928d731bd05b582d930ae7e11c27 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Tue, 17 Sep 2019 10:48:30 -0700 Subject: [PATCH 075/101] Pub/Sub: improve pub.py [(#2403)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2403) * print number of messages published * two nit's --- samples/snippets/quickstart/pub.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/samples/snippets/quickstart/pub.py b/samples/snippets/quickstart/pub.py index 9617b34ea..e340eb4f3 100644 --- a/samples/snippets/quickstart/pub.py +++ b/samples/snippets/quickstart/pub.py @@ -22,13 +22,13 @@ # [END pubsub_quickstart_pub_deps] -def get_callback(api_future, data): +def get_callback(api_future, data, ref): """Wrap message data in the context of the callback function.""" - def callback(api_future): try: print("Published message {} now has message ID {}".format( data, api_future.result())) + ref["num_messages"] += 1 except Exception: print("A problem occurred when publishing {}: {}\n".format( data, api_future.exception())) @@ -39,24 +39,28 @@ def callback(api_future): def pub(project_id, topic_name): """Publishes a message to a Pub/Sub topic.""" # [START pubsub_quickstart_pub_client] - # Initialize a Publisher client + # Initialize a Publisher client. client = pubsub_v1.PublisherClient() # [END pubsub_quickstart_pub_client] # Create a fully qualified identifier in the form of # `projects/{project_id}/topics/{topic_name}` topic_path = client.topic_path(project_id, topic_name) - # Data sent to Cloud Pub/Sub must be a bytestring + # Data sent to Cloud Pub/Sub must be a bytestring. data = b"Hello, World!" + # Keep track of the number of published messages. + ref = dict({"num_messages": 0}) + # When you publish a message, the client returns a future. api_future = client.publish(topic_path, data=data) - api_future.add_done_callback(get_callback(api_future, data)) + api_future.add_done_callback(get_callback(api_future, data, ref)) - # Keep the main thread from exiting until background message - # is processed. + # Keep the main thread from exiting while the message future + # gets resolved in the background. while api_future.running(): - time.sleep(0.1) + time.sleep(0.5) + print("Published {} message(s).".format(ref["num_messages"])) if __name__ == '__main__': From e497a9499d92ed97f807a5eeee3783afe5f47081 Mon Sep 17 00:00:00 2001 From: Gus Class Date: Mon, 7 Oct 2019 15:45:22 -0700 Subject: [PATCH 076/101] Adds updates for samples profiler ... vision [(#2439)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2439) --- samples/snippets/requirements.txt | 2 +- samples/snippets/subscriber_test.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index d8470ecf9..a97fc0997 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==0.39.1 +google-cloud-pubsub==1.0.0 diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index 2dcfb33e2..1a5de9930 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -16,6 +16,7 @@ import time from gcp_devrel.testing import eventually_consistent +from gcp_devrel.testing.flaky import flaky from google.cloud import pubsub_v1 import google.api_core.exceptions import mock @@ -193,6 +194,7 @@ def new_sleep(period): return mock.patch('time.sleep', new=new_sleep) +@flaky def test_receive(publisher_client, topic, subscription, capsys): _publish_messages(publisher_client, topic) From 1ec3338955783d8a3346f6aaedeaac050f8f29f7 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Mon, 21 Oct 2019 13:56:39 -0700 Subject: [PATCH 077/101] Pub/Sub: update how subscriber client listens to StreamingPullFuture [(#2475)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2475) * update sub.py & requirements.txt * fix flaky subscriber test with separate subscriptions --- samples/snippets/iam_test.py | 2 +- samples/snippets/publisher.py | 2 +- samples/snippets/publisher_test.py | 12 +- samples/snippets/quickstart.py | 2 +- samples/snippets/quickstart/sub.py | 19 +-- samples/snippets/quickstart/sub_test.py | 91 ++++++------ samples/snippets/quickstart_test.py | 2 +- samples/snippets/subscriber.py | 2 +- samples/snippets/subscriber_test.py | 180 +++++++++++------------- 9 files changed, 148 insertions(+), 164 deletions(-) diff --git a/samples/snippets/iam_test.py b/samples/snippets/iam_test.py index cfae98ffd..8a524c35a 100644 --- a/samples/snippets/iam_test.py +++ b/samples/snippets/iam_test.py @@ -1,4 +1,4 @@ -# Copyright 2019 Google Inc. All Rights Reserved. +# Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index 76554d025..490c903b2 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright 2019 Google LLC. All Rights Reserved. +# Copyright 2016 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/samples/snippets/publisher_test.py b/samples/snippets/publisher_test.py index b364553c2..5e550abd6 100644 --- a/samples/snippets/publisher_test.py +++ b/samples/snippets/publisher_test.py @@ -1,4 +1,4 @@ -# Copyright 2019 Google Inc. All Rights Reserved. +# Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -36,13 +36,11 @@ def topic(client): topic_path = client.topic_path(PROJECT, TOPIC) try: - client.delete_topic(topic_path) - except Exception: - pass - - client.create_topic(topic_path) + response = client.get_topic(topic_path) + except: # noqa + response = client.create_topic(topic_path) - yield topic_path + yield response.name def _make_sleep_patch(): diff --git a/samples/snippets/quickstart.py b/samples/snippets/quickstart.py index 10ff76f9b..f48d085e0 100644 --- a/samples/snippets/quickstart.py +++ b/samples/snippets/quickstart.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright 2019 Google Inc. All Rights Reserved. +# Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/samples/snippets/quickstart/sub.py b/samples/snippets/quickstart/sub.py index 520803d70..e39f14105 100644 --- a/samples/snippets/quickstart/sub.py +++ b/samples/snippets/quickstart/sub.py @@ -16,7 +16,6 @@ # [START pubsub_quickstart_sub_all] import argparse -import time # [START pubsub_quickstart_sub_deps] from google.cloud import pubsub_v1 # [END pubsub_quickstart_sub_deps] @@ -34,20 +33,22 @@ def sub(project_id, subscription_name): project_id, subscription_name) def callback(message): - print('Received message {} of message ID {}'.format( + print('Received message {} of message ID {}\n'.format( message, message.message_id)) # Acknowledge the message. Unack'ed messages will be redelivered. message.ack() - print('Acknowledged message of message ID {}\n'.format( - message.message_id)) + print('Acknowledged message {}\n'.format(message.message_id)) - client.subscribe(subscription_path, callback=callback) + streaming_pull_future = client.subscribe( + subscription_path, callback=callback) print('Listening for messages on {}..\n'.format(subscription_path)) - # Keep the main thread from exiting so the subscriber can - # process messages in the background. - while True: - time.sleep(60) + # Calling result() on StreamingPullFuture keeps the main thread from + # exiting while messages get processed in the callbacks. + try: + streaming_pull_future.result() + except: # noqa + streaming_pull_future.cancel() if __name__ == '__main__': diff --git a/samples/snippets/quickstart/sub_test.py b/samples/snippets/quickstart/sub_test.py index 9c70384ed..476139a02 100644 --- a/samples/snippets/quickstart/sub_test.py +++ b/samples/snippets/quickstart/sub_test.py @@ -14,10 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock import os import pytest -import time from google.api_core.exceptions import AlreadyExists from google.cloud import pubsub_v1 @@ -29,84 +27,79 @@ TOPIC = 'quickstart-sub-test-topic' SUBSCRIPTION = 'quickstart-sub-test-topic-sub' - -@pytest.fixture(scope='module') -def publisher_client(): - yield pubsub_v1.PublisherClient() +publisher_client = pubsub_v1.PublisherClient() +subscriber_client = pubsub_v1.SubscriberClient() @pytest.fixture(scope='module') -def topic_path(publisher_client): +def topic_path(): topic_path = publisher_client.topic_path(PROJECT, TOPIC) try: - publisher_client.create_topic(topic_path) + topic = publisher_client.create_topic(topic_path) + return topic.name except AlreadyExists: - pass - - yield topic_path - - -@pytest.fixture(scope='module') -def subscriber_client(): - yield pubsub_v1.SubscriberClient() + return topic_path @pytest.fixture(scope='module') -def subscription(subscriber_client, topic_path): +def subscription_path(topic_path): subscription_path = subscriber_client.subscription_path( PROJECT, SUBSCRIPTION) try: - subscriber_client.create_subscription(subscription_path, topic_path) + subscription = subscriber_client.create_subscription( + subscription_path, topic_path) + return subscription.name except AlreadyExists: - pass - - yield SUBSCRIPTION + return subscription_path -@pytest.fixture -def to_delete(publisher_client, subscriber_client): - doomed = [] - yield doomed - for client, item in doomed: +def _to_delete(resource_paths): + for item in resource_paths: if 'topics' in item: publisher_client.delete_topic(item) if 'subscriptions' in item: subscriber_client.delete_subscription(item) -def _make_sleep_patch(): - real_sleep = time.sleep +def _publish_messages(topic_path): + publish_future = publisher_client.publish(topic_path, data=b'Hello World!') + publish_future.result() + - def new_sleep(period): - if period == 60: - real_sleep(10) - raise RuntimeError('sigil') - else: - real_sleep(period) +def _sub_timeout(project_id, subscription_name): + # This is an exactly copy of `sub.py` except + # StreamingPullFuture.result() will time out after 10s. + client = pubsub_v1.SubscriberClient() + subscription_path = client.subscription_path( + project_id, subscription_name) - return mock.patch('time.sleep', new=new_sleep) + def callback(message): + print('Received message {} of message ID {}\n'.format( + message, message.message_id)) + message.ack() + print('Acknowledged message {}\n'.format(message.message_id)) + streaming_pull_future = client.subscribe( + subscription_path, callback=callback) + print('Listening for messages on {}..\n'.format(subscription_path)) + + try: + streaming_pull_future.result(timeout=10) + except: # noqa + streaming_pull_future.cancel() -def test_sub(publisher_client, - topic_path, - subscriber_client, - subscription, - to_delete, - capsys): - publisher_client.publish(topic_path, data=b'Hello, World!') +def test_sub(monkeypatch, topic_path, subscription_path, capsys): + monkeypatch.setattr(sub, 'sub', _sub_timeout) - to_delete.append((publisher_client, topic_path)) + _publish_messages(topic_path) - with _make_sleep_patch(): - with pytest.raises(RuntimeError, match='sigil'): - sub.sub(PROJECT, subscription) + sub.sub(PROJECT, SUBSCRIPTION) - to_delete.append((subscriber_client, - 'projects/{}/subscriptions/{}'.format(PROJECT, - SUBSCRIPTION))) + # Clean up resources. + _to_delete([topic_path, subscription_path]) out, _ = capsys.readouterr() assert "Received message" in out diff --git a/samples/snippets/quickstart_test.py b/samples/snippets/quickstart_test.py index 3fce09dc8..d318b260c 100644 --- a/samples/snippets/quickstart_test.py +++ b/samples/snippets/quickstart_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright 2019 Google Inc. All Rights Reserved. +# Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index dbaa396cd..3bbad0ead 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright 2019 Google Inc. All Rights Reserved. +# Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index 1a5de9930..4c5fd6122 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -1,4 +1,4 @@ -# Copyright 2019 Google Inc. All Rights Reserved. +# Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,9 +16,7 @@ import time from gcp_devrel.testing import eventually_consistent -from gcp_devrel.testing.flaky import flaky from google.cloud import pubsub_v1 -import google.api_core.exceptions import mock import pytest @@ -26,9 +24,9 @@ PROJECT = os.environ['GCLOUD_PROJECT'] TOPIC = 'subscription-test-topic' -SUBSCRIPTION = 'subscription-test-subscription' -SUBSCRIPTION_SYNC1 = 'subscription-test-subscription-sync1' -SUBSCRIPTION_SYNC2 = 'subscription-test-subscription-sync2' +SUBSCRIPTION_ONE = 'subscription-test-subscription-one' +SUBSCRIPTION_TWO = 'subscription-test-subscription-two' +SUBSCRIPTION_THREE = 'subscription-test-subscription-three' ENDPOINT = 'https://{}.appspot.com/push'.format(PROJECT) NEW_ENDPOINT = 'https://{}.appspot.com/push2'.format(PROJECT) @@ -43,13 +41,11 @@ def topic(publisher_client): topic_path = publisher_client.topic_path(PROJECT, TOPIC) try: - publisher_client.delete_topic(topic_path) - except Exception: - pass - - publisher_client.create_topic(topic_path) + response = publisher_client.get_topic(topic_path) + except: # noqa + response = publisher_client.create_topic(topic_path) - yield topic_path + yield response.name @pytest.fixture(scope='module') @@ -57,79 +53,74 @@ def subscriber_client(): yield pubsub_v1.SubscriberClient() -@pytest.fixture -def subscription(subscriber_client, topic): +@pytest.fixture(scope='module') +def subscription_one(subscriber_client, topic): subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION) + PROJECT, SUBSCRIPTION_ONE) try: - subscriber_client.delete_subscription(subscription_path) - except Exception: - pass + response = subscriber_client.get_subscription(subscription_path) + except: # noqa + response = subscriber_client.create_subscription( + subscription_path, topic=topic) - try: - subscriber_client.create_subscription(subscription_path, topic=topic) - except google.api_core.exceptions.AlreadyExists: - pass + yield response.name - yield subscription_path - -@pytest.fixture -def subscription_sync1(subscriber_client, topic): - subscription_sync_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_SYNC1) +@pytest.fixture(scope='module') +def subscription_two(subscriber_client, topic): + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION_TWO) try: - subscriber_client.delete_subscription(subscription_sync_path) - except Exception: - pass - - subscriber_client.create_subscription(subscription_sync_path, topic=topic) + response = subscriber_client.get_subscription(subscription_path) + except: # noqa + response = subscriber_client.create_subscription( + subscription_path, topic=topic) - yield subscription_sync_path + yield response.name -@pytest.fixture -def subscription_sync2(subscriber_client, topic): - subscription_sync_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_SYNC2) +@pytest.fixture(scope='module') +def subscription_three(subscriber_client, topic): + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION_THREE) try: - subscriber_client.delete_subscription(subscription_sync_path) - except Exception: - pass - - subscriber_client.create_subscription(subscription_sync_path, topic=topic) + response = subscriber_client.get_subscription(subscription_path) + except: # noqa + response = subscriber_client.create_subscription( + subscription_path, topic=topic) - yield subscription_sync_path + yield response.name -def test_list_in_topic(subscription, capsys): +def test_list_in_topic(subscription_one, capsys): @eventually_consistent.call def _(): subscriber.list_subscriptions_in_topic(PROJECT, TOPIC) out, _ = capsys.readouterr() - assert subscription in out + assert subscription_one in out -def test_list_in_project(subscription, capsys): +def test_list_in_project(subscription_one, capsys): @eventually_consistent.call def _(): subscriber.list_subscriptions_in_project(PROJECT) out, _ = capsys.readouterr() - assert subscription in out + assert subscription_one in out def test_create(subscriber_client): subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION) + PROJECT, SUBSCRIPTION_ONE) + try: subscriber_client.delete_subscription(subscription_path) except Exception: pass - subscriber.create_subscription(PROJECT, TOPIC, SUBSCRIPTION) + subscriber.create_subscription(PROJECT, TOPIC, SUBSCRIPTION_ONE) @eventually_consistent.call def _(): @@ -138,40 +129,40 @@ def _(): def test_create_push(subscriber_client): subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION) + PROJECT, SUBSCRIPTION_ONE) try: subscriber_client.delete_subscription(subscription_path) except Exception: pass - subscriber.create_push_subscription(PROJECT, TOPIC, SUBSCRIPTION, ENDPOINT) + subscriber.create_push_subscription( + PROJECT, TOPIC, SUBSCRIPTION_ONE, ENDPOINT) @eventually_consistent.call def _(): assert subscriber_client.get_subscription(subscription_path) -def test_delete(subscriber_client, subscription): - subscriber.delete_subscription(PROJECT, SUBSCRIPTION) +def test_update(subscriber_client, subscription_one, capsys): + subscriber.update_subscription(PROJECT, SUBSCRIPTION_ONE, NEW_ENDPOINT) - @eventually_consistent.call - def _(): - with pytest.raises(Exception): - subscriber_client.get_subscription(subscription) + out, _ = capsys.readouterr() + assert 'Subscription updated' in out -def test_update(subscriber_client, subscription, capsys): - subscriber.update_subscription(PROJECT, SUBSCRIPTION, NEW_ENDPOINT) +def test_delete(subscriber_client, subscription_one): + subscriber.delete_subscription(PROJECT, SUBSCRIPTION_ONE) - out, _ = capsys.readouterr() - assert 'Subscription updated' in out + @eventually_consistent.call + def _(): + with pytest.raises(Exception): + subscriber_client.get_subscription(subscription_one) def _publish_messages(publisher_client, topic): for n in range(5): data = u'Message {}'.format(n).encode('utf-8') - future = publisher_client.publish( - topic, data=data) + future = publisher_client.publish(topic, data=data) future.result() @@ -194,49 +185,28 @@ def new_sleep(period): return mock.patch('time.sleep', new=new_sleep) -@flaky -def test_receive(publisher_client, topic, subscription, capsys): +def test_receive(publisher_client, topic, subscription_two, capsys): _publish_messages(publisher_client, topic) with _make_sleep_patch(): with pytest.raises(RuntimeError, match='sigil'): - subscriber.receive_messages(PROJECT, SUBSCRIPTION) + subscriber.receive_messages(PROJECT, SUBSCRIPTION_TWO) out, _ = capsys.readouterr() assert 'Listening' in out - assert subscription in out + assert subscription_two in out assert 'Message 1' in out -def test_receive_synchronously( - publisher_client, topic, subscription_sync1, capsys): - _publish_messages(publisher_client, topic) - - subscriber.synchronous_pull(PROJECT, SUBSCRIPTION_SYNC1) - - out, _ = capsys.readouterr() - assert 'Done.' in out - - -def test_receive_synchronously_with_lease( - publisher_client, topic, subscription_sync2, capsys): - _publish_messages(publisher_client, topic) - - subscriber.synchronous_pull_with_lease_management( - PROJECT, SUBSCRIPTION_SYNC2) - - out, _ = capsys.readouterr() - assert 'Done.' in out - - def test_receive_with_custom_attributes( - publisher_client, topic, subscription, capsys): + publisher_client, topic, subscription_two, capsys): + _publish_messages_with_custom_attributes(publisher_client, topic) with _make_sleep_patch(): with pytest.raises(RuntimeError, match='sigil'): subscriber.receive_messages_with_custom_attributes( - PROJECT, SUBSCRIPTION) + PROJECT, SUBSCRIPTION_TWO) out, _ = capsys.readouterr() assert 'Test message' in out @@ -245,15 +215,37 @@ def test_receive_with_custom_attributes( def test_receive_with_flow_control( - publisher_client, topic, subscription, capsys): + publisher_client, topic, subscription_two, capsys): + _publish_messages(publisher_client, topic) with _make_sleep_patch(): with pytest.raises(RuntimeError, match='sigil'): subscriber.receive_messages_with_flow_control( - PROJECT, SUBSCRIPTION) + PROJECT, SUBSCRIPTION_TWO) out, _ = capsys.readouterr() assert 'Listening' in out - assert subscription in out + assert subscription_two in out assert 'Message 1' in out + + +def test_receive_synchronously( + publisher_client, topic, subscription_three, capsys): + _publish_messages(publisher_client, topic) + + subscriber.synchronous_pull(PROJECT, SUBSCRIPTION_THREE) + + out, _ = capsys.readouterr() + assert 'Done.' in out + + +def test_receive_synchronously_with_lease( + publisher_client, topic, subscription_three, capsys): + _publish_messages(publisher_client, topic) + + subscriber.synchronous_pull_with_lease_management( + PROJECT, SUBSCRIPTION_THREE) + + out, _ = capsys.readouterr() + assert 'Done.' in out From a2b4ac6244b8c959216ebf7522b954d1b7d1f029 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Tue, 10 Dec 2019 16:31:20 -0800 Subject: [PATCH 078/101] Pub/Sub: update how to test with mock [(#2555)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2555) * Update test with mock * Clean up resources after tests * Use unique resource names avoid test failures * Delete subscriptions in cleanup phase * Ensure unique topic name * Update assert to remove bytestring notation * Rewrite PubSubToGCS test using dataflow testing module --- samples/snippets/iam.py | 115 +++++----- samples/snippets/iam_test.py | 33 +-- samples/snippets/publisher.py | 167 ++++++++------- samples/snippets/publisher_test.py | 31 ++- samples/snippets/quickstart.py | 39 ++-- samples/snippets/quickstart/pub.py | 26 ++- samples/snippets/quickstart/pub_test.py | 24 +-- samples/snippets/quickstart/sub.py | 27 ++- samples/snippets/quickstart/sub_test.py | 82 ++++---- samples/snippets/quickstart_test.py | 23 +- samples/snippets/subscriber.py | 269 ++++++++++++++---------- samples/snippets/subscriber_test.py | 131 +++++++----- 12 files changed, 543 insertions(+), 424 deletions(-) diff --git a/samples/snippets/iam.py b/samples/snippets/iam.py index f9865ed39..f014ce749 100644 --- a/samples/snippets/iam.py +++ b/samples/snippets/iam.py @@ -34,9 +34,9 @@ def get_topic_policy(project, topic_name): policy = client.get_iam_policy(topic_path) - print('Policy for topic {}:'.format(topic_path)) + print("Policy for topic {}:".format(topic_path)) for binding in policy.bindings: - print('Role: {}, Members: {}'.format(binding.role, binding.members)) + print("Role: {}, Members: {}".format(binding.role, binding.members)) # [END pubsub_get_topic_policy] @@ -48,9 +48,9 @@ def get_subscription_policy(project, subscription_name): policy = client.get_iam_policy(subscription_path) - print('Policy for subscription {}:'.format(subscription_path)) + print("Policy for subscription {}:".format(subscription_path)) for binding in policy.bindings: - print('Role: {}, Members: {}'.format(binding.role, binding.members)) + print("Role: {}, Members: {}".format(binding.role, binding.members)) # [END pubsub_get_subscription_policy] @@ -63,20 +63,17 @@ def set_topic_policy(project, topic_name): policy = client.get_iam_policy(topic_path) # Add all users as viewers. - policy.bindings.add( - role='roles/pubsub.viewer', - members=['allUsers']) + policy.bindings.add(role="roles/pubsub.viewer", members=["allUsers"]) # Add a group as a publisher. policy.bindings.add( - role='roles/pubsub.publisher', - members=['group:cloud-logs@google.com']) + role="roles/pubsub.publisher", members=["group:cloud-logs@google.com"] + ) # Set the policy policy = client.set_iam_policy(topic_path, policy) - print('IAM policy for topic {} set: {}'.format( - topic_name, policy)) + print("IAM policy for topic {} set: {}".format(topic_name, policy)) # [END pubsub_set_topic_policy] @@ -89,20 +86,21 @@ def set_subscription_policy(project, subscription_name): policy = client.get_iam_policy(subscription_path) # Add all users as viewers. - policy.bindings.add( - role='roles/pubsub.viewer', - members=['allUsers']) + policy.bindings.add(role="roles/pubsub.viewer", members=["allUsers"]) # Add a group as an editor. policy.bindings.add( - role='roles/editor', - members=['group:cloud-logs@google.com']) + role="roles/editor", members=["group:cloud-logs@google.com"] + ) # Set the policy policy = client.set_iam_policy(subscription_path, policy) - print('IAM policy for subscription {} set: {}'.format( - subscription_name, policy)) + print( + "IAM policy for subscription {} set: {}".format( + subscription_name, policy + ) + ) # [END pubsub_set_subscription_policy] @@ -112,16 +110,17 @@ def check_topic_permissions(project, topic_name): client = pubsub_v1.PublisherClient() topic_path = client.topic_path(project, topic_name) - permissions_to_check = [ - 'pubsub.topics.publish', - 'pubsub.topics.update' - ] + permissions_to_check = ["pubsub.topics.publish", "pubsub.topics.update"] allowed_permissions = client.test_iam_permissions( - topic_path, permissions_to_check) + topic_path, permissions_to_check + ) - print('Allowed permissions for topic {}: {}'.format( - topic_path, allowed_permissions)) + print( + "Allowed permissions for topic {}: {}".format( + topic_path, allowed_permissions + ) + ) # [END pubsub_test_topic_permissions] @@ -132,63 +131,73 @@ def check_subscription_permissions(project, subscription_name): subscription_path = client.subscription_path(project, subscription_name) permissions_to_check = [ - 'pubsub.subscriptions.consume', - 'pubsub.subscriptions.update' + "pubsub.subscriptions.consume", + "pubsub.subscriptions.update", ] allowed_permissions = client.test_iam_permissions( - subscription_path, permissions_to_check) + subscription_path, permissions_to_check + ) - print('Allowed permissions for subscription {}: {}'.format( - subscription_path, allowed_permissions)) + print( + "Allowed permissions for subscription {}: {}".format( + subscription_path, allowed_permissions + ) + ) # [END pubsub_test_subscription_permissions] -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser( description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + formatter_class=argparse.RawDescriptionHelpFormatter, ) - parser.add_argument('project', help='Your Google Cloud project ID') + parser.add_argument("project", help="Your Google Cloud project ID") - subparsers = parser.add_subparsers(dest='command') + subparsers = parser.add_subparsers(dest="command") get_topic_policy_parser = subparsers.add_parser( - 'get-topic-policy', help=get_topic_policy.__doc__) - get_topic_policy_parser.add_argument('topic_name') + "get-topic-policy", help=get_topic_policy.__doc__ + ) + get_topic_policy_parser.add_argument("topic_name") get_subscription_policy_parser = subparsers.add_parser( - 'get-subscription-policy', help=get_subscription_policy.__doc__) - get_subscription_policy_parser.add_argument('subscription_name') + "get-subscription-policy", help=get_subscription_policy.__doc__ + ) + get_subscription_policy_parser.add_argument("subscription_name") set_topic_policy_parser = subparsers.add_parser( - 'set-topic-policy', help=set_topic_policy.__doc__) - set_topic_policy_parser.add_argument('topic_name') + "set-topic-policy", help=set_topic_policy.__doc__ + ) + set_topic_policy_parser.add_argument("topic_name") set_subscription_policy_parser = subparsers.add_parser( - 'set-subscription-policy', help=set_subscription_policy.__doc__) - set_subscription_policy_parser.add_argument('subscription_name') + "set-subscription-policy", help=set_subscription_policy.__doc__ + ) + set_subscription_policy_parser.add_argument("subscription_name") check_topic_permissions_parser = subparsers.add_parser( - 'check-topic-permissions', help=check_topic_permissions.__doc__) - check_topic_permissions_parser.add_argument('topic_name') + "check-topic-permissions", help=check_topic_permissions.__doc__ + ) + check_topic_permissions_parser.add_argument("topic_name") check_subscription_permissions_parser = subparsers.add_parser( - 'check-subscription-permissions', - help=check_subscription_permissions.__doc__) - check_subscription_permissions_parser.add_argument('subscription_name') + "check-subscription-permissions", + help=check_subscription_permissions.__doc__, + ) + check_subscription_permissions_parser.add_argument("subscription_name") args = parser.parse_args() - if args.command == 'get-topic-policy': + if args.command == "get-topic-policy": get_topic_policy(args.project, args.topic_name) - elif args.command == 'get-subscription-policy': + elif args.command == "get-subscription-policy": get_subscription_policy(args.project, args.subscription_name) - elif args.command == 'set-topic-policy': + elif args.command == "set-topic-policy": set_topic_policy(args.project, args.topic_name) - elif args.command == 'set-subscription-policy': + elif args.command == "set-subscription-policy": set_subscription_policy(args.project, args.subscription_name) - elif args.command == 'check-topic-permissions': + elif args.command == "check-topic-permissions": check_topic_permissions(args.project, args.topic_name) - elif args.command == 'check-subscription-permissions': + elif args.command == "check-subscription-permissions": check_subscription_permissions(args.project, args.subscription_name) diff --git a/samples/snippets/iam_test.py b/samples/snippets/iam_test.py index 8a524c35a..2b019f9ea 100644 --- a/samples/snippets/iam_test.py +++ b/samples/snippets/iam_test.py @@ -13,23 +13,25 @@ # limitations under the License. import os +import uuid from google.cloud import pubsub_v1 import pytest import iam -PROJECT = os.environ['GCLOUD_PROJECT'] -TOPIC = 'iam-test-topic' -SUBSCRIPTION = 'iam-test-subscription' +UUID = uuid.uuid4().hex +PROJECT = os.environ["GCLOUD_PROJECT"] +TOPIC = "iam-test-topic-" + UUID +SUBSCRIPTION = "iam-test-subscription-" + UUID -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def publisher_client(): yield pubsub_v1.PublisherClient() -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def topic(publisher_client): topic_path = publisher_client.topic_path(PROJECT, TOPIC) @@ -42,8 +44,10 @@ def topic(publisher_client): yield topic_path + publisher_client.delete_topic(topic_path) -@pytest.fixture(scope='module') + +@pytest.fixture(scope="module") def subscriber_client(): yield pubsub_v1.SubscriberClient() @@ -51,7 +55,8 @@ def subscriber_client(): @pytest.fixture def subscription(subscriber_client, topic): subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION) + PROJECT, SUBSCRIPTION + ) try: subscriber_client.delete_subscription(subscription_path) @@ -62,6 +67,8 @@ def subscription(subscriber_client, topic): yield subscription_path + subscriber_client.delete_subscription(subscription_path) + def test_get_topic_policy(topic, capsys): iam.get_topic_policy(PROJECT, TOPIC) @@ -81,16 +88,16 @@ def test_set_topic_policy(publisher_client, topic): iam.set_topic_policy(PROJECT, TOPIC) policy = publisher_client.get_iam_policy(topic) - assert 'roles/pubsub.publisher' in str(policy) - assert 'allUsers' in str(policy) + assert "roles/pubsub.publisher" in str(policy) + assert "allUsers" in str(policy) def test_set_subscription_policy(subscriber_client, subscription): iam.set_subscription_policy(PROJECT, SUBSCRIPTION) policy = subscriber_client.get_iam_policy(subscription) - assert 'roles/pubsub.viewer' in str(policy) - assert 'allUsers' in str(policy) + assert "roles/pubsub.viewer" in str(policy) + assert "allUsers" in str(policy) def test_check_topic_permissions(topic, capsys): @@ -99,7 +106,7 @@ def test_check_topic_permissions(topic, capsys): out, _ = capsys.readouterr() assert topic in out - assert 'pubsub.topics.publish' in out + assert "pubsub.topics.publish" in out def test_check_subscription_permissions(subscription, capsys): @@ -108,4 +115,4 @@ def test_check_subscription_permissions(subscription, capsys): out, _ = capsys.readouterr() assert subscription in out - assert 'pubsub.subscriptions.consume' in out + assert "pubsub.subscriptions.consume" in out diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index 490c903b2..d227baab9 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -53,7 +53,7 @@ def create_topic(project_id, topic_name): topic = publisher.create_topic(topic_path) - print('Topic created: {}'.format(topic)) + print("Topic created: {}".format(topic)) # [END pubsub_quickstart_create_topic] # [END pubsub_create_topic] @@ -71,7 +71,7 @@ def delete_topic(project_id, topic_name): publisher.delete_topic(topic_path) - print('Topic deleted: {}'.format(topic_path)) + print("Topic deleted: {}".format(topic_path)) # [END pubsub_delete_topic] @@ -90,14 +90,14 @@ def publish_messages(project_id, topic_name): topic_path = publisher.topic_path(project_id, topic_name) for n in range(1, 10): - data = u'Message number {}'.format(n) + data = u"Message number {}".format(n) # Data must be a bytestring - data = data.encode('utf-8') + data = data.encode("utf-8") # When you publish a message, the client returns a future. future = publisher.publish(topic_path, data=data) print(future.result()) - print('Published messages.') + print("Published messages.") # [END pubsub_quickstart_publisher] # [END pubsub_publish] @@ -115,16 +115,16 @@ def publish_messages_with_custom_attributes(project_id, topic_name): topic_path = publisher.topic_path(project_id, topic_name) for n in range(1, 10): - data = u'Message number {}'.format(n) + data = u"Message number {}".format(n) # Data must be a bytestring - data = data.encode('utf-8') + data = data.encode("utf-8") # Add two attributes, origin and username, to the message future = publisher.publish( - topic_path, data, origin='python-sample', username='gcp' + topic_path, data, origin="python-sample", username="gcp" ) print(future.result()) - print('Published messages with custom attributes.') + print("Published messages with custom attributes.") # [END pubsub_publish_custom_attributes] @@ -141,14 +141,14 @@ def publish_messages_with_futures(project_id, topic_name): topic_path = publisher.topic_path(project_id, topic_name) for n in range(1, 10): - data = u'Message number {}'.format(n) + data = u"Message number {}".format(n) # Data must be a bytestring - data = data.encode('utf-8') + data = data.encode("utf-8") # When you publish a message, the client returns a future. future = publisher.publish(topic_path, data=data) print(future.result()) - print('Published messages with futures.') + print("Published messages with futures.") # [END pubsub_publisher_concurrency_control] @@ -173,7 +173,7 @@ def callback(f): print(f.result()) futures.pop(data) except: # noqa - print('Please handle {} for {}.'.format(f.exception(), data)) + print("Please handle {} for {}.".format(f.exception(), data)) return callback @@ -182,7 +182,7 @@ def callback(f): futures.update({data: None}) # When you publish a message, the client returns a future. future = publisher.publish( - topic_path, data=data.encode('utf-8') # data must be a bytestring. + topic_path, data=data.encode("utf-8") # data must be a bytestring. ) futures[data] = future # Publish failures shall be handled in the callback function. @@ -192,7 +192,7 @@ def callback(f): while futures: time.sleep(5) - print('Published message with error handler.') + print("Published message with error handler.") # [END pubsub_publish_messages_error_handler] @@ -207,20 +207,19 @@ def publish_messages_with_batch_settings(project_id, topic_name): # Configure the batch to publish as soon as there is one kilobyte # of data or one second has passed. batch_settings = pubsub_v1.types.BatchSettings( - max_bytes=1024, # One kilobyte - max_latency=1, # One second + max_bytes=1024, max_latency=1 # One kilobyte # One second ) publisher = pubsub_v1.PublisherClient(batch_settings) topic_path = publisher.topic_path(project_id, topic_name) for n in range(1, 10): - data = u'Message number {}'.format(n) + data = u"Message number {}".format(n) # Data must be a bytestring - data = data.encode('utf-8') + data = data.encode("utf-8") future = publisher.publish(topic_path, data=data) print(future.result()) - print('Published messages with batch settings.') + print("Published messages with batch settings.") # [END pubsub_publisher_batch_settings] @@ -234,34 +233,34 @@ def publish_messages_with_retry_settings(project_id, topic_name): # Configure the retry settings. Defaults will be overwritten. retry_settings = { - 'interfaces': { - 'google.pubsub.v1.Publisher': { - 'retry_codes': { - 'publish': [ - 'ABORTED', - 'CANCELLED', - 'DEADLINE_EXCEEDED', - 'INTERNAL', - 'RESOURCE_EXHAUSTED', - 'UNAVAILABLE', - 'UNKNOWN', + "interfaces": { + "google.pubsub.v1.Publisher": { + "retry_codes": { + "publish": [ + "ABORTED", + "CANCELLED", + "DEADLINE_EXCEEDED", + "INTERNAL", + "RESOURCE_EXHAUSTED", + "UNAVAILABLE", + "UNKNOWN", ] }, - 'retry_params': { - 'messaging': { - 'initial_retry_delay_millis': 100, # default: 100 - 'retry_delay_multiplier': 1.3, # default: 1.3 - 'max_retry_delay_millis': 60000, # default: 60000 - 'initial_rpc_timeout_millis': 5000, # default: 25000 - 'rpc_timeout_multiplier': 1.0, # default: 1.0 - 'max_rpc_timeout_millis': 600000, # default: 30000 - 'total_timeout_millis': 600000, # default: 600000 + "retry_params": { + "messaging": { + "initial_retry_delay_millis": 100, # default: 100 + "retry_delay_multiplier": 1.3, # default: 1.3 + "max_retry_delay_millis": 60000, # default: 60000 + "initial_rpc_timeout_millis": 5000, # default: 25000 + "rpc_timeout_multiplier": 1.0, # default: 1.0 + "max_rpc_timeout_millis": 600000, # default: 30000 + "total_timeout_millis": 600000, # default: 600000 } }, - 'methods': { - 'Publish': { - 'retry_codes_name': 'publish', - 'retry_params_name': 'messaging', + "methods": { + "Publish": { + "retry_codes_name": "publish", + "retry_params_name": "messaging", } }, } @@ -272,85 +271,85 @@ def publish_messages_with_retry_settings(project_id, topic_name): topic_path = publisher.topic_path(project_id, topic_name) for n in range(1, 10): - data = u'Message number {}'.format(n) + data = u"Message number {}".format(n) # Data must be a bytestring - data = data.encode('utf-8') + data = data.encode("utf-8") future = publisher.publish(topic_path, data=data) print(future.result()) - print('Published messages with retry settings.') + print("Published messages with retry settings.") # [END pubsub_publisher_retry_settings] if __name__ == "__main__": parser = argparse.ArgumentParser( description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + formatter_class=argparse.RawDescriptionHelpFormatter, ) - parser.add_argument('project_id', help='Your Google Cloud project ID') + parser.add_argument("project_id", help="Your Google Cloud project ID") - subparsers = parser.add_subparsers(dest='command') - subparsers.add_parser('list', help=list_topics.__doc__) + subparsers = parser.add_subparsers(dest="command") + subparsers.add_parser("list", help=list_topics.__doc__) - create_parser = subparsers.add_parser('create', - help=create_topic.__doc__) - create_parser.add_argument('topic_name') + create_parser = subparsers.add_parser("create", help=create_topic.__doc__) + create_parser.add_argument("topic_name") - delete_parser = subparsers.add_parser('delete', - help=delete_topic.__doc__) - delete_parser.add_argument('topic_name') + delete_parser = subparsers.add_parser("delete", help=delete_topic.__doc__) + delete_parser.add_argument("topic_name") - publish_parser = subparsers.add_parser('publish', - help=publish_messages.__doc__) - publish_parser.add_argument('topic_name') + publish_parser = subparsers.add_parser( + "publish", help=publish_messages.__doc__ + ) + publish_parser.add_argument("topic_name") publish_with_custom_attributes_parser = subparsers.add_parser( - 'publish-with-custom-attributes', + "publish-with-custom-attributes", help=publish_messages_with_custom_attributes.__doc__, ) - publish_with_custom_attributes_parser.add_argument('topic_name') + publish_with_custom_attributes_parser.add_argument("topic_name") publish_with_futures_parser = subparsers.add_parser( - 'publish-with-futures', help=publish_messages_with_futures.__doc__ + "publish-with-futures", help=publish_messages_with_futures.__doc__ ) - publish_with_futures_parser.add_argument('topic_name') + publish_with_futures_parser.add_argument("topic_name") publish_with_error_handler_parser = subparsers.add_parser( - 'publish-with-error-handler', - help=publish_messages_with_error_handler.__doc__ + "publish-with-error-handler", + help=publish_messages_with_error_handler.__doc__, ) - publish_with_error_handler_parser.add_argument('topic_name') + publish_with_error_handler_parser.add_argument("topic_name") publish_with_batch_settings_parser = subparsers.add_parser( - 'publish-with-batch-settings', - help=publish_messages_with_batch_settings.__doc__ + "publish-with-batch-settings", + help=publish_messages_with_batch_settings.__doc__, ) - publish_with_batch_settings_parser.add_argument('topic_name') + publish_with_batch_settings_parser.add_argument("topic_name") publish_with_retry_settings_parser = subparsers.add_parser( - 'publish-with-retry-settings', - help=publish_messages_with_retry_settings.__doc__ + "publish-with-retry-settings", + help=publish_messages_with_retry_settings.__doc__, ) - publish_with_retry_settings_parser.add_argument('topic_name') + publish_with_retry_settings_parser.add_argument("topic_name") args = parser.parse_args() - if args.command == 'list': + if args.command == "list": list_topics(args.project_id) - elif args.command == 'create': + elif args.command == "create": create_topic(args.project_id, args.topic_name) - elif args.command == 'delete': + elif args.command == "delete": delete_topic(args.project_id, args.topic_name) - elif args.command == 'publish': + elif args.command == "publish": publish_messages(args.project_id, args.topic_name) - elif args.command == 'publish-with-custom-attributes': - publish_messages_with_custom_attributes(args.project_id, - args.topic_name) - elif args.command == 'publish-with-futures': + elif args.command == "publish-with-custom-attributes": + publish_messages_with_custom_attributes( + args.project_id, args.topic_name + ) + elif args.command == "publish-with-futures": publish_messages_with_futures(args.project_id, args.topic_name) - elif args.command == 'publish-with-error-handler': + elif args.command == "publish-with-error-handler": publish_messages_with_error_handler(args.project_id, args.topic_name) - elif args.command == 'publish-with-batch-settings': + elif args.command == "publish-with-batch-settings": publish_messages_with_batch_settings(args.project_id, args.topic_name) - elif args.command == 'publish-with-retry-settings': + elif args.command == "publish-with-retry-settings": publish_messages_with_retry_settings(args.project_id, args.topic_name) diff --git a/samples/snippets/publisher_test.py b/samples/snippets/publisher_test.py index 5e550abd6..125fae3c0 100644 --- a/samples/snippets/publisher_test.py +++ b/samples/snippets/publisher_test.py @@ -14,6 +14,7 @@ import os import time +import uuid from gcp_devrel.testing import eventually_consistent from google.cloud import pubsub_v1 @@ -22,8 +23,9 @@ import publisher -PROJECT = os.environ['GCLOUD_PROJECT'] -TOPIC = 'publisher-test-topic' +UUID = uuid.uuid4().hex +PROJECT = os.environ["GCLOUD_PROJECT"] +TOPIC = "publisher-test-topic-" + UUID @pytest.fixture @@ -49,11 +51,18 @@ def _make_sleep_patch(): def new_sleep(period): if period == 60: real_sleep(5) - raise RuntimeError('sigil') + raise RuntimeError("sigil") else: real_sleep(period) - return mock.patch('time.sleep', new=new_sleep) + return mock.patch("time.sleep", new=new_sleep) + + +def _to_delete(): + publisher_client = pubsub_v1.PublisherClient() + publisher_client.delete_topic( + "projects/{}/topics/{}".format(PROJECT, TOPIC) + ) def test_list(client, topic, capsys): @@ -91,39 +100,41 @@ def test_publish(topic, capsys): publisher.publish_messages(PROJECT, TOPIC) out, _ = capsys.readouterr() - assert 'Published' in out + assert "Published" in out def test_publish_with_custom_attributes(topic, capsys): publisher.publish_messages_with_custom_attributes(PROJECT, TOPIC) out, _ = capsys.readouterr() - assert 'Published' in out + assert "Published" in out def test_publish_with_batch_settings(topic, capsys): publisher.publish_messages_with_batch_settings(PROJECT, TOPIC) out, _ = capsys.readouterr() - assert 'Published' in out + assert "Published" in out def test_publish_with_retry_settings(topic, capsys): publisher.publish_messages_with_retry_settings(PROJECT, TOPIC) out, _ = capsys.readouterr() - assert 'Published' in out + assert "Published" in out def test_publish_with_error_handler(topic, capsys): publisher.publish_messages_with_error_handler(PROJECT, TOPIC) out, _ = capsys.readouterr() - assert 'Published' in out + assert "Published" in out def test_publish_with_futures(topic, capsys): publisher.publish_messages_with_futures(PROJECT, TOPIC) out, _ = capsys.readouterr() - assert 'Published' in out + assert "Published" in out + + _to_delete() diff --git a/samples/snippets/quickstart.py b/samples/snippets/quickstart.py index f48d085e0..d01105885 100644 --- a/samples/snippets/quickstart.py +++ b/samples/snippets/quickstart.py @@ -39,34 +39,36 @@ def end_to_end(project_id, topic_name, subscription_name, num_messages): # The `subscription_path` method creates a fully qualified identifier # in the form `projects/{project_id}/subscriptions/{subscription_name}` subscription_path = subscriber.subscription_path( - project_id, subscription_name) + project_id, subscription_name + ) # Create the topic. topic = publisher.create_topic(topic_path) - print('\nTopic created: {}'.format(topic.name)) + print("\nTopic created: {}".format(topic.name)) # Create a subscription. subscription = subscriber.create_subscription( - subscription_path, topic_path) - print('\nSubscription created: {}\n'.format(subscription.name)) + subscription_path, topic_path + ) + print("\nSubscription created: {}\n".format(subscription.name)) publish_begin = time.time() # Publish messages. for n in range(num_messages): - data = u'Message number {}'.format(n) + data = u"Message number {}".format(n) # Data must be a bytestring - data = data.encode('utf-8') + data = data.encode("utf-8") # When you publish a message, the client returns a future. future = publisher.publish(topic_path, data=data) - print('Published {} of message ID {}.'.format(data, future.result())) + print("Published {} of message ID {}.".format(data, future.result())) publish_time = time.time() - publish_begin messages = set() def callback(message): - print('Received message: {}'.format(message)) + print("Received message: {}".format(message)) # Unacknowledged messages will be sent again. message.ack() messages.add(message) @@ -76,7 +78,7 @@ def callback(message): # Receive messages. The subscriber is nonblocking. subscriber.subscribe(subscription_path, callback=callback) - print('\nListening for messages on {}...\n'.format(subscription_path)) + print("\nListening for messages on {}...\n".format(subscription_path)) while True: if len(messages) == num_messages: @@ -87,22 +89,23 @@ def callback(message): break else: # Sleeps the thread at 50Hz to save on resources. - time.sleep(1. / 50) + time.sleep(1.0 / 50) # [END pubsub_end_to_end] -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser( description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + formatter_class=argparse.RawDescriptionHelpFormatter, ) - parser.add_argument('project_id', help='Your Google Cloud project ID') - parser.add_argument('topic_name', help='Your topic name') - parser.add_argument('subscription_name', help='Your subscription name') - parser.add_argument('num_msgs', type=int, help='Number of test messages') + parser.add_argument("project_id", help="Your Google Cloud project ID") + parser.add_argument("topic_name", help="Your topic name") + parser.add_argument("subscription_name", help="Your subscription name") + parser.add_argument("num_msgs", type=int, help="Number of test messages") args = parser.parse_args() - end_to_end(args.project_id, args.topic_name, args.subscription_name, - args.num_msgs) + end_to_end( + args.project_id, args.topic_name, args.subscription_name, args.num_msgs + ) diff --git a/samples/snippets/quickstart/pub.py b/samples/snippets/quickstart/pub.py index e340eb4f3..a3f8087ec 100644 --- a/samples/snippets/quickstart/pub.py +++ b/samples/snippets/quickstart/pub.py @@ -17,22 +17,32 @@ # [START pubsub_quickstart_pub_all] import argparse import time + # [START pubsub_quickstart_pub_deps] from google.cloud import pubsub_v1 + # [END pubsub_quickstart_pub_deps] def get_callback(api_future, data, ref): """Wrap message data in the context of the callback function.""" + def callback(api_future): try: - print("Published message {} now has message ID {}".format( - data, api_future.result())) + print( + "Published message {} now has message ID {}".format( + data, api_future.result() + ) + ) ref["num_messages"] += 1 except Exception: - print("A problem occurred when publishing {}: {}\n".format( - data, api_future.exception())) + print( + "A problem occurred when publishing {}: {}\n".format( + data, api_future.exception() + ) + ) raise + return callback @@ -63,13 +73,13 @@ def pub(project_id, topic_name): print("Published {} message(s).".format(ref["num_messages"])) -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser( description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + formatter_class=argparse.RawDescriptionHelpFormatter, ) - parser.add_argument('project_id', help='Google Cloud project ID') - parser.add_argument('topic_name', help='Pub/Sub topic name') + parser.add_argument("project_id", help="Google Cloud project ID") + parser.add_argument("topic_name", help="Pub/Sub topic name") args = parser.parse_args() diff --git a/samples/snippets/quickstart/pub_test.py b/samples/snippets/quickstart/pub_test.py index 09443364a..b9a6f807f 100644 --- a/samples/snippets/quickstart/pub_test.py +++ b/samples/snippets/quickstart/pub_test.py @@ -16,22 +16,24 @@ import os import pytest +import uuid from google.api_core.exceptions import AlreadyExists from google.cloud import pubsub_v1 import pub -PROJECT = os.environ['GCLOUD_PROJECT'] -TOPIC = 'quickstart-pub-test-topic' +UUID = uuid.uuid4().hex +PROJECT = os.environ["GCLOUD_PROJECT"] +TOPIC = "quickstart-pub-test-topic-" + UUID -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def publisher_client(): yield pubsub_v1.PublisherClient() -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def topic(publisher_client): topic_path = publisher_client.topic_path(PROJECT, TOPIC) @@ -42,20 +44,12 @@ def topic(publisher_client): yield TOPIC + publisher_client.delete_topic(topic_path) -@pytest.fixture -def to_delete(publisher_client): - doomed = [] - yield doomed - for item in doomed: - publisher_client.delete_topic(item) - -def test_pub(publisher_client, topic, to_delete, capsys): +def test_pub(publisher_client, topic, capsys): pub.pub(PROJECT, topic) - to_delete.append('projects/{}/topics/{}'.format(PROJECT, TOPIC)) - out, _ = capsys.readouterr() - assert "Published message b'Hello, World!'" in out + assert "Hello, World!" in out diff --git a/samples/snippets/quickstart/sub.py b/samples/snippets/quickstart/sub.py index e39f14105..5791af14d 100644 --- a/samples/snippets/quickstart/sub.py +++ b/samples/snippets/quickstart/sub.py @@ -16,8 +16,10 @@ # [START pubsub_quickstart_sub_all] import argparse + # [START pubsub_quickstart_sub_deps] from google.cloud import pubsub_v1 + # [END pubsub_quickstart_sub_deps] @@ -29,19 +31,22 @@ def sub(project_id, subscription_name): # [END pubsub_quickstart_sub_client] # Create a fully qualified identifier in the form of # `projects/{project_id}/subscriptions/{subscription_name}` - subscription_path = client.subscription_path( - project_id, subscription_name) + subscription_path = client.subscription_path(project_id, subscription_name) def callback(message): - print('Received message {} of message ID {}\n'.format( - message, message.message_id)) + print( + "Received message {} of message ID {}\n".format( + message, message.message_id + ) + ) # Acknowledge the message. Unack'ed messages will be redelivered. message.ack() - print('Acknowledged message {}\n'.format(message.message_id)) + print("Acknowledged message {}\n".format(message.message_id)) streaming_pull_future = client.subscribe( - subscription_path, callback=callback) - print('Listening for messages on {}..\n'.format(subscription_path)) + subscription_path, callback=callback + ) + print("Listening for messages on {}..\n".format(subscription_path)) # Calling result() on StreamingPullFuture keeps the main thread from # exiting while messages get processed in the callbacks. @@ -51,13 +56,13 @@ def callback(message): streaming_pull_future.cancel() -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser( description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + formatter_class=argparse.RawDescriptionHelpFormatter, ) - parser.add_argument('project_id', help='Google Cloud project ID') - parser.add_argument('subscription_name', help='Pub/Sub subscription name') + parser.add_argument("project_id", help="Google Cloud project ID") + parser.add_argument("subscription_name", help="Pub/Sub subscription name") args = parser.parse_args() diff --git a/samples/snippets/quickstart/sub_test.py b/samples/snippets/quickstart/sub_test.py index 476139a02..07edfad7c 100644 --- a/samples/snippets/quickstart/sub_test.py +++ b/samples/snippets/quickstart/sub_test.py @@ -14,8 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import mock import os import pytest +import uuid from google.api_core.exceptions import AlreadyExists from google.cloud import pubsub_v1 @@ -23,84 +25,80 @@ import sub -PROJECT = os.environ['GCLOUD_PROJECT'] -TOPIC = 'quickstart-sub-test-topic' -SUBSCRIPTION = 'quickstart-sub-test-topic-sub' +UUID = uuid.uuid4().hex +PROJECT = os.environ["GCLOUD_PROJECT"] +TOPIC = "quickstart-sub-test-topic-" + UUID +SUBSCRIPTION = "quickstart-sub-test-topic-sub-" + UUID publisher_client = pubsub_v1.PublisherClient() subscriber_client = pubsub_v1.SubscriberClient() -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def topic_path(): topic_path = publisher_client.topic_path(PROJECT, TOPIC) try: topic = publisher_client.create_topic(topic_path) - return topic.name + yield topic.name except AlreadyExists: - return topic_path + yield topic_path + publisher_client.delete_topic(topic_path) -@pytest.fixture(scope='module') + +@pytest.fixture(scope="module") def subscription_path(topic_path): subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION) + PROJECT, SUBSCRIPTION + ) try: subscription = subscriber_client.create_subscription( - subscription_path, topic_path) - return subscription.name + subscription_path, topic_path + ) + yield subscription.name except AlreadyExists: - return subscription_path - + yield subscription_path -def _to_delete(resource_paths): - for item in resource_paths: - if 'topics' in item: - publisher_client.delete_topic(item) - if 'subscriptions' in item: - subscriber_client.delete_subscription(item) + subscriber_client.delete_subscription(subscription_path) def _publish_messages(topic_path): - publish_future = publisher_client.publish(topic_path, data=b'Hello World!') + publish_future = publisher_client.publish(topic_path, data=b"Hello World!") publish_future.result() -def _sub_timeout(project_id, subscription_name): - # This is an exactly copy of `sub.py` except - # StreamingPullFuture.result() will time out after 10s. - client = pubsub_v1.SubscriberClient() - subscription_path = client.subscription_path( - project_id, subscription_name) +def test_sub(monkeypatch, topic_path, subscription_path, capsys): - def callback(message): - print('Received message {} of message ID {}\n'.format( - message, message.message_id)) - message.ack() - print('Acknowledged message {}\n'.format(message.message_id)) + real_client = pubsub_v1.SubscriberClient() + mock_client = mock.Mock(spec=pubsub_v1.SubscriberClient, wraps=real_client) - streaming_pull_future = client.subscribe( - subscription_path, callback=callback) - print('Listening for messages on {}..\n'.format(subscription_path)) + # Attributes on mock_client_constructor uses the corresponding + # attributes on pubsub_v1.SubscriberClient. + mock_client_constructor = mock.create_autospec(pubsub_v1.SubscriberClient) + mock_client_constructor.return_value = mock_client - try: - streaming_pull_future.result(timeout=10) - except: # noqa - streaming_pull_future.cancel() + monkeypatch.setattr(pubsub_v1, "SubscriberClient", mock_client_constructor) + def mock_subscribe(subscription_path, callback=None): + real_future = real_client.subscribe( + subscription_path, callback=callback + ) + mock_future = mock.Mock(spec=real_future, wraps=real_future) -def test_sub(monkeypatch, topic_path, subscription_path, capsys): - monkeypatch.setattr(sub, 'sub', _sub_timeout) + def mock_result(): + return real_future.result(timeout=10) + + mock_future.result.side_effect = mock_result + return mock_future + + mock_client.subscribe.side_effect = mock_subscribe _publish_messages(topic_path) sub.sub(PROJECT, SUBSCRIPTION) - # Clean up resources. - _to_delete([topic_path, subscription_path]) - out, _ = capsys.readouterr() assert "Received message" in out assert "Acknowledged message" in out diff --git a/samples/snippets/quickstart_test.py b/samples/snippets/quickstart_test.py index d318b260c..6a1d4aae1 100644 --- a/samples/snippets/quickstart_test.py +++ b/samples/snippets/quickstart_test.py @@ -15,23 +15,25 @@ # limitations under the License. import os +import uuid from google.cloud import pubsub_v1 import pytest import quickstart -PROJECT = os.environ['GCLOUD_PROJECT'] -TOPIC = 'end-to-end-test-topic' -SUBSCRIPTION = 'end-to-end-test-topic-sub' +UUID = uuid.uuid4().hex +PROJECT = os.environ["GCLOUD_PROJECT"] +TOPIC = "end-to-end-test-topic-" + UUID +SUBSCRIPTION = "end-to-end-test-topic-sub-" + UUID N = 10 -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def publisher_client(): yield pubsub_v1.PublisherClient() -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def topic(publisher_client): topic_path = publisher_client.topic_path(PROJECT, TOPIC) @@ -42,16 +44,19 @@ def topic(publisher_client): yield TOPIC + publisher_client.delete_topic(topic_path) -@pytest.fixture(scope='module') + +@pytest.fixture(scope="module") def subscriber_client(): yield pubsub_v1.SubscriberClient() -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def subscription(subscriber_client, topic): subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION) + PROJECT, SUBSCRIPTION + ) try: subscriber_client.delete_subscription(subscription_path) @@ -60,6 +65,8 @@ def subscription(subscriber_client, topic): yield SUBSCRIPTION + subscriber_client.delete_subscription(subscription_path) + def test_end_to_end(topic, subscription, capsys): diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 3bbad0ead..ea1cc9ff9 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -67,19 +67,20 @@ def create_subscription(project_id, topic_name, subscription_name): subscriber = pubsub_v1.SubscriberClient() topic_path = subscriber.topic_path(project_id, topic_name) subscription_path = subscriber.subscription_path( - project_id, subscription_name) + project_id, subscription_name + ) subscription = subscriber.create_subscription( - subscription_path, topic_path) + subscription_path, topic_path + ) - print('Subscription created: {}'.format(subscription)) + print("Subscription created: {}".format(subscription)) # [END pubsub_create_pull_subscription] -def create_push_subscription(project_id, - topic_name, - subscription_name, - endpoint): +def create_push_subscription( + project_id, topic_name, subscription_name, endpoint +): """Create a new push subscription on the given topic.""" # [START pubsub_create_push_subscription] from google.cloud import pubsub_v1 @@ -92,16 +93,17 @@ def create_push_subscription(project_id, subscriber = pubsub_v1.SubscriberClient() topic_path = subscriber.topic_path(project_id, topic_name) subscription_path = subscriber.subscription_path( - project_id, subscription_name) + project_id, subscription_name + ) - push_config = pubsub_v1.types.PushConfig( - push_endpoint=endpoint) + push_config = pubsub_v1.types.PushConfig(push_endpoint=endpoint) subscription = subscriber.create_subscription( - subscription_path, topic_path, push_config) + subscription_path, topic_path, push_config + ) - print('Push subscription created: {}'.format(subscription)) - print('Endpoint for subscription is: {}'.format(endpoint)) + print("Push subscription created: {}".format(subscription)) + print("Endpoint for subscription is: {}".format(endpoint)) # [END pubsub_create_push_subscription] @@ -115,11 +117,12 @@ def delete_subscription(project_id, subscription_name): subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( - project_id, subscription_name) + project_id, subscription_name + ) subscriber.delete_subscription(subscription_path) - print('Subscription deleted: {}'.format(subscription_path)) + print("Subscription deleted: {}".format(subscription_path)) # [END pubsub_delete_subscription] @@ -139,27 +142,22 @@ def update_subscription(project_id, subscription_name, endpoint): subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( - project_id, subscription_name) + project_id, subscription_name + ) - push_config = pubsub_v1.types.PushConfig( - push_endpoint=endpoint) + push_config = pubsub_v1.types.PushConfig(push_endpoint=endpoint) subscription = pubsub_v1.types.Subscription( - name=subscription_path, - push_config=push_config) + name=subscription_path, push_config=push_config + ) - update_mask = { - 'paths': { - 'push_config', - } - } + update_mask = {"paths": {"push_config"}} subscriber.update_subscription(subscription, update_mask) result = subscriber.get_subscription(subscription_path) - print('Subscription updated: {}'.format(subscription_path)) - print('New endpoint for subscription is: {}'.format( - result.push_config)) + print("Subscription updated: {}".format(subscription_path)) + print("New endpoint for subscription is: {}".format(result.push_config)) # [END pubsub_update_push_configuration] @@ -178,17 +176,18 @@ def receive_messages(project_id, subscription_name): # The `subscription_path` method creates a fully qualified identifier # in the form `projects/{project_id}/subscriptions/{subscription_name}` subscription_path = subscriber.subscription_path( - project_id, subscription_name) + project_id, subscription_name + ) def callback(message): - print('Received message: {}'.format(message)) + print("Received message: {}".format(message)) message.ack() subscriber.subscribe(subscription_path, callback=callback) # The subscriber is non-blocking. We must keep the main thread from # exiting to allow it to process messages asynchronously in the background. - print('Listening for messages on {}'.format(subscription_path)) + print("Listening for messages on {}".format(subscription_path)) while True: time.sleep(60) # [END pubsub_subscriber_async_pull] @@ -208,22 +207,23 @@ def receive_messages_with_custom_attributes(project_id, subscription_name): subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( - project_id, subscription_name) + project_id, subscription_name + ) def callback(message): - print('Received message: {}'.format(message.data)) + print("Received message: {}".format(message.data)) if message.attributes: - print('Attributes:') + print("Attributes:") for key in message.attributes: value = message.attributes.get(key) - print('{}: {}'.format(key, value)) + print("{}: {}".format(key, value)) message.ack() subscriber.subscribe(subscription_path, callback=callback) # The subscriber is non-blocking, so we must keep the main thread from # exiting to allow it to process messages in the background. - print('Listening for messages on {}'.format(subscription_path)) + print("Listening for messages on {}".format(subscription_path)) while True: time.sleep(60) # [END pubsub_subscriber_async_pull_custom_attributes] @@ -242,20 +242,22 @@ def receive_messages_with_flow_control(project_id, subscription_name): subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( - project_id, subscription_name) + project_id, subscription_name + ) def callback(message): - print('Received message: {}'.format(message.data)) + print("Received message: {}".format(message.data)) message.ack() # Limit the subscriber to only have ten outstanding messages at a time. flow_control = pubsub_v1.types.FlowControl(max_messages=10) subscriber.subscribe( - subscription_path, callback=callback, flow_control=flow_control) + subscription_path, callback=callback, flow_control=flow_control + ) # The subscriber is non-blocking, so we must keep the main thread from # exiting to allow it to process messages in the background. - print('Listening for messages on {}'.format(subscription_path)) + print("Listening for messages on {}".format(subscription_path)) while True: time.sleep(60) # [END pubsub_subscriber_flow_settings] @@ -271,7 +273,8 @@ def synchronous_pull(project_id, subscription_name): subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( - project_id, subscription_name) + project_id, subscription_name + ) NUM_MESSAGES = 3 @@ -286,8 +289,11 @@ def synchronous_pull(project_id, subscription_name): # Acknowledges the received messages so they will not be sent again. subscriber.acknowledge(subscription_path, ack_ids) - print('Received and acknowledged {} messages. Done.'.format( - len(response.received_messages))) + print( + "Received and acknowledged {} messages. Done.".format( + len(response.received_messages) + ) + ) # [END pubsub_subscriber_sync_pull] @@ -306,7 +312,8 @@ def synchronous_pull_with_lease_management(project_id, subscription_name): subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( - project_id, subscription_name) + project_id, subscription_name + ) NUM_MESSAGES = 2 ACK_DEADLINE = 30 @@ -322,8 +329,11 @@ def synchronous_pull_with_lease_management(project_id, subscription_name): def worker(msg): """Simulates a long-running process.""" RUN_TIME = random.randint(1, 60) - logger.info('{}: Running {} for {}s'.format( - time.strftime("%X", time.gmtime()), msg.message.data, RUN_TIME)) + logger.info( + "{}: Running {} for {}s".format( + time.strftime("%X", time.gmtime()), msg.message.data, RUN_TIME + ) + ) time.sleep(RUN_TIME) # `processes` stores process as key and ack id and message as values. @@ -344,24 +354,35 @@ def worker(msg): subscriber.modify_ack_deadline( subscription_path, [ack_id], - ack_deadline_seconds=ACK_DEADLINE) - logger.info('{}: Reset ack deadline for {} for {}s'.format( - time.strftime("%X", time.gmtime()), - msg_data, ACK_DEADLINE)) + ack_deadline_seconds=ACK_DEADLINE, + ) + logger.info( + "{}: Reset ack deadline for {} for {}s".format( + time.strftime("%X", time.gmtime()), + msg_data, + ACK_DEADLINE, + ) + ) # If the processs is finished, acknowledges using `ack_id`. else: subscriber.acknowledge(subscription_path, [ack_id]) - logger.info("{}: Acknowledged {}".format( - time.strftime("%X", time.gmtime()), msg_data)) + logger.info( + "{}: Acknowledged {}".format( + time.strftime("%X", time.gmtime()), msg_data + ) + ) processes.pop(process) # If there are still processes running, sleeps the thread. if processes: time.sleep(SLEEP_TIME) - print('Received and acknowledged {} messages. Done.'.format( - len(response.received_messages))) + print( + "Received and acknowledged {} messages. Done.".format( + len(response.received_messages) + ) + ) # [END pubsub_subscriber_sync_pull_with_lease] @@ -375,10 +396,11 @@ def listen_for_errors(project_id, subscription_name): subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( - project_id, subscription_name) + project_id, subscription_name + ) def callback(message): - print('Received message: {}'.format(message)) + print("Received message: {}".format(message)) message.ack() future = subscriber.subscribe(subscription_path, callback=callback) @@ -390,109 +412,126 @@ def callback(message): future.result(timeout=30) except Exception as e: print( - 'Listening for messages on {} threw an Exception: {}.'.format( - subscription_name, e)) + "Listening for messages on {} threw an Exception: {}.".format( + subscription_name, e + ) + ) # [END pubsub_subscriber_error_listener] -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser( description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + formatter_class=argparse.RawDescriptionHelpFormatter, ) - parser.add_argument('project_id', help='Your Google Cloud project ID') + parser.add_argument("project_id", help="Your Google Cloud project ID") - subparsers = parser.add_subparsers(dest='command') + subparsers = parser.add_subparsers(dest="command") list_in_topic_parser = subparsers.add_parser( - 'list_in_topic', help=list_subscriptions_in_topic.__doc__) - list_in_topic_parser.add_argument('topic_name') + "list_in_topic", help=list_subscriptions_in_topic.__doc__ + ) + list_in_topic_parser.add_argument("topic_name") list_in_project_parser = subparsers.add_parser( - 'list_in_project', help=list_subscriptions_in_project.__doc__) + "list_in_project", help=list_subscriptions_in_project.__doc__ + ) create_parser = subparsers.add_parser( - 'create', help=create_subscription.__doc__) - create_parser.add_argument('topic_name') - create_parser.add_argument('subscription_name') + "create", help=create_subscription.__doc__ + ) + create_parser.add_argument("topic_name") + create_parser.add_argument("subscription_name") create_push_parser = subparsers.add_parser( - 'create-push', help=create_push_subscription.__doc__) - create_push_parser.add_argument('topic_name') - create_push_parser.add_argument('subscription_name') - create_push_parser.add_argument('endpoint') + "create-push", help=create_push_subscription.__doc__ + ) + create_push_parser.add_argument("topic_name") + create_push_parser.add_argument("subscription_name") + create_push_parser.add_argument("endpoint") delete_parser = subparsers.add_parser( - 'delete', help=delete_subscription.__doc__) - delete_parser.add_argument('subscription_name') + "delete", help=delete_subscription.__doc__ + ) + delete_parser.add_argument("subscription_name") update_parser = subparsers.add_parser( - 'update', help=update_subscription.__doc__) - update_parser.add_argument('subscription_name') - update_parser.add_argument('endpoint') + "update", help=update_subscription.__doc__ + ) + update_parser.add_argument("subscription_name") + update_parser.add_argument("endpoint") receive_parser = subparsers.add_parser( - 'receive', help=receive_messages.__doc__) - receive_parser.add_argument('subscription_name') + "receive", help=receive_messages.__doc__ + ) + receive_parser.add_argument("subscription_name") receive_with_custom_attributes_parser = subparsers.add_parser( - 'receive-custom-attributes', - help=receive_messages_with_custom_attributes.__doc__) - receive_with_custom_attributes_parser.add_argument('subscription_name') + "receive-custom-attributes", + help=receive_messages_with_custom_attributes.__doc__, + ) + receive_with_custom_attributes_parser.add_argument("subscription_name") receive_with_flow_control_parser = subparsers.add_parser( - 'receive-flow-control', - help=receive_messages_with_flow_control.__doc__) - receive_with_flow_control_parser.add_argument('subscription_name') + "receive-flow-control", help=receive_messages_with_flow_control.__doc__ + ) + receive_with_flow_control_parser.add_argument("subscription_name") synchronous_pull_parser = subparsers.add_parser( - 'receive-synchronously', - help=synchronous_pull.__doc__) - synchronous_pull_parser.add_argument('subscription_name') + "receive-synchronously", help=synchronous_pull.__doc__ + ) + synchronous_pull_parser.add_argument("subscription_name") synchronous_pull_with_lease_management_parser = subparsers.add_parser( - 'receive-synchronously-with-lease', - help=synchronous_pull_with_lease_management.__doc__) + "receive-synchronously-with-lease", + help=synchronous_pull_with_lease_management.__doc__, + ) synchronous_pull_with_lease_management_parser.add_argument( - 'subscription_name') + "subscription_name" + ) listen_for_errors_parser = subparsers.add_parser( - 'listen_for_errors', help=listen_for_errors.__doc__) - listen_for_errors_parser.add_argument('subscription_name') + "listen_for_errors", help=listen_for_errors.__doc__ + ) + listen_for_errors_parser.add_argument("subscription_name") args = parser.parse_args() - if args.command == 'list_in_topic': + if args.command == "list_in_topic": list_subscriptions_in_topic(args.project_id, args.topic_name) - elif args.command == 'list_in_project': + elif args.command == "list_in_project": list_subscriptions_in_project(args.project_id) - elif args.command == 'create': + elif args.command == "create": create_subscription( - args.project_id, args.topic_name, args.subscription_name) - elif args.command == 'create-push': + args.project_id, args.topic_name, args.subscription_name + ) + elif args.command == "create-push": create_push_subscription( args.project_id, args.topic_name, args.subscription_name, - args.endpoint) - elif args.command == 'delete': - delete_subscription( - args.project_id, args.subscription_name) - elif args.command == 'update': + args.endpoint, + ) + elif args.command == "delete": + delete_subscription(args.project_id, args.subscription_name) + elif args.command == "update": update_subscription( - args.project_id, args.subscription_name, args.endpoint) - elif args.command == 'receive': + args.project_id, args.subscription_name, args.endpoint + ) + elif args.command == "receive": receive_messages(args.project_id, args.subscription_name) - elif args.command == 'receive-custom-attributes': + elif args.command == "receive-custom-attributes": receive_messages_with_custom_attributes( - args.project_id, args.subscription_name) - elif args.command == 'receive-flow-control': + args.project_id, args.subscription_name + ) + elif args.command == "receive-flow-control": receive_messages_with_flow_control( - args.project_id, args.subscription_name) - elif args.command == 'receive-synchronously': - synchronous_pull( - args.project_id, args.subscription_name) - elif args.command == 'receive-synchronously-with-lease': + args.project_id, args.subscription_name + ) + elif args.command == "receive-synchronously": + synchronous_pull(args.project_id, args.subscription_name) + elif args.command == "receive-synchronously-with-lease": synchronous_pull_with_lease_management( - args.project_id, args.subscription_name) - elif args.command == 'listen_for_errors': + args.project_id, args.subscription_name + ) + elif args.command == "listen_for_errors": listen_for_errors(args.project_id, args.subscription_name) diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index 4c5fd6122..0645c0738 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -14,6 +14,7 @@ import os import time +import uuid from gcp_devrel.testing import eventually_consistent from google.cloud import pubsub_v1 @@ -22,21 +23,22 @@ import subscriber -PROJECT = os.environ['GCLOUD_PROJECT'] -TOPIC = 'subscription-test-topic' -SUBSCRIPTION_ONE = 'subscription-test-subscription-one' -SUBSCRIPTION_TWO = 'subscription-test-subscription-two' -SUBSCRIPTION_THREE = 'subscription-test-subscription-three' -ENDPOINT = 'https://{}.appspot.com/push'.format(PROJECT) -NEW_ENDPOINT = 'https://{}.appspot.com/push2'.format(PROJECT) +UUID = uuid.uuid4().hex +PROJECT = os.environ["GCLOUD_PROJECT"] +TOPIC = "subscription-test-topic-" + UUID +SUBSCRIPTION_ONE = "subscription-test-subscription-one-" + UUID +SUBSCRIPTION_TWO = "subscription-test-subscription-two-" + UUID +SUBSCRIPTION_THREE = "subscription-test-subscription-three-" + UUID +ENDPOINT = "https://{}.appspot.com/push".format(PROJECT) +NEW_ENDPOINT = "https://{}.appspot.com/push2".format(PROJECT) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def publisher_client(): yield pubsub_v1.PublisherClient() -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def topic(publisher_client): topic_path = publisher_client.topic_path(PROJECT, TOPIC) @@ -48,49 +50,55 @@ def topic(publisher_client): yield response.name -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def subscriber_client(): yield pubsub_v1.SubscriberClient() -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def subscription_one(subscriber_client, topic): subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_ONE) + PROJECT, SUBSCRIPTION_ONE + ) try: response = subscriber_client.get_subscription(subscription_path) except: # noqa response = subscriber_client.create_subscription( - subscription_path, topic=topic) + subscription_path, topic=topic + ) yield response.name -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def subscription_two(subscriber_client, topic): subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_TWO) + PROJECT, SUBSCRIPTION_TWO + ) try: response = subscriber_client.get_subscription(subscription_path) except: # noqa response = subscriber_client.create_subscription( - subscription_path, topic=topic) + subscription_path, topic=topic + ) yield response.name -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def subscription_three(subscriber_client, topic): subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_THREE) + PROJECT, SUBSCRIPTION_THREE + ) try: response = subscriber_client.get_subscription(subscription_path) except: # noqa response = subscriber_client.create_subscription( - subscription_path, topic=topic) + subscription_path, topic=topic + ) yield response.name @@ -113,7 +121,8 @@ def _(): def test_create(subscriber_client): subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_ONE) + PROJECT, SUBSCRIPTION_ONE + ) try: subscriber_client.delete_subscription(subscription_path) @@ -129,14 +138,16 @@ def _(): def test_create_push(subscriber_client): subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_ONE) + PROJECT, SUBSCRIPTION_ONE + ) try: subscriber_client.delete_subscription(subscription_path) except Exception: pass subscriber.create_push_subscription( - PROJECT, TOPIC, SUBSCRIPTION_ONE, ENDPOINT) + PROJECT, TOPIC, SUBSCRIPTION_ONE, ENDPOINT + ) @eventually_consistent.call def _(): @@ -147,7 +158,7 @@ def test_update(subscriber_client, subscription_one, capsys): subscriber.update_subscription(PROJECT, SUBSCRIPTION_ONE, NEW_ENDPOINT) out, _ = capsys.readouterr() - assert 'Subscription updated' in out + assert "Subscription updated" in out def test_delete(subscriber_client, subscription_one): @@ -161,14 +172,14 @@ def _(): def _publish_messages(publisher_client, topic): for n in range(5): - data = u'Message {}'.format(n).encode('utf-8') + data = u"Message {}".format(n).encode("utf-8") future = publisher_client.publish(topic, data=data) future.result() def _publish_messages_with_custom_attributes(publisher_client, topic): - data = u'Test message'.encode('utf-8') - future = publisher_client.publish(topic, data=data, origin='python-sample') + data = u"Test message".encode("utf-8") + future = publisher_client.publish(topic, data=data, origin="python-sample") future.result() @@ -178,74 +189,100 @@ def _make_sleep_patch(): def new_sleep(period): if period == 60: real_sleep(5) - raise RuntimeError('sigil') + raise RuntimeError("sigil") else: real_sleep(period) - return mock.patch('time.sleep', new=new_sleep) + return mock.patch("time.sleep", new=new_sleep) + + +def _to_delete(): + publisher_client = pubsub_v1.PublisherClient() + subscriber_client = pubsub_v1.SubscriberClient() + resources = [TOPIC, SUBSCRIPTION_TWO, SUBSCRIPTION_THREE] + + for item in resources: + if "subscription-test-topic" in item: + publisher_client.delete_topic( + "projects/{}/topics/{}".format(PROJECT, item) + ) + if "subscription-test-subscription" in item: + subscriber_client.delete_subscription( + "projects/{}/subscriptions/{}".format(PROJECT, item) + ) def test_receive(publisher_client, topic, subscription_two, capsys): _publish_messages(publisher_client, topic) with _make_sleep_patch(): - with pytest.raises(RuntimeError, match='sigil'): + with pytest.raises(RuntimeError, match="sigil"): subscriber.receive_messages(PROJECT, SUBSCRIPTION_TWO) out, _ = capsys.readouterr() - assert 'Listening' in out + assert "Listening" in out assert subscription_two in out - assert 'Message 1' in out + assert "Message" in out def test_receive_with_custom_attributes( - publisher_client, topic, subscription_two, capsys): + publisher_client, topic, subscription_two, capsys +): _publish_messages_with_custom_attributes(publisher_client, topic) with _make_sleep_patch(): - with pytest.raises(RuntimeError, match='sigil'): + with pytest.raises(RuntimeError, match="sigil"): subscriber.receive_messages_with_custom_attributes( - PROJECT, SUBSCRIPTION_TWO) + PROJECT, SUBSCRIPTION_TWO + ) out, _ = capsys.readouterr() - assert 'Test message' in out - assert 'origin' in out - assert 'python-sample' in out + assert "Test message" in out + assert "origin" in out + assert "python-sample" in out def test_receive_with_flow_control( - publisher_client, topic, subscription_two, capsys): + publisher_client, topic, subscription_two, capsys +): _publish_messages(publisher_client, topic) with _make_sleep_patch(): - with pytest.raises(RuntimeError, match='sigil'): + with pytest.raises(RuntimeError, match="sigil"): subscriber.receive_messages_with_flow_control( - PROJECT, SUBSCRIPTION_TWO) + PROJECT, SUBSCRIPTION_TWO + ) out, _ = capsys.readouterr() - assert 'Listening' in out + assert "Listening" in out assert subscription_two in out - assert 'Message 1' in out + assert "Message" in out def test_receive_synchronously( - publisher_client, topic, subscription_three, capsys): + publisher_client, topic, subscription_three, capsys +): _publish_messages(publisher_client, topic) subscriber.synchronous_pull(PROJECT, SUBSCRIPTION_THREE) out, _ = capsys.readouterr() - assert 'Done.' in out + assert "Done." in out def test_receive_synchronously_with_lease( - publisher_client, topic, subscription_three, capsys): + publisher_client, topic, subscription_three, capsys +): _publish_messages(publisher_client, topic) subscriber.synchronous_pull_with_lease_management( - PROJECT, SUBSCRIPTION_THREE) + PROJECT, SUBSCRIPTION_THREE + ) out, _ = capsys.readouterr() - assert 'Done.' in out + assert "Done." in out + + # Clean up resources after all the tests. + _to_delete() From 0120862393539542b8584126c1d54258b6c34ef6 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Thu, 12 Dec 2019 12:22:16 -0800 Subject: [PATCH 079/101] Pub/Sub: remove infinite while loops in subscriber examples [(#2604)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2604) * use result() on streaming pull futures instead of infinite while * remove unused imports --- samples/snippets/publisher_test.py | 75 ++++++------ samples/snippets/subscriber.py | 98 +++++++++------ samples/snippets/subscriber_test.py | 180 ++++++++++++---------------- 3 files changed, 179 insertions(+), 174 deletions(-) diff --git a/samples/snippets/publisher_test.py b/samples/snippets/publisher_test.py index 125fae3c0..fbe30694a 100644 --- a/samples/snippets/publisher_test.py +++ b/samples/snippets/publisher_test.py @@ -25,7 +25,8 @@ UUID = uuid.uuid4().hex PROJECT = os.environ["GCLOUD_PROJECT"] -TOPIC = "publisher-test-topic-" + UUID +TOPIC_ADMIN = "publisher-test-topic-admin-" + UUID +TOPIC_PUBLISH = "publisher-test-topic-publish-" + UUID @pytest.fixture @@ -34,15 +35,30 @@ def client(): @pytest.fixture -def topic(client): - topic_path = client.topic_path(PROJECT, TOPIC) +def topic_admin(client): + topic_path = client.topic_path(PROJECT, TOPIC_ADMIN) try: - response = client.get_topic(topic_path) + topic = client.get_topic(topic_path) except: # noqa - response = client.create_topic(topic_path) + topic = client.create_topic(topic_path) - yield response.name + yield topic.name + # Teardown of `topic_admin` is handled in `test_delete()`. + + +@pytest.fixture +def topic_publish(client): + topic_path = client.topic_path(PROJECT, TOPIC_PUBLISH) + + try: + topic = client.get_topic(topic_path) + except: # noqa + topic = client.create_topic(topic_path) + + yield topic.name + + client.delete_topic(topic.name) def _make_sleep_patch(): @@ -58,83 +74,74 @@ def new_sleep(period): return mock.patch("time.sleep", new=new_sleep) -def _to_delete(): - publisher_client = pubsub_v1.PublisherClient() - publisher_client.delete_topic( - "projects/{}/topics/{}".format(PROJECT, TOPIC) - ) - - -def test_list(client, topic, capsys): +def test_list(client, topic_admin, capsys): @eventually_consistent.call def _(): publisher.list_topics(PROJECT) out, _ = capsys.readouterr() - assert topic in out + assert topic_admin in out def test_create(client): - topic_path = client.topic_path(PROJECT, TOPIC) + topic_path = client.topic_path(PROJECT, TOPIC_ADMIN) try: client.delete_topic(topic_path) except Exception: pass - publisher.create_topic(PROJECT, TOPIC) + publisher.create_topic(PROJECT, TOPIC_ADMIN) @eventually_consistent.call def _(): assert client.get_topic(topic_path) -def test_delete(client, topic): - publisher.delete_topic(PROJECT, TOPIC) +def test_delete(client, topic_admin): + publisher.delete_topic(PROJECT, TOPIC_ADMIN) @eventually_consistent.call def _(): with pytest.raises(Exception): - client.get_topic(client.topic_path(PROJECT, TOPIC)) + client.get_topic(client.topic_path(PROJECT, TOPIC_ADMIN)) -def test_publish(topic, capsys): - publisher.publish_messages(PROJECT, TOPIC) +def test_publish(topic_publish, capsys): + publisher.publish_messages(PROJECT, TOPIC_PUBLISH) out, _ = capsys.readouterr() assert "Published" in out -def test_publish_with_custom_attributes(topic, capsys): - publisher.publish_messages_with_custom_attributes(PROJECT, TOPIC) +def test_publish_with_custom_attributes(topic_publish, capsys): + publisher.publish_messages_with_custom_attributes(PROJECT, TOPIC_PUBLISH) out, _ = capsys.readouterr() assert "Published" in out -def test_publish_with_batch_settings(topic, capsys): - publisher.publish_messages_with_batch_settings(PROJECT, TOPIC) +def test_publish_with_batch_settings(topic_publish, capsys): + publisher.publish_messages_with_batch_settings(PROJECT, TOPIC_PUBLISH) out, _ = capsys.readouterr() assert "Published" in out -def test_publish_with_retry_settings(topic, capsys): - publisher.publish_messages_with_retry_settings(PROJECT, TOPIC) +def test_publish_with_retry_settings(topic_publish, capsys): + publisher.publish_messages_with_retry_settings(PROJECT, TOPIC_PUBLISH) out, _ = capsys.readouterr() assert "Published" in out -def test_publish_with_error_handler(topic, capsys): - publisher.publish_messages_with_error_handler(PROJECT, TOPIC) +def test_publish_with_error_handler(topic_publish, capsys): + publisher.publish_messages_with_error_handler(PROJECT, TOPIC_PUBLISH) out, _ = capsys.readouterr() assert "Published" in out -def test_publish_with_futures(topic, capsys): - publisher.publish_messages_with_futures(PROJECT, TOPIC) +def test_publish_with_futures(topic_publish, capsys): + publisher.publish_messages_with_futures(PROJECT, TOPIC_PUBLISH) out, _ = capsys.readouterr() assert "Published" in out - - _to_delete() diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index ea1cc9ff9..0d328d232 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -161,16 +161,16 @@ def update_subscription(project_id, subscription_name, endpoint): # [END pubsub_update_push_configuration] -def receive_messages(project_id, subscription_name): +def receive_messages(project_id, subscription_name, timeout=None): """Receives messages from a pull subscription.""" # [START pubsub_subscriber_async_pull] # [START pubsub_quickstart_subscriber] - import time - from google.cloud import pubsub_v1 # TODO project_id = "Your Google Cloud Project ID" # TODO subscription_name = "Your Pub/Sub subscription name" + # TODO timeout = 5.0 # "How long the subscriber should listen for + # messages in seconds" subscriber = pubsub_v1.SubscriberClient() # The `subscription_path` method creates a fully qualified identifier @@ -183,27 +183,33 @@ def callback(message): print("Received message: {}".format(message)) message.ack() - subscriber.subscribe(subscription_path, callback=callback) + streaming_pull_future = subscriber.subscribe( + subscription_path, callback=callback + ) + print("Listening for messages on {}..\n".format(subscription_path)) - # The subscriber is non-blocking. We must keep the main thread from - # exiting to allow it to process messages asynchronously in the background. - print("Listening for messages on {}".format(subscription_path)) - while True: - time.sleep(60) + # result() in a future will block indefinitely if `timeout` is not set, + # unless an exception is encountered first. + try: + streaming_pull_future.result(timeout=timeout) + except: # noqa + streaming_pull_future.cancel() # [END pubsub_subscriber_async_pull] # [END pubsub_quickstart_subscriber] -def receive_messages_with_custom_attributes(project_id, subscription_name): +def receive_messages_with_custom_attributes( + project_id, subscription_name, timeout=None +): """Receives messages from a pull subscription.""" # [START pubsub_subscriber_sync_pull_custom_attributes] # [START pubsub_subscriber_async_pull_custom_attributes] - import time - from google.cloud import pubsub_v1 # TODO project_id = "Your Google Cloud Project ID" # TODO subscription_name = "Your Pub/Sub subscription name" + # TODO timeout = 5.0 # "How long the subscriber should listen for + # messages in seconds" subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( @@ -219,26 +225,32 @@ def callback(message): print("{}: {}".format(key, value)) message.ack() - subscriber.subscribe(subscription_path, callback=callback) + streaming_pull_future = subscriber.subscribe( + subscription_path, callback=callback + ) + print("Listening for messages on {}..\n".format(subscription_path)) - # The subscriber is non-blocking, so we must keep the main thread from - # exiting to allow it to process messages in the background. - print("Listening for messages on {}".format(subscription_path)) - while True: - time.sleep(60) + # result() in a future will block indefinitely if `timeout` is not set, + # unless an exception is encountered first. + try: + streaming_pull_future.result(timeout=timeout) + except: # noqa + streaming_pull_future.cancel() # [END pubsub_subscriber_async_pull_custom_attributes] # [END pubsub_subscriber_sync_pull_custom_attributes] -def receive_messages_with_flow_control(project_id, subscription_name): +def receive_messages_with_flow_control( + project_id, subscription_name, timeout=None +): """Receives messages from a pull subscription with flow control.""" # [START pubsub_subscriber_flow_settings] - import time - from google.cloud import pubsub_v1 # TODO project_id = "Your Google Cloud Project ID" # TODO subscription_name = "Your Pub/Sub subscription name" + # TODO timeout = 5.0 # "How long the subscriber should listen for + # messages in seconds" subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( @@ -251,15 +263,18 @@ def callback(message): # Limit the subscriber to only have ten outstanding messages at a time. flow_control = pubsub_v1.types.FlowControl(max_messages=10) - subscriber.subscribe( + + streaming_pull_future = subscriber.subscribe( subscription_path, callback=callback, flow_control=flow_control ) + print("Listening for messages on {}..\n".format(subscription_path)) - # The subscriber is non-blocking, so we must keep the main thread from - # exiting to allow it to process messages in the background. - print("Listening for messages on {}".format(subscription_path)) - while True: - time.sleep(60) + # result() in a future will block indefinitely if `timeout` is not set, + # unless an exception is encountered first. + try: + streaming_pull_future.result(timeout=timeout) + except: # noqa + streaming_pull_future.cancel() # [END pubsub_subscriber_flow_settings] @@ -386,13 +401,15 @@ def worker(msg): # [END pubsub_subscriber_sync_pull_with_lease] -def listen_for_errors(project_id, subscription_name): +def listen_for_errors(project_id, subscription_name, timeout=None): """Receives messages and catches errors from a pull subscription.""" # [START pubsub_subscriber_error_listener] from google.cloud import pubsub_v1 # TODO project_id = "Your Google Cloud Project ID" # TODO subscription_name = "Your Pubsub subscription name" + # TODO timeout = 5.0 # "How long the subscriber should listen for + # messages in seconds" subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path( @@ -403,16 +420,19 @@ def callback(message): print("Received message: {}".format(message)) message.ack() - future = subscriber.subscribe(subscription_path, callback=callback) + streaming_pull_future = subscriber.subscribe( + subscription_path, callback=callback + ) + print("Listening for messages on {}..\n".format(subscription_path)) - # Blocks the thread while messages are coming in through the stream. Any - # exceptions that crop up on the thread will be set on the future. + # result() in a future will block indefinitely if `timeout` is not set, + # unless an exception is encountered first. try: - # When timeout is unspecified, the result method waits indefinitely. - future.result(timeout=30) + streaming_pull_future.result(timeout=timeout) except Exception as e: + streaming_pull_future.cancel() print( - "Listening for messages on {} threw an Exception: {}.".format( + "Listening for messages on {} threw an exception: {}.".format( subscription_name, e ) ) @@ -518,14 +538,14 @@ def callback(message): args.project_id, args.subscription_name, args.endpoint ) elif args.command == "receive": - receive_messages(args.project_id, args.subscription_name) + receive_messages(args.project_id, args.subscription_name, args.timeout) elif args.command == "receive-custom-attributes": receive_messages_with_custom_attributes( - args.project_id, args.subscription_name + args.project_id, args.subscription_name, args.timeout ) elif args.command == "receive-flow-control": receive_messages_with_flow_control( - args.project_id, args.subscription_name + args.project_id, args.subscription_name, args.timeout ) elif args.command == "receive-synchronously": synchronous_pull(args.project_id, args.subscription_name) @@ -534,4 +554,6 @@ def callback(message): args.project_id, args.subscription_name ) elif args.command == "listen_for_errors": - listen_for_errors(args.project_id, args.subscription_name) + listen_for_errors( + args.project_id, args.subscription_name, args.timeout + ) diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index 0645c0738..50353c1c6 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -13,12 +13,10 @@ # limitations under the License. import os -import time import uuid from gcp_devrel.testing import eventually_consistent from google.cloud import pubsub_v1 -import mock import pytest import subscriber @@ -26,9 +24,9 @@ UUID = uuid.uuid4().hex PROJECT = os.environ["GCLOUD_PROJECT"] TOPIC = "subscription-test-topic-" + UUID -SUBSCRIPTION_ONE = "subscription-test-subscription-one-" + UUID -SUBSCRIPTION_TWO = "subscription-test-subscription-two-" + UUID -SUBSCRIPTION_THREE = "subscription-test-subscription-three-" + UUID +SUBSCRIPTION_ADMIN = "subscription-test-subscription-admin-" + UUID +SUBSCRIPTION_ASYNC = "subscription-test-subscription-async-" + UUID +SUBSCRIPTION_SYNC = "subscription-test-subscription-sync-" + UUID ENDPOINT = "https://{}.appspot.com/push".format(PROJECT) NEW_ENDPOINT = "https://{}.appspot.com/push2".format(PROJECT) @@ -43,11 +41,13 @@ def topic(publisher_client): topic_path = publisher_client.topic_path(PROJECT, TOPIC) try: - response = publisher_client.get_topic(topic_path) + subscription = publisher_client.get_topic(topic_path) except: # noqa - response = publisher_client.create_topic(topic_path) + subscription = publisher_client.create_topic(topic_path) - yield response.name + yield subscription.name + + publisher_client.delete_topic(subscription.name) @pytest.fixture(scope="module") @@ -56,72 +56,76 @@ def subscriber_client(): @pytest.fixture(scope="module") -def subscription_one(subscriber_client, topic): +def subscription_admin(subscriber_client, topic): subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_ONE + PROJECT, SUBSCRIPTION_ADMIN ) try: - response = subscriber_client.get_subscription(subscription_path) + subscription = subscriber_client.get_subscription(subscription_path) except: # noqa - response = subscriber_client.create_subscription( + subscription = subscriber_client.create_subscription( subscription_path, topic=topic ) - yield response.name + yield subscription.name @pytest.fixture(scope="module") -def subscription_two(subscriber_client, topic): +def subscription_sync(subscriber_client, topic): subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_TWO + PROJECT, SUBSCRIPTION_SYNC ) try: - response = subscriber_client.get_subscription(subscription_path) + subscription = subscriber_client.get_subscription(subscription_path) except: # noqa - response = subscriber_client.create_subscription( + subscription = subscriber_client.create_subscription( subscription_path, topic=topic ) - yield response.name + yield subscription.name + + subscriber_client.delete_subscription(subscription.name) @pytest.fixture(scope="module") -def subscription_three(subscriber_client, topic): +def subscription_async(subscriber_client, topic): subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_THREE + PROJECT, SUBSCRIPTION_ASYNC ) try: - response = subscriber_client.get_subscription(subscription_path) + subscription = subscriber_client.get_subscription(subscription_path) except: # noqa - response = subscriber_client.create_subscription( + subscription = subscriber_client.create_subscription( subscription_path, topic=topic ) - yield response.name + yield subscription.name + + subscriber_client.delete_subscription(subscription.name) -def test_list_in_topic(subscription_one, capsys): +def test_list_in_topic(subscription_admin, capsys): @eventually_consistent.call def _(): subscriber.list_subscriptions_in_topic(PROJECT, TOPIC) out, _ = capsys.readouterr() - assert subscription_one in out + assert subscription_admin in out -def test_list_in_project(subscription_one, capsys): +def test_list_in_project(subscription_admin, capsys): @eventually_consistent.call def _(): subscriber.list_subscriptions_in_project(PROJECT) out, _ = capsys.readouterr() - assert subscription_one in out + assert subscription_admin in out def test_create(subscriber_client): subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_ONE + PROJECT, SUBSCRIPTION_ADMIN ) try: @@ -129,7 +133,7 @@ def test_create(subscriber_client): except Exception: pass - subscriber.create_subscription(PROJECT, TOPIC, SUBSCRIPTION_ONE) + subscriber.create_subscription(PROJECT, TOPIC, SUBSCRIPTION_ADMIN) @eventually_consistent.call def _(): @@ -138,7 +142,7 @@ def _(): def test_create_push(subscriber_client): subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_ONE + PROJECT, SUBSCRIPTION_ADMIN ) try: subscriber_client.delete_subscription(subscription_path) @@ -146,7 +150,7 @@ def test_create_push(subscriber_client): pass subscriber.create_push_subscription( - PROJECT, TOPIC, SUBSCRIPTION_ONE, ENDPOINT + PROJECT, TOPIC, SUBSCRIPTION_ADMIN, ENDPOINT ) @eventually_consistent.call @@ -154,135 +158,107 @@ def _(): assert subscriber_client.get_subscription(subscription_path) -def test_update(subscriber_client, subscription_one, capsys): - subscriber.update_subscription(PROJECT, SUBSCRIPTION_ONE, NEW_ENDPOINT) +def test_update(subscriber_client, subscription_admin, capsys): + subscriber.update_subscription(PROJECT, SUBSCRIPTION_ADMIN, NEW_ENDPOINT) out, _ = capsys.readouterr() assert "Subscription updated" in out -def test_delete(subscriber_client, subscription_one): - subscriber.delete_subscription(PROJECT, SUBSCRIPTION_ONE) +def test_delete(subscriber_client, subscription_admin): + subscriber.delete_subscription(PROJECT, SUBSCRIPTION_ADMIN) @eventually_consistent.call def _(): with pytest.raises(Exception): - subscriber_client.get_subscription(subscription_one) + subscriber_client.get_subscription(subscription_admin) def _publish_messages(publisher_client, topic): for n in range(5): - data = u"Message {}".format(n).encode("utf-8") - future = publisher_client.publish(topic, data=data) - future.result() - - -def _publish_messages_with_custom_attributes(publisher_client, topic): - data = u"Test message".encode("utf-8") - future = publisher_client.publish(topic, data=data, origin="python-sample") - future.result() - - -def _make_sleep_patch(): - real_sleep = time.sleep - - def new_sleep(period): - if period == 60: - real_sleep(5) - raise RuntimeError("sigil") - else: - real_sleep(period) - - return mock.patch("time.sleep", new=new_sleep) - - -def _to_delete(): - publisher_client = pubsub_v1.PublisherClient() - subscriber_client = pubsub_v1.SubscriberClient() - resources = [TOPIC, SUBSCRIPTION_TWO, SUBSCRIPTION_THREE] - - for item in resources: - if "subscription-test-topic" in item: - publisher_client.delete_topic( - "projects/{}/topics/{}".format(PROJECT, item) - ) - if "subscription-test-subscription" in item: - subscriber_client.delete_subscription( - "projects/{}/subscriptions/{}".format(PROJECT, item) - ) + data = u"message {}".format(n).encode("utf-8") + publish_future = publisher_client.publish( + topic, data=data, origin="python-sample" + ) + publish_future.result() -def test_receive(publisher_client, topic, subscription_two, capsys): +def test_receive(publisher_client, topic, subscription_async, capsys): _publish_messages(publisher_client, topic) - with _make_sleep_patch(): - with pytest.raises(RuntimeError, match="sigil"): - subscriber.receive_messages(PROJECT, SUBSCRIPTION_TWO) + subscriber.receive_messages(PROJECT, SUBSCRIPTION_ASYNC, 5) out, _ = capsys.readouterr() assert "Listening" in out - assert subscription_two in out - assert "Message" in out + assert subscription_async in out + assert "message" in out def test_receive_with_custom_attributes( - publisher_client, topic, subscription_two, capsys + publisher_client, topic, subscription_async, capsys ): - _publish_messages_with_custom_attributes(publisher_client, topic) + _publish_messages(publisher_client, topic) - with _make_sleep_patch(): - with pytest.raises(RuntimeError, match="sigil"): - subscriber.receive_messages_with_custom_attributes( - PROJECT, SUBSCRIPTION_TWO - ) + subscriber.receive_messages_with_custom_attributes( + PROJECT, SUBSCRIPTION_ASYNC, 5 + ) out, _ = capsys.readouterr() - assert "Test message" in out + assert "message" in out assert "origin" in out assert "python-sample" in out def test_receive_with_flow_control( - publisher_client, topic, subscription_two, capsys + publisher_client, topic, subscription_async, capsys ): _publish_messages(publisher_client, topic) - with _make_sleep_patch(): - with pytest.raises(RuntimeError, match="sigil"): - subscriber.receive_messages_with_flow_control( - PROJECT, SUBSCRIPTION_TWO - ) + subscriber.receive_messages_with_flow_control( + PROJECT, SUBSCRIPTION_ASYNC, 5 + ) out, _ = capsys.readouterr() assert "Listening" in out - assert subscription_two in out - assert "Message" in out + assert subscription_async in out + assert "message" in out def test_receive_synchronously( - publisher_client, topic, subscription_three, capsys + publisher_client, topic, subscription_sync, capsys ): _publish_messages(publisher_client, topic) - subscriber.synchronous_pull(PROJECT, SUBSCRIPTION_THREE) + subscriber.synchronous_pull(PROJECT, SUBSCRIPTION_SYNC) out, _ = capsys.readouterr() assert "Done." in out def test_receive_synchronously_with_lease( - publisher_client, topic, subscription_three, capsys + publisher_client, topic, subscription_sync, capsys ): _publish_messages(publisher_client, topic) subscriber.synchronous_pull_with_lease_management( - PROJECT, SUBSCRIPTION_THREE + PROJECT, SUBSCRIPTION_SYNC ) out, _ = capsys.readouterr() assert "Done." in out - # Clean up resources after all the tests. - _to_delete() + +def test_listen_for_errors( + publisher_client, topic, subscription_async, capsys +): + + _publish_messages(publisher_client, topic) + + subscriber.listen_for_errors(PROJECT, SUBSCRIPTION_ASYNC, 5) + + out, _ = capsys.readouterr() + assert "Listening" in out + assert subscription_async in out + assert "threw an exception" in out From d453424a6c9b8994de8424a11895625881922742 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Wed, 18 Dec 2019 14:31:23 -0800 Subject: [PATCH 080/101] Pub/Sub: add timeout in argparse [(#2637)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2637) --- samples/snippets/subscriber.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 0d328d232..79c9bc4a0 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -484,17 +484,24 @@ def callback(message): "receive", help=receive_messages.__doc__ ) receive_parser.add_argument("subscription_name") + receive_parser.add_argument("--timeout", default=None, type=float) receive_with_custom_attributes_parser = subparsers.add_parser( "receive-custom-attributes", help=receive_messages_with_custom_attributes.__doc__, ) receive_with_custom_attributes_parser.add_argument("subscription_name") + receive_with_custom_attributes_parser.add_argument( + "--timeout", default=None, type=float + ) receive_with_flow_control_parser = subparsers.add_parser( "receive-flow-control", help=receive_messages_with_flow_control.__doc__ ) receive_with_flow_control_parser.add_argument("subscription_name") + receive_with_flow_control_parser.add_argument( + "--timeout", default=None, type=float + ) synchronous_pull_parser = subparsers.add_parser( "receive-synchronously", help=synchronous_pull.__doc__ @@ -513,6 +520,9 @@ def callback(message): "listen_for_errors", help=listen_for_errors.__doc__ ) listen_for_errors_parser.add_argument("subscription_name") + listen_for_errors_parser.add_argument( + "--timeout", default=None, type=float + ) args = parser.parse_args() From 7d4b6c53bfabd5679f66561ee131ccea0dfa0f4b Mon Sep 17 00:00:00 2001 From: DPEBot Date: Fri, 20 Dec 2019 17:41:38 -0800 Subject: [PATCH 081/101] Auto-update dependencies. [(#2005)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2005) * Auto-update dependencies. * Revert update of appengine/flexible/datastore. * revert update of appengine/flexible/scipy * revert update of bigquery/bqml * revert update of bigquery/cloud-client * revert update of bigquery/datalab-migration * revert update of bigtable/quickstart * revert update of compute/api * revert update of container_registry/container_analysis * revert update of dataflow/run_template * revert update of datastore/cloud-ndb * revert update of dialogflow/cloud-client * revert update of dlp * revert update of functions/imagemagick * revert update of functions/ocr/app * revert update of healthcare/api-client/fhir * revert update of iam/api-client * revert update of iot/api-client/gcs_file_to_device * revert update of iot/api-client/mqtt_example * revert update of language/automl * revert update of run/image-processing * revert update of vision/automl * revert update testing/requirements.txt * revert update of vision/cloud-client/detect * revert update of vision/cloud-client/product_search * revert update of jobs/v2/api_client * revert update of jobs/v3/api_client * revert update of opencensus * revert update of translate/cloud-client * revert update to speech/cloud-client Co-authored-by: Kurtis Van Gent <31518063+kurtisvg@users.noreply.github.com> Co-authored-by: Doug Mahugh --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index a97fc0997..a5a8b2bb9 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==1.0.0 +google-cloud-pubsub==1.1.0 From 1486d27c05f49f9172594151147643e21a179af0 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Mon, 24 Feb 2020 10:03:17 -0800 Subject: [PATCH 082/101] remove publish concurrency control sample [(#2960)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2960) --- samples/snippets/README.rst | 36 ++++++++++++++++-------------- samples/snippets/publisher.py | 31 ------------------------- samples/snippets/publisher_test.py | 7 ------ 3 files changed, 19 insertions(+), 55 deletions(-) diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index e0e265f8d..21a4b231f 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -93,8 +93,8 @@ To run this sample: $ python publisher.py usage: publisher.py [-h] - project - {list,create,delete,publish,publish-with-custom-attributes,publish-with-futures,publish-with-batch-settings} + project_id + {list,create,delete,publish,publish-with-custom-attributes,publish-with-error-handler,publish-with-batch-settings,publish-with-retry-settings} ... This application demonstrates how to perform basic operations on topics @@ -104,8 +104,8 @@ To run this sample: at https://cloud.google.com/pubsub/docs. positional arguments: - project Your Google Cloud project ID - {list,create,delete,publish,publish-with-custom-attributes,publish-with-futures,publish-with-batch-settings} + project_id Your Google Cloud project ID + {list,create,delete,publish,publish-with-custom-attributes,publish-with-error-handler,publish-with-batch-settings,publish-with-retry-settings} list Lists all Pub/Sub topics in the given project. create Create a new Pub/Sub topic. delete Deletes an existing Pub/Sub topic. @@ -113,12 +113,14 @@ To run this sample: publish-with-custom-attributes Publishes multiple messages with custom attributes to a Pub/Sub topic. - publish-with-futures - Publishes multiple messages to a Pub/Sub topic and - prints their message IDs. + publish-with-error-handler + Publishes multiple messages to a Pub/Sub topic with an + error handler. publish-with-batch-settings Publishes multiple messages to a Pub/Sub topic with batch settings. + publish-with-retry-settings + Publishes messages with custom retry settings. optional arguments: -h, --help show this help message and exit @@ -141,8 +143,8 @@ To run this sample: $ python subscriber.py usage: subscriber.py [-h] - project - {list_in_topic,list_in_project,create,create-push,delete,update,receive,receive-custom-attributes,receive-flow-control,listen_for_errors} + project_id + {list_in_topic,list_in_project,create,create-push,delete,update,receive,receive-custom-attributes,receive-flow-control,receive-synchronously,receive-synchronously-with-lease,listen_for_errors} ... This application demonstrates how to perform basic operations on @@ -152,26 +154,26 @@ To run this sample: at https://cloud.google.com/pubsub/docs. positional arguments: - project Your Google Cloud project ID - {list_in_topic,list_in_project,create,create-push,delete,update,receive,receive-custom-attributes,receive-flow-control,listen_for_errors} + project_id Your Google Cloud project ID + {list_in_topic,list_in_project,create,create-push,delete,update,receive,receive-custom-attributes,receive-flow-control,receive-synchronously,receive-synchronously-with-lease,listen_for_errors} list_in_topic Lists all subscriptions for a given topic. list_in_project Lists all subscriptions in the current project. create Create a new pull subscription on the given topic. - create-push Create a new push subscription on the given topic. For - example, endpoint is "https://my-test- - project.appspot.com/push". + create-push Create a new push subscription on the given topic. delete Deletes an existing Pub/Sub topic. update Updates an existing Pub/Sub subscription's push endpoint URL. Note that certain properties of a subscription, such as its topic, are not modifiable. - For example, endpoint is "https://my-test- - project.appspot.com/push". receive Receives messages from a pull subscription. receive-custom-attributes Receives messages from a pull subscription. receive-flow-control Receives messages from a pull subscription with flow control. + receive-synchronously + Pulling messages synchronously. + receive-synchronously-with-lease + Pulling messages synchronously with lease management listen_for_errors Receives messages and catches errors from a pull subscription. @@ -244,4 +246,4 @@ to `browse the source`_ and `report issues`_. https://github.com/GoogleCloudPlatform/google-cloud-python/issues -.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file +.. _Google Cloud SDK: https://cloud.google.com/sdk/ diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index d227baab9..df7a9f23f 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -128,30 +128,6 @@ def publish_messages_with_custom_attributes(project_id, topic_name): # [END pubsub_publish_custom_attributes] -def publish_messages_with_futures(project_id, topic_name): - """Publishes multiple messages to a Pub/Sub topic and prints their - message IDs.""" - # [START pubsub_publisher_concurrency_control] - from google.cloud import pubsub_v1 - - # TODO project_id = "Your Google Cloud Project ID" - # TODO topic_name = "Your Pub/Sub topic name" - - publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project_id, topic_name) - - for n in range(1, 10): - data = u"Message number {}".format(n) - # Data must be a bytestring - data = data.encode("utf-8") - # When you publish a message, the client returns a future. - future = publisher.publish(topic_path, data=data) - print(future.result()) - - print("Published messages with futures.") - # [END pubsub_publisher_concurrency_control] - - def publish_messages_with_error_handler(project_id, topic_name): # [START pubsub_publish_messages_error_handler] """Publishes multiple messages to a Pub/Sub topic with an error handler.""" @@ -308,11 +284,6 @@ def publish_messages_with_retry_settings(project_id, topic_name): ) publish_with_custom_attributes_parser.add_argument("topic_name") - publish_with_futures_parser = subparsers.add_parser( - "publish-with-futures", help=publish_messages_with_futures.__doc__ - ) - publish_with_futures_parser.add_argument("topic_name") - publish_with_error_handler_parser = subparsers.add_parser( "publish-with-error-handler", help=publish_messages_with_error_handler.__doc__, @@ -345,8 +316,6 @@ def publish_messages_with_retry_settings(project_id, topic_name): publish_messages_with_custom_attributes( args.project_id, args.topic_name ) - elif args.command == "publish-with-futures": - publish_messages_with_futures(args.project_id, args.topic_name) elif args.command == "publish-with-error-handler": publish_messages_with_error_handler(args.project_id, args.topic_name) elif args.command == "publish-with-batch-settings": diff --git a/samples/snippets/publisher_test.py b/samples/snippets/publisher_test.py index fbe30694a..aa55011c1 100644 --- a/samples/snippets/publisher_test.py +++ b/samples/snippets/publisher_test.py @@ -138,10 +138,3 @@ def test_publish_with_error_handler(topic_publish, capsys): out, _ = capsys.readouterr() assert "Published" in out - - -def test_publish_with_futures(topic_publish, capsys): - publisher.publish_messages_with_futures(PROJECT, TOPIC_PUBLISH) - - out, _ = capsys.readouterr() - assert "Published" in out From f2336e62e371d77833353e2d400e543c359e58b6 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Wed, 26 Feb 2020 10:13:03 -0800 Subject: [PATCH 083/101] Pub/Sub: remove unreferenced samples [(#2986)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2986) * remove qs samples * update README --- samples/snippets/README.rst | 16 ---- samples/snippets/quickstart.py | 111 ---------------------------- samples/snippets/quickstart_test.py | 78 ------------------- 3 files changed, 205 deletions(-) delete mode 100644 samples/snippets/quickstart.py delete mode 100644 samples/snippets/quickstart_test.py diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index 21a4b231f..c30fd190a 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -61,22 +61,6 @@ Install Dependencies Samples ------------------------------------------------------------------------------- -Quickstart -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/quickstart.py,pubsub/cloud-client/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python quickstart.py - - Publisher +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ diff --git a/samples/snippets/quickstart.py b/samples/snippets/quickstart.py deleted file mode 100644 index d01105885..000000000 --- a/samples/snippets/quickstart.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import argparse - - -def end_to_end(project_id, topic_name, subscription_name, num_messages): - # [START pubsub_end_to_end] - import time - - from google.cloud import pubsub_v1 - - # TODO project_id = "Your Google Cloud Project ID" - # TODO topic_name = "Your Pub/Sub topic name" - # TODO num_messages = number of messages to test end-to-end - - # Instantiates a publisher and subscriber client - publisher = pubsub_v1.PublisherClient() - subscriber = pubsub_v1.SubscriberClient() - - # The `topic_path` method creates a fully qualified identifier - # in the form `projects/{project_id}/topics/{topic_name}` - topic_path = subscriber.topic_path(project_id, topic_name) - - # The `subscription_path` method creates a fully qualified identifier - # in the form `projects/{project_id}/subscriptions/{subscription_name}` - subscription_path = subscriber.subscription_path( - project_id, subscription_name - ) - - # Create the topic. - topic = publisher.create_topic(topic_path) - print("\nTopic created: {}".format(topic.name)) - - # Create a subscription. - subscription = subscriber.create_subscription( - subscription_path, topic_path - ) - print("\nSubscription created: {}\n".format(subscription.name)) - - publish_begin = time.time() - - # Publish messages. - for n in range(num_messages): - data = u"Message number {}".format(n) - # Data must be a bytestring - data = data.encode("utf-8") - # When you publish a message, the client returns a future. - future = publisher.publish(topic_path, data=data) - print("Published {} of message ID {}.".format(data, future.result())) - - publish_time = time.time() - publish_begin - - messages = set() - - def callback(message): - print("Received message: {}".format(message)) - # Unacknowledged messages will be sent again. - message.ack() - messages.add(message) - - subscribe_begin = time.time() - - # Receive messages. The subscriber is nonblocking. - subscriber.subscribe(subscription_path, callback=callback) - - print("\nListening for messages on {}...\n".format(subscription_path)) - - while True: - if len(messages) == num_messages: - subscribe_time = time.time() - subscribe_begin - print("\nReceived all messages.") - print("Publish time lapsed: {:.2f}s.".format(publish_time)) - print("Subscribe time lapsed: {:.2f}s.".format(subscribe_time)) - break - else: - # Sleeps the thread at 50Hz to save on resources. - time.sleep(1.0 / 50) - # [END pubsub_end_to_end] - - -if __name__ == "__main__": - - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument("project_id", help="Your Google Cloud project ID") - parser.add_argument("topic_name", help="Your topic name") - parser.add_argument("subscription_name", help="Your subscription name") - parser.add_argument("num_msgs", type=int, help="Number of test messages") - - args = parser.parse_args() - - end_to_end( - args.project_id, args.topic_name, args.subscription_name, args.num_msgs - ) diff --git a/samples/snippets/quickstart_test.py b/samples/snippets/quickstart_test.py deleted file mode 100644 index 6a1d4aae1..000000000 --- a/samples/snippets/quickstart_test.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import uuid - -from google.cloud import pubsub_v1 -import pytest -import quickstart - -UUID = uuid.uuid4().hex -PROJECT = os.environ["GCLOUD_PROJECT"] -TOPIC = "end-to-end-test-topic-" + UUID -SUBSCRIPTION = "end-to-end-test-topic-sub-" + UUID -N = 10 - - -@pytest.fixture(scope="module") -def publisher_client(): - yield pubsub_v1.PublisherClient() - - -@pytest.fixture(scope="module") -def topic(publisher_client): - topic_path = publisher_client.topic_path(PROJECT, TOPIC) - - try: - publisher_client.delete_topic(topic_path) - except Exception: - pass - - yield TOPIC - - publisher_client.delete_topic(topic_path) - - -@pytest.fixture(scope="module") -def subscriber_client(): - yield pubsub_v1.SubscriberClient() - - -@pytest.fixture(scope="module") -def subscription(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION - ) - - try: - subscriber_client.delete_subscription(subscription_path) - except Exception: - pass - - yield SUBSCRIPTION - - subscriber_client.delete_subscription(subscription_path) - - -def test_end_to_end(topic, subscription, capsys): - - quickstart.end_to_end(PROJECT, topic, subscription, N) - out, _ = capsys.readouterr() - - assert "Received all messages" in out - assert "Publish time lapsed" in out - assert "Subscribe time lapsed" in out From bd44acdcb2fd7b3256b9ea2e622f199c87680ebd Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Mon, 23 Mar 2020 15:17:49 -0700 Subject: [PATCH 084/101] Pub/Sub: add SubscriberClient.close() to examples [(#3118)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3118) * Add SubscriberClient.close() to examples. Co-authored-by: Prad Nelluru Co-authored-by: Prad Nelluru --- samples/snippets/iam.py | 6 ++ samples/snippets/iam_test.py | 4 +- samples/snippets/quickstart/sub.py | 14 +++-- samples/snippets/quickstart/sub_test.py | 3 + samples/snippets/requirements.txt | 2 +- samples/snippets/subscriber.py | 78 ++++++++++++++++--------- samples/snippets/subscriber_test.py | 4 +- 7 files changed, 75 insertions(+), 36 deletions(-) diff --git a/samples/snippets/iam.py b/samples/snippets/iam.py index f014ce749..eb0c82463 100644 --- a/samples/snippets/iam.py +++ b/samples/snippets/iam.py @@ -51,6 +51,8 @@ def get_subscription_policy(project, subscription_name): print("Policy for subscription {}:".format(subscription_path)) for binding in policy.bindings: print("Role: {}, Members: {}".format(binding.role, binding.members)) + + client.close() # [END pubsub_get_subscription_policy] @@ -101,6 +103,8 @@ def set_subscription_policy(project, subscription_name): subscription_name, policy ) ) + + client.close() # [END pubsub_set_subscription_policy] @@ -144,6 +148,8 @@ def check_subscription_permissions(project, subscription_name): subscription_path, allowed_permissions ) ) + + client.close() # [END pubsub_test_subscription_permissions] diff --git a/samples/snippets/iam_test.py b/samples/snippets/iam_test.py index 2b019f9ea..f88cde851 100644 --- a/samples/snippets/iam_test.py +++ b/samples/snippets/iam_test.py @@ -49,7 +49,9 @@ def topic(publisher_client): @pytest.fixture(scope="module") def subscriber_client(): - yield pubsub_v1.SubscriberClient() + subscriber_client = pubsub_v1.SubscriberClient() + yield subscriber_client + subscriber_client.close() @pytest.fixture diff --git a/samples/snippets/quickstart/sub.py b/samples/snippets/quickstart/sub.py index 5791af14d..1d90726f5 100644 --- a/samples/snippets/quickstart/sub.py +++ b/samples/snippets/quickstart/sub.py @@ -27,11 +27,13 @@ def sub(project_id, subscription_name): """Receives messages from a Pub/Sub subscription.""" # [START pubsub_quickstart_sub_client] # Initialize a Subscriber client - client = pubsub_v1.SubscriberClient() + subscriber_client = pubsub_v1.SubscriberClient() # [END pubsub_quickstart_sub_client] # Create a fully qualified identifier in the form of # `projects/{project_id}/subscriptions/{subscription_name}` - subscription_path = client.subscription_path(project_id, subscription_name) + subscription_path = subscriber_client.subscription_path( + project_id, subscription_name + ) def callback(message): print( @@ -43,18 +45,20 @@ def callback(message): message.ack() print("Acknowledged message {}\n".format(message.message_id)) - streaming_pull_future = client.subscribe( + streaming_pull_future = subscriber_client.subscribe( subscription_path, callback=callback ) print("Listening for messages on {}..\n".format(subscription_path)) - # Calling result() on StreamingPullFuture keeps the main thread from - # exiting while messages get processed in the callbacks. try: + # Calling result() on StreamingPullFuture keeps the main thread from + # exiting while messages get processed in the callbacks. streaming_pull_future.result() except: # noqa streaming_pull_future.cancel() + subscriber_client.close() + if __name__ == "__main__": parser = argparse.ArgumentParser( diff --git a/samples/snippets/quickstart/sub_test.py b/samples/snippets/quickstart/sub_test.py index 07edfad7c..1b59a3d04 100644 --- a/samples/snippets/quickstart/sub_test.py +++ b/samples/snippets/quickstart/sub_test.py @@ -62,6 +62,7 @@ def subscription_path(topic_path): yield subscription_path subscriber_client.delete_subscription(subscription_path) + subscriber_client.close() def _publish_messages(topic_path): @@ -102,3 +103,5 @@ def mock_result(): out, _ = capsys.readouterr() assert "Received message" in out assert "Acknowledged message" in out + + real_client.close() diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index a5a8b2bb9..cc192f6f7 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==1.1.0 +google-cloud-pubsub==1.3.0 diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 79c9bc4a0..e22efc7b1 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -52,6 +52,8 @@ def list_subscriptions_in_project(project_id): for subscription in subscriber.list_subscriptions(project_path): print(subscription.name) + + subscriber.close() # [END pubsub_list_subscriptions] @@ -75,6 +77,8 @@ def create_subscription(project_id, topic_name, subscription_name): ) print("Subscription created: {}".format(subscription)) + + subscriber.close() # [END pubsub_create_pull_subscription] @@ -104,6 +108,8 @@ def create_push_subscription( print("Push subscription created: {}".format(subscription)) print("Endpoint for subscription is: {}".format(endpoint)) + + subscriber.close() # [END pubsub_create_push_subscription] @@ -123,6 +129,8 @@ def delete_subscription(project_id, subscription_name): subscriber.delete_subscription(subscription_path) print("Subscription deleted: {}".format(subscription_path)) + + subscriber.close() # [END pubsub_delete_subscription] @@ -158,6 +166,8 @@ def update_subscription(project_id, subscription_name, endpoint): print("Subscription updated: {}".format(subscription_path)) print("New endpoint for subscription is: {}".format(result.push_config)) + + subscriber.close() # [END pubsub_update_push_configuration] @@ -188,12 +198,14 @@ def callback(message): ) print("Listening for messages on {}..\n".format(subscription_path)) - # result() in a future will block indefinitely if `timeout` is not set, - # unless an exception is encountered first. - try: - streaming_pull_future.result(timeout=timeout) - except: # noqa - streaming_pull_future.cancel() + # Wrap subscriber in a 'with' block to automatically call close() when done. + with subscriber: + try: + # When `timeout` is not set, result() will block indefinitely, + # unless an exception is encountered first. + streaming_pull_future.result(timeout=timeout) + except: # noqa + streaming_pull_future.cancel() # [END pubsub_subscriber_async_pull] # [END pubsub_quickstart_subscriber] @@ -230,12 +242,14 @@ def callback(message): ) print("Listening for messages on {}..\n".format(subscription_path)) - # result() in a future will block indefinitely if `timeout` is not set, - # unless an exception is encountered first. - try: - streaming_pull_future.result(timeout=timeout) - except: # noqa - streaming_pull_future.cancel() + # Wrap subscriber in a 'with' block to automatically call close() when done. + with subscriber: + try: + # When `timeout` is not set, result() will block indefinitely, + # unless an exception is encountered first. + streaming_pull_future.result(timeout=timeout) + except: # noqa + streaming_pull_future.cancel() # [END pubsub_subscriber_async_pull_custom_attributes] # [END pubsub_subscriber_sync_pull_custom_attributes] @@ -269,12 +283,14 @@ def callback(message): ) print("Listening for messages on {}..\n".format(subscription_path)) - # result() in a future will block indefinitely if `timeout` is not set, - # unless an exception is encountered first. - try: - streaming_pull_future.result(timeout=timeout) - except: # noqa - streaming_pull_future.cancel() + # Wrap subscriber in a 'with' block to automatically call close() when done. + with subscriber: + try: + # When `timeout` is not set, result() will block indefinitely, + # unless an exception is encountered first. + streaming_pull_future.result(timeout=timeout) + except: # noqa + streaming_pull_future.cancel() # [END pubsub_subscriber_flow_settings] @@ -309,6 +325,8 @@ def synchronous_pull(project_id, subscription_name): len(response.received_messages) ) ) + + subscriber.close() # [END pubsub_subscriber_sync_pull] @@ -398,6 +416,8 @@ def worker(msg): len(response.received_messages) ) ) + + subscriber.close() # [END pubsub_subscriber_sync_pull_with_lease] @@ -425,17 +445,19 @@ def callback(message): ) print("Listening for messages on {}..\n".format(subscription_path)) - # result() in a future will block indefinitely if `timeout` is not set, - # unless an exception is encountered first. - try: - streaming_pull_future.result(timeout=timeout) - except Exception as e: - streaming_pull_future.cancel() - print( - "Listening for messages on {} threw an exception: {}.".format( - subscription_name, e + # Wrap subscriber in a 'with' block to automatically call close() when done. + with subscriber: + # When `timeout` is not set, result() will block indefinitely, + # unless an exception is encountered first. + try: + streaming_pull_future.result(timeout=timeout) + except Exception as e: + streaming_pull_future.cancel() + print( + "Listening for messages on {} threw an exception: {}.".format( + subscription_name, e + ) ) - ) # [END pubsub_subscriber_error_listener] diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index 50353c1c6..94905d635 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -52,7 +52,9 @@ def topic(publisher_client): @pytest.fixture(scope="module") def subscriber_client(): - yield pubsub_v1.SubscriberClient() + subscriber_client = pubsub_v1.SubscriberClient() + yield subscriber_client + subscriber_client.close() @pytest.fixture(scope="module") From c22259855e644d6c85b624ec5c9b35583351ae51 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Wed, 25 Mar 2020 12:47:33 -0700 Subject: [PATCH 085/101] Pub/Sub: update publish with batch settings sample [(#3137)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3137) * non-blocking publish * remove unused lib * lint * add defaults --- samples/snippets/publisher.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index df7a9f23f..6802ec85f 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -180,20 +180,28 @@ def publish_messages_with_batch_settings(project_id, topic_name): # TODO project_id = "Your Google Cloud Project ID" # TODO topic_name = "Your Pub/Sub topic name" - # Configure the batch to publish as soon as there is one kilobyte - # of data or one second has passed. + # Configure the batch to publish as soon as there is ten messages, + # one kilobyte of data, or one second has passed. batch_settings = pubsub_v1.types.BatchSettings( - max_bytes=1024, max_latency=1 # One kilobyte # One second + max_messages=10, # default 100 + max_bytes=1024, # default 1 MB + max_latency=1, # default 10 ms ) publisher = pubsub_v1.PublisherClient(batch_settings) topic_path = publisher.topic_path(project_id, topic_name) + # Resolve the publish future in a separate thread. + def callback(future): + message_id = future.result() + print(message_id) + for n in range(1, 10): data = u"Message number {}".format(n) # Data must be a bytestring data = data.encode("utf-8") future = publisher.publish(topic_path, data=data) - print(future.result()) + # Non-blocking. Allow the publisher client to batch multiple messages. + future.add_done_callback(callback) print("Published messages with batch settings.") # [END pubsub_publisher_batch_settings] From faee6414b86143f6f76c602b388aa304516dc781 Mon Sep 17 00:00:00 2001 From: Kurtis Van Gent <31518063+kurtisvg@users.noreply.github.com> Date: Wed, 1 Apr 2020 19:11:50 -0700 Subject: [PATCH 086/101] Simplify noxfile setup. [(#2806)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2806) * chore(deps): update dependency requests to v2.23.0 * Simplify noxfile and add version control. * Configure appengine/standard to only test Python 2.7. * Update Kokokro configs to match noxfile. * Add requirements-test to each folder. * Remove Py2 versions from everything execept appengine/standard. * Remove conftest.py. * Remove appengine/standard/conftest.py * Remove 'no-sucess-flaky-report' from pytest.ini. * Add GAE SDK back to appengine/standard tests. * Fix typo. * Roll pytest to python 2 version. * Add a bunch of testing requirements. * Remove typo. * Add appengine lib directory back in. * Add some additional requirements. * Fix issue with flake8 args. * Even more requirements. * Readd appengine conftest.py. * Add a few more requirements. * Even more Appengine requirements. * Add webtest for appengine/standard/mailgun. * Add some additional requirements. * Add workaround for issue with mailjet-rest. * Add responses for appengine/standard/mailjet. Co-authored-by: Renovate Bot --- samples/snippets/requirements-test.txt | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 samples/snippets/requirements-test.txt diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt new file mode 100644 index 000000000..c445bcb1a --- /dev/null +++ b/samples/snippets/requirements-test.txt @@ -0,0 +1,4 @@ +pytest==5.3.2 +gcp-devrel-py-tools==0.0.15 +mock==3.0.5 +google-cloud-core==1.3.0 From 103ac43d5c251c26b966ce30b8cd4cd90399c1d5 Mon Sep 17 00:00:00 2001 From: Takashi Matsuo Date: Wed, 22 Apr 2020 18:08:44 -0700 Subject: [PATCH 087/101] chore: remove gcp-devrel-py-tools from iot and pubsub [(#3470)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3470) * [iot] chore: remove unused dependency * [pubsub] chore: remove gcp-devrel-py-tools --- samples/snippets/publisher_test.py | 20 ++++++++++------ samples/snippets/requirements-test.txt | 3 +-- samples/snippets/subscriber_test.py | 32 +++++++++++++++++--------- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/samples/snippets/publisher_test.py b/samples/snippets/publisher_test.py index aa55011c1..dc6095508 100644 --- a/samples/snippets/publisher_test.py +++ b/samples/snippets/publisher_test.py @@ -16,7 +16,7 @@ import time import uuid -from gcp_devrel.testing import eventually_consistent +import backoff from google.cloud import pubsub_v1 import mock import pytest @@ -75,12 +75,14 @@ def new_sleep(period): def test_list(client, topic_admin, capsys): - @eventually_consistent.call - def _(): + @backoff.on_exception(backoff.expo, AssertionError, max_time=60) + def eventually_consistent_test(): publisher.list_topics(PROJECT) out, _ = capsys.readouterr() assert topic_admin in out + eventually_consistent_test() + def test_create(client): topic_path = client.topic_path(PROJECT, TOPIC_ADMIN) @@ -91,19 +93,23 @@ def test_create(client): publisher.create_topic(PROJECT, TOPIC_ADMIN) - @eventually_consistent.call - def _(): + @backoff.on_exception(backoff.expo, AssertionError, max_time=60) + def eventually_consistent_test(): assert client.get_topic(topic_path) + eventually_consistent_test() + def test_delete(client, topic_admin): publisher.delete_topic(PROJECT, TOPIC_ADMIN) - @eventually_consistent.call - def _(): + @backoff.on_exception(backoff.expo, AssertionError, max_time=60) + def eventually_consistent_test(): with pytest.raises(Exception): client.get_topic(client.topic_path(PROJECT, TOPIC_ADMIN)) + eventually_consistent_test() + def test_publish(topic_publish, capsys): publisher.publish_messages(PROJECT, TOPIC_PUBLISH) diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index c445bcb1a..adf26b9f9 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,4 +1,3 @@ +backoff==1.10.0 pytest==5.3.2 -gcp-devrel-py-tools==0.0.15 mock==3.0.5 -google-cloud-core==1.3.0 diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index 94905d635..1c9520866 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -15,7 +15,7 @@ import os import uuid -from gcp_devrel.testing import eventually_consistent +import backoff from google.cloud import pubsub_v1 import pytest @@ -110,20 +110,24 @@ def subscription_async(subscriber_client, topic): def test_list_in_topic(subscription_admin, capsys): - @eventually_consistent.call - def _(): + @backoff.on_exception(backoff.expo, AssertionError, max_time=60) + def eventually_consistent_test(): subscriber.list_subscriptions_in_topic(PROJECT, TOPIC) out, _ = capsys.readouterr() assert subscription_admin in out + eventually_consistent_test() + def test_list_in_project(subscription_admin, capsys): - @eventually_consistent.call - def _(): + @backoff.on_exception(backoff.expo, AssertionError, max_time=60) + def eventually_consistent_test(): subscriber.list_subscriptions_in_project(PROJECT) out, _ = capsys.readouterr() assert subscription_admin in out + eventually_consistent_test() + def test_create(subscriber_client): subscription_path = subscriber_client.subscription_path( @@ -137,10 +141,12 @@ def test_create(subscriber_client): subscriber.create_subscription(PROJECT, TOPIC, SUBSCRIPTION_ADMIN) - @eventually_consistent.call - def _(): + @backoff.on_exception(backoff.expo, AssertionError, max_time=60) + def eventually_consistent_test(): assert subscriber_client.get_subscription(subscription_path) + eventually_consistent_test() + def test_create_push(subscriber_client): subscription_path = subscriber_client.subscription_path( @@ -155,10 +161,12 @@ def test_create_push(subscriber_client): PROJECT, TOPIC, SUBSCRIPTION_ADMIN, ENDPOINT ) - @eventually_consistent.call - def _(): + @backoff.on_exception(backoff.expo, AssertionError, max_time=60) + def eventually_consistent_test(): assert subscriber_client.get_subscription(subscription_path) + eventually_consistent_test() + def test_update(subscriber_client, subscription_admin, capsys): subscriber.update_subscription(PROJECT, SUBSCRIPTION_ADMIN, NEW_ENDPOINT) @@ -170,11 +178,13 @@ def test_update(subscriber_client, subscription_admin, capsys): def test_delete(subscriber_client, subscription_admin): subscriber.delete_subscription(PROJECT, SUBSCRIPTION_ADMIN) - @eventually_consistent.call - def _(): + @backoff.on_exception(backoff.expo, AssertionError, max_time=60) + def eventually_consistent_test(): with pytest.raises(Exception): subscriber_client.get_subscription(subscription_admin) + eventually_consistent_test() + def _publish_messages(publisher_client, topic): for n in range(5): From b17135610735445ef6d7c260f2b074a5a91c2a44 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Fri, 24 Apr 2020 12:28:38 -0700 Subject: [PATCH 088/101] Update dependency google-cloud-pubsub to v1.4.2 in Storage and Pub/Sub [(#3343)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3343) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index cc192f6f7..fbfc0f4d3 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==1.3.0 +google-cloud-pubsub==1.4.2 From 5871203d3bd1bac127fde2b072c9c297fffbdf7d Mon Sep 17 00:00:00 2001 From: Takashi Matsuo Date: Tue, 12 May 2020 17:30:32 -0700 Subject: [PATCH 089/101] chore: some lint fixes [(#3748)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3748) --- samples/snippets/quickstart/pub_test.py | 5 +++-- samples/snippets/quickstart/sub_test.py | 7 +++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/samples/snippets/quickstart/pub_test.py b/samples/snippets/quickstart/pub_test.py index b9a6f807f..24010c76e 100644 --- a/samples/snippets/quickstart/pub_test.py +++ b/samples/snippets/quickstart/pub_test.py @@ -15,13 +15,14 @@ # limitations under the License. import os -import pytest import uuid from google.api_core.exceptions import AlreadyExists from google.cloud import pubsub_v1 +import pytest + +import pub # noqa -import pub UUID = uuid.uuid4().hex PROJECT = os.environ["GCLOUD_PROJECT"] diff --git a/samples/snippets/quickstart/sub_test.py b/samples/snippets/quickstart/sub_test.py index 1b59a3d04..2754dc56b 100644 --- a/samples/snippets/quickstart/sub_test.py +++ b/samples/snippets/quickstart/sub_test.py @@ -13,16 +13,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -import mock import os -import pytest import uuid from google.api_core.exceptions import AlreadyExists from google.cloud import pubsub_v1 +import mock +import pytest -import sub +import sub # noqa UUID = uuid.uuid4().hex From 1ed1dc9db5648f2fb37452ec122ebfa109548363 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 13 May 2020 07:31:59 +0200 Subject: [PATCH 090/101] chore(deps): update dependency google-cloud-pubsub to v1.4.3 [(#3725)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3725) Co-authored-by: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Co-authored-by: Takashi Matsuo --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index fbfc0f4d3..110c3b79a 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==1.4.2 +google-cloud-pubsub==1.4.3 From 1407699063b564c29726c740a42aec90156bb0d7 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 21 May 2020 04:50:04 +0200 Subject: [PATCH 091/101] chore(deps): update dependency google-cloud-pubsub to v1.5.0 [(#3781)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3781) Co-authored-by: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 110c3b79a..9cc17af84 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==1.4.3 +google-cloud-pubsub==1.5.0 From c93a485d30b547df017b6d36cfa08f2eefe36e78 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Fri, 29 May 2020 17:12:50 -0700 Subject: [PATCH 092/101] samples: add Pub/Sub dead letter queue samples [(#3904)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3904) --- samples/snippets/README.rst | 23 +- samples/snippets/publisher.py | 52 ++-- samples/snippets/subscriber.py | 458 ++++++++++++++++++++-------- samples/snippets/subscriber_test.py | 147 ++++++--- 4 files changed, 471 insertions(+), 209 deletions(-) diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index c30fd190a..f27f9438e 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -74,7 +74,7 @@ To run this sample: .. code-block:: bash - $ python publisher.py + $ python publisher.py --help usage: publisher.py [-h] project_id @@ -124,11 +124,11 @@ To run this sample: .. code-block:: bash - $ python subscriber.py + $ python subscriber.py --help usage: subscriber.py [-h] project_id - {list_in_topic,list_in_project,create,create-push,delete,update,receive,receive-custom-attributes,receive-flow-control,receive-synchronously,receive-synchronously-with-lease,listen_for_errors} + {list-in-topic,list-in-project,create,create-with-dead-letter-policy,create-push,delete,update-push,update-dead-letter-policy,remove-dead-letter-policy,receive,receive-custom-attributes,receive-flow-control,receive-synchronously,receive-synchronously-with-lease,listen-for-errors,receive-messages-with-delivery-attempts} ... This application demonstrates how to perform basic operations on @@ -139,15 +139,21 @@ To run this sample: positional arguments: project_id Your Google Cloud project ID - {list_in_topic,list_in_project,create,create-push,delete,update,receive,receive-custom-attributes,receive-flow-control,receive-synchronously,receive-synchronously-with-lease,listen_for_errors} - list_in_topic Lists all subscriptions for a given topic. - list_in_project Lists all subscriptions in the current project. + {list-in-topic,list-in-project,create,create-with-dead-letter-policy,create-push,delete,update-push,update-dead-letter-policy,remove-dead-letter-policy,receive,receive-custom-attributes,receive-flow-control,receive-synchronously,receive-synchronously-with-lease,listen-for-errors,receive-messages-with-delivery-attempts} + list-in-topic Lists all subscriptions for a given topic. + list-in-project Lists all subscriptions in the current project. create Create a new pull subscription on the given topic. + create-with-dead-letter-policy + Create a subscription with dead letter policy. create-push Create a new push subscription on the given topic. delete Deletes an existing Pub/Sub topic. - update Updates an existing Pub/Sub subscription's push + update-push Updates an existing Pub/Sub subscription's push endpoint URL. Note that certain properties of a subscription, such as its topic, are not modifiable. + update-dead-letter-policy + Update a subscription's dead letter policy. + remove-dead-letter-policy + Remove dead letter policy from a subscription. receive Receives messages from a pull subscription. receive-custom-attributes Receives messages from a pull subscription. @@ -158,8 +164,9 @@ To run this sample: Pulling messages synchronously. receive-synchronously-with-lease Pulling messages synchronously with lease management - listen_for_errors Receives messages and catches errors from a pull + listen-for-errors Receives messages and catches errors from a pull subscription. + receive-messages-with-delivery-attempts optional arguments: -h, --help show this help message and exit diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index 6802ec85f..9e7820fbf 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -29,7 +29,8 @@ def list_topics(project_id): # [START pubsub_list_topics] from google.cloud import pubsub_v1 - # TODO project_id = "Your Google Cloud Project ID" + # TODO(developer) + # project_id = "your-project-id" publisher = pubsub_v1.PublisherClient() project_path = publisher.project_path(project_id) @@ -45,8 +46,9 @@ def create_topic(project_id, topic_name): # [START pubsub_create_topic] from google.cloud import pubsub_v1 - # TODO project_id = "Your Google Cloud Project ID" - # TODO topic_name = "Your Pub/Sub topic name" + # TODO(developer) + # project_id = "your-project-id" + # topic_name = "your-topic-id" publisher = pubsub_v1.PublisherClient() topic_path = publisher.topic_path(project_id, topic_name) @@ -63,8 +65,9 @@ def delete_topic(project_id, topic_name): # [START pubsub_delete_topic] from google.cloud import pubsub_v1 - # TODO project_id = "Your Google Cloud Project ID" - # TODO topic_name = "Your Pub/Sub topic name" + # TODO(developer) + # project_id = "your-project-id" + # topic_name = "your-topic-id" publisher = pubsub_v1.PublisherClient() topic_path = publisher.topic_path(project_id, topic_name) @@ -81,8 +84,9 @@ def publish_messages(project_id, topic_name): # [START pubsub_publish] from google.cloud import pubsub_v1 - # TODO project_id = "Your Google Cloud Project ID" - # TODO topic_name = "Your Pub/Sub topic name" + # TODO(developer) + # project_id = "your-project-id" + # topic_name = "your-topic-id" publisher = pubsub_v1.PublisherClient() # The `topic_path` method creates a fully qualified identifier @@ -108,8 +112,9 @@ def publish_messages_with_custom_attributes(project_id, topic_name): # [START pubsub_publish_custom_attributes] from google.cloud import pubsub_v1 - # TODO project_id = "Your Google Cloud Project ID" - # TODO topic_name = "Your Pub/Sub topic name" + # TODO(developer) + # project_id = "your-project-id" + # topic_name = "your-topic-id" publisher = pubsub_v1.PublisherClient() topic_path = publisher.topic_path(project_id, topic_name) @@ -135,8 +140,9 @@ def publish_messages_with_error_handler(project_id, topic_name): from google.cloud import pubsub_v1 - # TODO project_id = "Your Google Cloud Project ID" - # TODO topic_name = "Your Pub/Sub topic name" + # TODO(developer) + # project_id = "your-project-id" + # topic_name = "your-topic-id" publisher = pubsub_v1.PublisherClient() topic_path = publisher.topic_path(project_id, topic_name) @@ -177,8 +183,9 @@ def publish_messages_with_batch_settings(project_id, topic_name): # [START pubsub_publisher_batch_settings] from google.cloud import pubsub_v1 - # TODO project_id = "Your Google Cloud Project ID" - # TODO topic_name = "Your Pub/Sub topic name" + # TODO(developer) + # project_id = "your-project-id" + # topic_name = "your-topic-id" # Configure the batch to publish as soon as there is ten messages, # one kilobyte of data, or one second has passed. @@ -212,8 +219,9 @@ def publish_messages_with_retry_settings(project_id, topic_name): # [START pubsub_publisher_retry_settings] from google.cloud import pubsub_v1 - # TODO project_id = "Your Google Cloud Project ID" - # TODO topic_name = "Your Pub/Sub topic name" + # TODO(developer) + # project_id = "your-project-id" + # topic_name = "your-topic-id" # Configure the retry settings. Defaults will be overwritten. retry_settings = { @@ -267,8 +275,7 @@ def publish_messages_with_retry_settings(project_id, topic_name): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("project_id", help="Your Google Cloud project ID") @@ -281,9 +288,7 @@ def publish_messages_with_retry_settings(project_id, topic_name): delete_parser = subparsers.add_parser("delete", help=delete_topic.__doc__) delete_parser.add_argument("topic_name") - publish_parser = subparsers.add_parser( - "publish", help=publish_messages.__doc__ - ) + publish_parser = subparsers.add_parser("publish", help=publish_messages.__doc__) publish_parser.add_argument("topic_name") publish_with_custom_attributes_parser = subparsers.add_parser( @@ -293,8 +298,7 @@ def publish_messages_with_retry_settings(project_id, topic_name): publish_with_custom_attributes_parser.add_argument("topic_name") publish_with_error_handler_parser = subparsers.add_parser( - "publish-with-error-handler", - help=publish_messages_with_error_handler.__doc__, + "publish-with-error-handler", help=publish_messages_with_error_handler.__doc__, ) publish_with_error_handler_parser.add_argument("topic_name") @@ -321,9 +325,7 @@ def publish_messages_with_retry_settings(project_id, topic_name): elif args.command == "publish": publish_messages(args.project_id, args.topic_name) elif args.command == "publish-with-custom-attributes": - publish_messages_with_custom_attributes( - args.project_id, args.topic_name - ) + publish_messages_with_custom_attributes(args.project_id, args.topic_name) elif args.command == "publish-with-error-handler": publish_messages_with_error_handler(args.project_id, args.topic_name) elif args.command == "publish-with-batch-settings": diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index e22efc7b1..b5af760ae 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -29,8 +29,9 @@ def list_subscriptions_in_topic(project_id, topic_name): # [START pubsub_list_topic_subscriptions] from google.cloud import pubsub_v1 - # TODO project_id = "Your Google Cloud Project ID" - # TODO topic_name = "Your Pub/Sub topic name" + # TODO(developer) + # project_id = "your-project-id" + # topic_name = "your-topic-id" publisher = pubsub_v1.PublisherClient() topic_path = publisher.topic_path(project_id, topic_name) @@ -45,7 +46,8 @@ def list_subscriptions_in_project(project_id): # [START pubsub_list_subscriptions] from google.cloud import pubsub_v1 - # TODO project_id = "Your Google Cloud Project ID" + # TODO(developer) + # project_id = "your-project-id" subscriber = pubsub_v1.SubscriberClient() project_path = subscriber.project_path(project_id) @@ -62,19 +64,16 @@ def create_subscription(project_id, topic_name, subscription_name): # [START pubsub_create_pull_subscription] from google.cloud import pubsub_v1 - # TODO project_id = "Your Google Cloud Project ID" - # TODO topic_name = "Your Pub/Sub topic name" - # TODO subscription_name = "Your Pub/Sub subscription name" + # TODO(developer) + # project_id = "your-project-id" + # topic_name = "your-topic-id" + # subscription_name = "your-subscription-id" subscriber = pubsub_v1.SubscriberClient() topic_path = subscriber.topic_path(project_id, topic_name) - subscription_path = subscriber.subscription_path( - project_id, subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) - subscription = subscriber.create_subscription( - subscription_path, topic_path - ) + subscription = subscriber.create_subscription(subscription_path, topic_path) print("Subscription created: {}".format(subscription)) @@ -82,23 +81,68 @@ def create_subscription(project_id, topic_name, subscription_name): # [END pubsub_create_pull_subscription] -def create_push_subscription( - project_id, topic_name, subscription_name, endpoint +def create_subscription_with_dead_letter_topic( + project_id, topic_name, subscription_name, dead_letter_topic_name ): + """Create a subscription with dead letter policy.""" + # [START pubsub_dead_letter_create_subscription] + from google.cloud import pubsub_v1 + from google.cloud.pubsub_v1.types import DeadLetterPolicy + + # TODO(developer) + # project_id = "your-project-id" + # endpoint = "https://my-test-project.appspot.com/push" + # TODO(developer): This is an existing topic that the subscription + # with dead letter policy is attached to. + # topic_name = "your-topic-id" + # TODO(developer): This is an existing subscription with a dead letter policy. + # subscription_name = "your-subscription-id" + # TODO(developer): This is an existing dead letter topic that the subscription + # with dead letter policy will forward dead letter messages to. + # dead_letter_topic_name = "your-dead-letter-topic-id" + + subscriber = pubsub_v1.SubscriberClient() + topic_path = subscriber.topic_path(project_id, topic_name) + subscription_path = subscriber.subscription_path(project_id, subscription_name) + dead_letter_topic_path = subscriber.topic_path(project_id, dead_letter_topic_name) + + dead_letter_policy = DeadLetterPolicy( + dead_letter_topic=dead_letter_topic_path, max_delivery_attempts=10 + ) + + with subscriber: + subscription = subscriber.create_subscription( + subscription_path, topic_path, dead_letter_policy=dead_letter_policy + ) + + print("Subscription created: {}".format(subscription.name)) + print( + "It will forward dead letter messages to: {}".format( + subscription.dead_letter_policy.dead_letter_topic + ) + ) + print( + "After {} delivery attempts.".format( + subscription.dead_letter_policy.max_delivery_attempts + ) + ) + # [END pubsub_dead_letter_create_subscription] + + +def create_push_subscription(project_id, topic_name, subscription_name, endpoint): """Create a new push subscription on the given topic.""" # [START pubsub_create_push_subscription] from google.cloud import pubsub_v1 - # TODO project_id = "Your Google Cloud Project ID" - # TODO topic_name = "Your Pub/Sub topic name" - # TODO subscription_name = "Your Pub/Sub subscription name" - # TODO endpoint = "https://my-test-project.appspot.com/push" + # TODO(developer) + # project_id = "your-project-id" + # topic_name = "your-topic-id" + # subscription_name = "your-subscription-id" + # endpoint = "https://my-test-project.appspot.com/push" subscriber = pubsub_v1.SubscriberClient() topic_path = subscriber.topic_path(project_id, topic_name) - subscription_path = subscriber.subscription_path( - project_id, subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) push_config = pubsub_v1.types.PushConfig(push_endpoint=endpoint) @@ -118,13 +162,12 @@ def delete_subscription(project_id, subscription_name): # [START pubsub_delete_subscription] from google.cloud import pubsub_v1 - # TODO project_id = "Your Google Cloud Project ID" - # TODO subscription_name = "Your Pub/Sub subscription name" + # TODO(developer) + # project_id = "your-project-id" + # subscription_name = "your-subscription-id" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) subscriber.delete_subscription(subscription_path) @@ -134,7 +177,7 @@ def delete_subscription(project_id, subscription_name): # [END pubsub_delete_subscription] -def update_subscription(project_id, subscription_name, endpoint): +def update_push_subscription(project_id, topic_name, subscription_name, endpoint): """ Updates an existing Pub/Sub subscription's push endpoint URL. Note that certain properties of a subscription, such as @@ -143,26 +186,24 @@ def update_subscription(project_id, subscription_name, endpoint): # [START pubsub_update_push_configuration] from google.cloud import pubsub_v1 - # TODO project_id = "Your Google Cloud Project ID" - # TODO topic_name = "Your Pub/Sub topic name" - # TODO subscription_name = "Your Pub/Sub subscription name" - # TODO endpoint = "https://my-test-project.appspot.com/push" + # TODO(developer) + # project_id = "your-project-id" + # topic_name = "your-topic-id" + # subscription_name = "your-subscription-id" + # endpoint = "https://my-test-project.appspot.com/push" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) push_config = pubsub_v1.types.PushConfig(push_endpoint=endpoint) subscription = pubsub_v1.types.Subscription( - name=subscription_path, push_config=push_config + name=subscription_path, topic=topic_name, push_config=push_config ) update_mask = {"paths": {"push_config"}} - subscriber.update_subscription(subscription, update_mask) - result = subscriber.get_subscription(subscription_path) + result = subscriber.update_subscription(subscription, update_mask) print("Subscription updated: {}".format(subscription_path)) print("New endpoint for subscription is: {}".format(result.push_config)) @@ -171,31 +212,126 @@ def update_subscription(project_id, subscription_name, endpoint): # [END pubsub_update_push_configuration] +def update_subscription_with_dead_letter_policy( + project_id, topic_name, subscription_name, dead_letter_topic_name +): + """Update a subscription's dead letter policy.""" + # [START pubsub_dead_letter_update_subscription] + from google.cloud import pubsub_v1 + from google.cloud.pubsub_v1.types import DeadLetterPolicy, FieldMask + + # TODO(developer) + # project_id = "your-project-id" + # TODO(developer): This is an existing topic that the subscription + # with dead letter policy is attached to. + # topic_name = "your-topic-name" + # TODO(developer): This is an existing subscription with a dead letter policy. + # subscription_name = "your-subscription-id" + # TODO(developer): This is an existing dead letter topic that the subscription + # with dead letter policy will forward dead letter messages to. + # dead_letter_topic_name = "your-dead-letter-topic-id" + + subscriber = pubsub_v1.SubscriberClient() + topic_path = subscriber.topic_path(project_id, topic_name) + subscription_path = subscriber.subscription_path(project_id, subscription_name) + dead_letter_topic_path = subscriber.topic_path(project_id, dead_letter_topic_name) + + subscription_before_update = subscriber.get_subscription(subscription_path) + print("Before the update: {}".format(subscription_before_update)) + + # Indicates which fields in the provided subscription to update. + update_mask = FieldMask(paths=["dead_letter_policy.max_delivery_attempts"]) + + # Construct a dead letter policy you expect to have after the update. + dead_letter_policy = DeadLetterPolicy( + dead_letter_topic=dead_letter_topic_path, max_delivery_attempts=20 + ) + + # Construct the subscription with the dead letter policy you expect to have + # after the update. Here, values in the required fields (name, topic) help + # identify the subscription. + subscription = pubsub_v1.types.Subscription( + name=subscription_path, topic=topic_path, dead_letter_policy=dead_letter_policy, + ) + + with subscriber: + subscription_after_update = subscriber.update_subscription( + subscription, update_mask + ) + + print("After the update: {}".format(subscription_after_update)) + # [END pubsub_dead_letter_update_subscription] + return subscription_after_update + + +def remove_dead_letter_policy(project_id, topic_name, subscription_name): + """Remove dead letter policy from a subscription.""" + # [START pubsub_dead_letter_remove] + from google.cloud import pubsub_v1 + from google.cloud.pubsub_v1.types import FieldMask + + # TODO(developer) + # project_id = "your-project-id" + # TODO(developer): This is an existing topic that the subscription + # with dead letter policy is attached to. + # topic_name = "your-topic-name" + # TODO(developer): This is an existing subscription with a dead letter policy. + # subscription_name = "your-subscription-id" + + subscriber = pubsub_v1.SubscriberClient() + topic_path = subscriber.topic_path(project_id, topic_name) + subscription_path = subscriber.subscription_path(project_id, subscription_name) + + subscription_before_update = subscriber.get_subscription(subscription_path) + print("Before removing the policy: {}".format(subscription_before_update)) + + # Indicates which fields in the provided subscription to update. + update_mask = FieldMask( + paths=[ + "dead_letter_policy.dead_letter_topic", + "dead_letter_policy.max_delivery_attempts", + ] + ) + + # Construct the subscription (without any dead letter policy) that you + # expect to have after the update. + subscription = pubsub_v1.types.Subscription( + name=subscription_path, topic=topic_path + ) + + with subscriber: + subscription_after_update = subscriber.update_subscription( + subscription, update_mask + ) + + print("After removing the policy: {}".format(subscription_after_update)) + # [END pubsub_dead_letter_remove] + return subscription_after_update + + def receive_messages(project_id, subscription_name, timeout=None): """Receives messages from a pull subscription.""" # [START pubsub_subscriber_async_pull] # [START pubsub_quickstart_subscriber] + from concurrent.futures import TimeoutError from google.cloud import pubsub_v1 - # TODO project_id = "Your Google Cloud Project ID" - # TODO subscription_name = "Your Pub/Sub subscription name" - # TODO timeout = 5.0 # "How long the subscriber should listen for - # messages in seconds" + # TODO(developer) + # project_id = "your-project-id" + # subscription_name = "your-subscription-id" + # Number of seconds the subscriber should listen for messages + # timeout = 5.0 subscriber = pubsub_v1.SubscriberClient() # The `subscription_path` method creates a fully qualified identifier # in the form `projects/{project_id}/subscriptions/{subscription_name}` - subscription_path = subscriber.subscription_path( - project_id, subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) def callback(message): print("Received message: {}".format(message)) message.ack() - streaming_pull_future = subscriber.subscribe( - subscription_path, callback=callback - ) + streaming_pull_future = subscriber.subscribe(subscription_path, callback=callback) print("Listening for messages on {}..\n".format(subscription_path)) # Wrap subscriber in a 'with' block to automatically call close() when done. @@ -204,7 +340,7 @@ def callback(message): # When `timeout` is not set, result() will block indefinitely, # unless an exception is encountered first. streaming_pull_future.result(timeout=timeout) - except: # noqa + except TimeoutError: streaming_pull_future.cancel() # [END pubsub_subscriber_async_pull] # [END pubsub_quickstart_subscriber] @@ -216,17 +352,17 @@ def receive_messages_with_custom_attributes( """Receives messages from a pull subscription.""" # [START pubsub_subscriber_sync_pull_custom_attributes] # [START pubsub_subscriber_async_pull_custom_attributes] + from concurrent.futures import TimeoutError from google.cloud import pubsub_v1 - # TODO project_id = "Your Google Cloud Project ID" - # TODO subscription_name = "Your Pub/Sub subscription name" - # TODO timeout = 5.0 # "How long the subscriber should listen for - # messages in seconds" + # TODO(developer) + # project_id = "your-project-id" + # subscription_name = "your-subscription-id" + # Number of seconds the subscriber should listen for messages + # timeout = 5.0 subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) def callback(message): print("Received message: {}".format(message.data)) @@ -237,9 +373,7 @@ def callback(message): print("{}: {}".format(key, value)) message.ack() - streaming_pull_future = subscriber.subscribe( - subscription_path, callback=callback - ) + streaming_pull_future = subscriber.subscribe(subscription_path, callback=callback) print("Listening for messages on {}..\n".format(subscription_path)) # Wrap subscriber in a 'with' block to automatically call close() when done. @@ -248,28 +382,26 @@ def callback(message): # When `timeout` is not set, result() will block indefinitely, # unless an exception is encountered first. streaming_pull_future.result(timeout=timeout) - except: # noqa + except TimeoutError: streaming_pull_future.cancel() # [END pubsub_subscriber_async_pull_custom_attributes] # [END pubsub_subscriber_sync_pull_custom_attributes] -def receive_messages_with_flow_control( - project_id, subscription_name, timeout=None -): +def receive_messages_with_flow_control(project_id, subscription_name, timeout=None): """Receives messages from a pull subscription with flow control.""" # [START pubsub_subscriber_flow_settings] + from concurrent.futures import TimeoutError from google.cloud import pubsub_v1 - # TODO project_id = "Your Google Cloud Project ID" - # TODO subscription_name = "Your Pub/Sub subscription name" - # TODO timeout = 5.0 # "How long the subscriber should listen for - # messages in seconds" + # TODO(developer) + # project_id = "your-project-id" + # subscription_name = "your-subscription-id" + # Number of seconds the subscriber should listen for messages + # timeout = 5.0 subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) def callback(message): print("Received message: {}".format(message.data)) @@ -289,7 +421,7 @@ def callback(message): # When `timeout` is not set, result() will block indefinitely, # unless an exception is encountered first. streaming_pull_future.result(timeout=timeout) - except: # noqa + except TimeoutError: streaming_pull_future.cancel() # [END pubsub_subscriber_flow_settings] @@ -299,13 +431,12 @@ def synchronous_pull(project_id, subscription_name): # [START pubsub_subscriber_sync_pull] from google.cloud import pubsub_v1 - # TODO project_id = "Your Google Cloud Project ID" - # TODO subscription_name = "Your Pub/Sub subscription name" + # TODO(developer) + # project_id = "your-project-id" + # subscription_name = "your-subscription-id" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) NUM_MESSAGES = 3 @@ -340,13 +471,12 @@ def synchronous_pull_with_lease_management(project_id, subscription_name): from google.cloud import pubsub_v1 - # TODO project_id = "Your Google Cloud Project ID" - # TODO subscription_name = "Your Pub/Sub subscription name" + # TODO(developer) + # project_id = "your-project-id" + # subscription_name = "your-subscription-id" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) NUM_MESSAGES = 2 ACK_DEADLINE = 30 @@ -385,15 +515,11 @@ def worker(msg): if process.is_alive(): # `ack_deadline_seconds` must be between 10 to 600. subscriber.modify_ack_deadline( - subscription_path, - [ack_id], - ack_deadline_seconds=ACK_DEADLINE, + subscription_path, [ack_id], ack_deadline_seconds=ACK_DEADLINE, ) logger.info( "{}: Reset ack deadline for {} for {}s".format( - time.strftime("%X", time.gmtime()), - msg_data, - ACK_DEADLINE, + time.strftime("%X", time.gmtime()), msg_data, ACK_DEADLINE, ) ) @@ -426,23 +552,20 @@ def listen_for_errors(project_id, subscription_name, timeout=None): # [START pubsub_subscriber_error_listener] from google.cloud import pubsub_v1 - # TODO project_id = "Your Google Cloud Project ID" - # TODO subscription_name = "Your Pubsub subscription name" - # TODO timeout = 5.0 # "How long the subscriber should listen for - # messages in seconds" + # TODO(developer) + # project_id = "Your Google Cloud Project ID" + # subscription_name = "Your Pubsub subscription name" + # Number of seconds the subscriber should listen for messages + # timeout = 5.0 subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) def callback(message): print("Received message: {}".format(message)) message.ack() - streaming_pull_future = subscriber.subscribe( - subscription_path, callback=callback - ) + streaming_pull_future = subscriber.subscribe(subscription_path, callback=callback) print("Listening for messages on {}..\n".format(subscription_path)) # Wrap subscriber in a 'with' block to automatically call close() when done. @@ -461,29 +584,67 @@ def callback(message): # [END pubsub_subscriber_error_listener] +def receive_messages_with_delivery_attempts( + project_id, subscription_name, timeout=None +): + # [START pubsub_dead_letter_delivery_attempt] + from concurrent.futures import TimeoutError + from google.cloud import pubsub_v1 + + # TODO(developer) + # project_id = "your-project-id" + # subscription_name = "your-subscription-id" + + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path(project_id, subscription_name) + + def callback(message): + print("Received message: {}".format(message)) + print("With delivery attempts: {}".format(message.delivery_attempt)) + message.ack() + + streaming_pull_future = subscriber.subscribe(subscription_path, callback=callback) + print("Listening for messages on {}..\n".format(subscription_path)) + + # Wrap subscriber in a 'with' block to automatically call close() when done. + with subscriber: + # When `timeout` is not set, result() will block indefinitely, + # unless an exception is encountered first. + try: + streaming_pull_future.result(timeout=timeout) + except TimeoutError: + streaming_pull_future.cancel() + # [END pubsub_dead_letter_delivery_attempt] + + if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("project_id", help="Your Google Cloud project ID") subparsers = parser.add_subparsers(dest="command") list_in_topic_parser = subparsers.add_parser( - "list_in_topic", help=list_subscriptions_in_topic.__doc__ + "list-in-topic", help=list_subscriptions_in_topic.__doc__ ) list_in_topic_parser.add_argument("topic_name") list_in_project_parser = subparsers.add_parser( - "list_in_project", help=list_subscriptions_in_project.__doc__ + "list-in-project", help=list_subscriptions_in_project.__doc__ ) - create_parser = subparsers.add_parser( - "create", help=create_subscription.__doc__ - ) + create_parser = subparsers.add_parser("create", help=create_subscription.__doc__) create_parser.add_argument("topic_name") create_parser.add_argument("subscription_name") + create_with_dead_letter_policy_parser = subparsers.add_parser( + "create-with-dead-letter-policy", + help=create_subscription_with_dead_letter_topic.__doc__, + ) + create_with_dead_letter_policy_parser.add_argument("topic_name") + create_with_dead_letter_policy_parser.add_argument("subscription_name") + create_with_dead_letter_policy_parser.add_argument("dead_letter_topic_name") + create_push_parser = subparsers.add_parser( "create-push", help=create_push_subscription.__doc__ ) @@ -491,20 +652,31 @@ def callback(message): create_push_parser.add_argument("subscription_name") create_push_parser.add_argument("endpoint") - delete_parser = subparsers.add_parser( - "delete", help=delete_subscription.__doc__ - ) + delete_parser = subparsers.add_parser("delete", help=delete_subscription.__doc__) delete_parser.add_argument("subscription_name") - update_parser = subparsers.add_parser( - "update", help=update_subscription.__doc__ + update_push_parser = subparsers.add_parser( + "update-push", help=update_push_subscription.__doc__ + ) + update_push_parser.add_argument("topic_name") + update_push_parser.add_argument("subscription_name") + update_push_parser.add_argument("endpoint") + + update_dead_letter_policy_parser = subparsers.add_parser( + "update-dead-letter-policy", + help=update_subscription_with_dead_letter_policy.__doc__, ) - update_parser.add_argument("subscription_name") - update_parser.add_argument("endpoint") + update_dead_letter_policy_parser.add_argument("topic_name") + update_dead_letter_policy_parser.add_argument("subscription_name") + update_dead_letter_policy_parser.add_argument("dead_letter_topic_name") - receive_parser = subparsers.add_parser( - "receive", help=receive_messages.__doc__ + remove_dead_letter_policy_parser = subparsers.add_parser( + "remove-dead-letter-policy", help=remove_dead_letter_policy.__doc__ ) + remove_dead_letter_policy_parser.add_argument("topic_name") + remove_dead_letter_policy_parser.add_argument("subscription_name") + + receive_parser = subparsers.add_parser("receive", help=receive_messages.__doc__) receive_parser.add_argument("subscription_name") receive_parser.add_argument("--timeout", default=None, type=float) @@ -521,9 +693,7 @@ def callback(message): "receive-flow-control", help=receive_messages_with_flow_control.__doc__ ) receive_with_flow_control_parser.add_argument("subscription_name") - receive_with_flow_control_parser.add_argument( - "--timeout", default=None, type=float - ) + receive_with_flow_control_parser.add_argument("--timeout", default=None, type=float) synchronous_pull_parser = subparsers.add_parser( "receive-synchronously", help=synchronous_pull.__doc__ @@ -534,40 +704,58 @@ def callback(message): "receive-synchronously-with-lease", help=synchronous_pull_with_lease_management.__doc__, ) - synchronous_pull_with_lease_management_parser.add_argument( - "subscription_name" - ) + synchronous_pull_with_lease_management_parser.add_argument("subscription_name") listen_for_errors_parser = subparsers.add_parser( - "listen_for_errors", help=listen_for_errors.__doc__ + "listen-for-errors", help=listen_for_errors.__doc__ ) listen_for_errors_parser.add_argument("subscription_name") - listen_for_errors_parser.add_argument( + listen_for_errors_parser.add_argument("--timeout", default=None, type=float) + + receive_messages_with_delivery_attempts_parser = subparsers.add_parser( + "receive-messages-with-delivery-attempts", + help=receive_messages_with_delivery_attempts.__doc__, + ) + receive_messages_with_delivery_attempts_parser.add_argument("subscription_name") + receive_messages_with_delivery_attempts_parser.add_argument( "--timeout", default=None, type=float ) args = parser.parse_args() - if args.command == "list_in_topic": + if args.command == "list-in-topic": list_subscriptions_in_topic(args.project_id, args.topic_name) - elif args.command == "list_in_project": + elif args.command == "list-in-project": list_subscriptions_in_project(args.project_id) elif args.command == "create": - create_subscription( - args.project_id, args.topic_name, args.subscription_name + create_subscription(args.project_id, args.topic_name, args.subscription_name) + elif args.command == "create-with-dead-letter-policy": + create_subscription_with_dead_letter_topic( + args.project_id, + args.topic_name, + args.subscription_name, + args.dead_letter_topic_name, ) elif args.command == "create-push": create_push_subscription( + args.project_id, args.topic_name, args.subscription_name, args.endpoint, + ) + elif args.command == "delete": + delete_subscription(args.project_id, args.subscription_name) + elif args.command == "update-push": + update_push_subscription( + args.project_id, args.topic_name, args.subscription_name, args.endpoint, + ) + elif args.command == "update-dead-letter-policy": + update_subscription_with_dead_letter_policy( args.project_id, args.topic_name, args.subscription_name, - args.endpoint, + args.dead_letter_topic_name, ) - elif args.command == "delete": - delete_subscription(args.project_id, args.subscription_name) - elif args.command == "update": - update_subscription( - args.project_id, args.subscription_name, args.endpoint + elif args.command == "remove-dead-letter-policy": + remove_dead_letter_policy( + args.project_id, args.topic_name, args.subscription_name ) elif args.command == "receive": receive_messages(args.project_id, args.subscription_name, args.timeout) @@ -582,10 +770,10 @@ def callback(message): elif args.command == "receive-synchronously": synchronous_pull(args.project_id, args.subscription_name) elif args.command == "receive-synchronously-with-lease": - synchronous_pull_with_lease_management( - args.project_id, args.subscription_name - ) - elif args.command == "listen_for_errors": - listen_for_errors( + synchronous_pull_with_lease_management(args.project_id, args.subscription_name) + elif args.command == "listen-for-errors": + listen_for_errors(args.project_id, args.subscription_name, args.timeout) + elif args.command == "receive-messages-with-delivery-attempts": + receive_messages_with_delivery_attempts( args.project_id, args.subscription_name, args.timeout ) diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index 1c9520866..6b90396f9 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -24,9 +24,11 @@ UUID = uuid.uuid4().hex PROJECT = os.environ["GCLOUD_PROJECT"] TOPIC = "subscription-test-topic-" + UUID +DEAD_LETTER_TOPIC = "subscription-test-dead-letter-topic-" + UUID SUBSCRIPTION_ADMIN = "subscription-test-subscription-admin-" + UUID SUBSCRIPTION_ASYNC = "subscription-test-subscription-async-" + UUID SUBSCRIPTION_SYNC = "subscription-test-subscription-sync-" + UUID +SUBSCRIPTION_DLQ = "subscription-test-subscription-dlq-" + UUID ENDPOINT = "https://{}.appspot.com/push".format(PROJECT) NEW_ENDPOINT = "https://{}.appspot.com/push2".format(PROJECT) @@ -41,13 +43,27 @@ def topic(publisher_client): topic_path = publisher_client.topic_path(PROJECT, TOPIC) try: - subscription = publisher_client.get_topic(topic_path) + topic = publisher_client.get_topic(topic_path) except: # noqa - subscription = publisher_client.create_topic(topic_path) + topic = publisher_client.create_topic(topic_path) - yield subscription.name + yield topic.name + + publisher_client.delete_topic(topic.name) + + +@pytest.fixture(scope="module") +def dead_letter_topic(publisher_client): + topic_path = publisher_client.topic_path(PROJECT, DEAD_LETTER_TOPIC) + + try: + dead_letter_topic = publisher_client.get_topic(topic_path) + except: # noqa + dead_letter_topic = publisher_client.create_topic(topic_path) + + yield dead_letter_topic.name - publisher_client.delete_topic(subscription.name) + publisher_client.delete_topic(dead_letter_topic.name) @pytest.fixture(scope="module") @@ -59,9 +75,7 @@ def subscriber_client(): @pytest.fixture(scope="module") def subscription_admin(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_ADMIN - ) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_ADMIN) try: subscription = subscriber_client.get_subscription(subscription_path) @@ -75,9 +89,7 @@ def subscription_admin(subscriber_client, topic): @pytest.fixture(scope="module") def subscription_sync(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_SYNC - ) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_SYNC) try: subscription = subscriber_client.get_subscription(subscription_path) @@ -93,9 +105,23 @@ def subscription_sync(subscriber_client, topic): @pytest.fixture(scope="module") def subscription_async(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_ASYNC - ) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_ASYNC) + + try: + subscription = subscriber_client.get_subscription(subscription_path) + except: # noqa + subscription = subscriber_client.create_subscription( + subscription_path, topic=topic + ) + + yield subscription.name + + subscriber_client.delete_subscription(subscription.name) + + +@pytest.fixture(scope="module") +def subscription_dlq(subscriber_client, topic): + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_DLQ) try: subscription = subscriber_client.get_subscription(subscription_path) @@ -130,9 +156,7 @@ def eventually_consistent_test(): def test_create(subscriber_client): - subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_ADMIN - ) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_ADMIN) try: subscriber_client.delete_subscription(subscription_path) @@ -148,19 +172,36 @@ def eventually_consistent_test(): eventually_consistent_test() -def test_create_push(subscriber_client): - subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_ADMIN - ) +def test_create_subscription_with_dead_letter_policy( + subscriber_client, publisher_client, topic, dead_letter_topic, capsys +): + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_DLQ) + dead_letter_topic_path = publisher_client.topic_path(PROJECT, DEAD_LETTER_TOPIC) + try: subscriber_client.delete_subscription(subscription_path) except Exception: pass - subscriber.create_push_subscription( - PROJECT, TOPIC, SUBSCRIPTION_ADMIN, ENDPOINT + subscriber.create_subscription_with_dead_letter_topic( + PROJECT, TOPIC, SUBSCRIPTION_DLQ, DEAD_LETTER_TOPIC ) + out, _ = capsys.readouterr() + assert "Subscription created: " + subscription_path in out + assert "It will forward dead letter messages to: " + dead_letter_topic_path in out + assert "After 10 delivery attempts." in out + + +def test_create_push(subscriber_client): + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_ADMIN) + try: + subscriber_client.delete_subscription(subscription_path) + except Exception: + pass + + subscriber.create_push_subscription(PROJECT, TOPIC, SUBSCRIPTION_ADMIN, ENDPOINT) + @backoff.on_exception(backoff.expo, AssertionError, max_time=60) def eventually_consistent_test(): assert subscriber_client.get_subscription(subscription_path) @@ -169,12 +210,25 @@ def eventually_consistent_test(): def test_update(subscriber_client, subscription_admin, capsys): - subscriber.update_subscription(PROJECT, SUBSCRIPTION_ADMIN, NEW_ENDPOINT) + subscriber.update_push_subscription( + PROJECT, TOPIC, SUBSCRIPTION_ADMIN, NEW_ENDPOINT + ) out, _ = capsys.readouterr() assert "Subscription updated" in out +def test_update_dead_letter_policy( + subscriber_client, topic, subscription_dlq, dead_letter_topic, capsys +): + _ = subscriber.update_subscription_with_dead_letter_policy( + PROJECT, TOPIC, SUBSCRIPTION_DLQ, DEAD_LETTER_TOPIC + ) + + out, _ = capsys.readouterr() + assert "max_delivery_attempts: 20" in out + + def test_delete(subscriber_client, subscription_admin): subscriber.delete_subscription(PROJECT, SUBSCRIPTION_ADMIN) @@ -212,9 +266,7 @@ def test_receive_with_custom_attributes( _publish_messages(publisher_client, topic) - subscriber.receive_messages_with_custom_attributes( - PROJECT, SUBSCRIPTION_ASYNC, 5 - ) + subscriber.receive_messages_with_custom_attributes(PROJECT, SUBSCRIPTION_ASYNC, 5) out, _ = capsys.readouterr() assert "message" in out @@ -222,15 +274,11 @@ def test_receive_with_custom_attributes( assert "python-sample" in out -def test_receive_with_flow_control( - publisher_client, topic, subscription_async, capsys -): +def test_receive_with_flow_control(publisher_client, topic, subscription_async, capsys): _publish_messages(publisher_client, topic) - subscriber.receive_messages_with_flow_control( - PROJECT, SUBSCRIPTION_ASYNC, 5 - ) + subscriber.receive_messages_with_flow_control(PROJECT, SUBSCRIPTION_ASYNC, 5) out, _ = capsys.readouterr() assert "Listening" in out @@ -238,9 +286,7 @@ def test_receive_with_flow_control( assert "message" in out -def test_receive_synchronously( - publisher_client, topic, subscription_sync, capsys -): +def test_receive_synchronously(publisher_client, topic, subscription_sync, capsys): _publish_messages(publisher_client, topic) subscriber.synchronous_pull(PROJECT, SUBSCRIPTION_SYNC) @@ -254,17 +300,13 @@ def test_receive_synchronously_with_lease( ): _publish_messages(publisher_client, topic) - subscriber.synchronous_pull_with_lease_management( - PROJECT, SUBSCRIPTION_SYNC - ) + subscriber.synchronous_pull_with_lease_management(PROJECT, SUBSCRIPTION_SYNC) out, _ = capsys.readouterr() assert "Done." in out -def test_listen_for_errors( - publisher_client, topic, subscription_async, capsys -): +def test_listen_for_errors(publisher_client, topic, subscription_async, capsys): _publish_messages(publisher_client, topic) @@ -274,3 +316,26 @@ def test_listen_for_errors( assert "Listening" in out assert subscription_async in out assert "threw an exception" in out + + +def test_receive_with_delivery_attempts( + publisher_client, topic, subscription_dlq, dead_letter_topic, capsys +): + _publish_messages(publisher_client, topic) + + subscriber.receive_messages_with_delivery_attempts(PROJECT, SUBSCRIPTION_DLQ, 10) + + out, _ = capsys.readouterr() + assert "Listening" in out + assert subscription_dlq in out + assert "Received message: " in out + assert "message 4" in out + assert "With delivery attempts: " in out + + +def test_remove_dead_letter_policy(subscriber_client, subscription_dlq): + subscription_after_update = subscriber.remove_dead_letter_policy( + PROJECT, TOPIC, SUBSCRIPTION_DLQ + ) + + assert subscription_after_update.dead_letter_policy.dead_letter_topic == "" From 47e051dcf2b58ced0e86cc067e8b20eb1989cc44 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Tue, 2 Jun 2020 17:04:56 -0700 Subject: [PATCH 093/101] fix: make timeout an optional positional arg [(#3938)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3938) * fix: make timeout an optional positional arg * place `none` back in function signature Co-authored-by: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> --- samples/snippets/subscriber.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index b5af760ae..5dc468e23 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -678,7 +678,7 @@ def callback(message): receive_parser = subparsers.add_parser("receive", help=receive_messages.__doc__) receive_parser.add_argument("subscription_name") - receive_parser.add_argument("--timeout", default=None, type=float) + receive_parser.add_argument("timeout", default=None, type=float, nargs="?") receive_with_custom_attributes_parser = subparsers.add_parser( "receive-custom-attributes", @@ -686,14 +686,16 @@ def callback(message): ) receive_with_custom_attributes_parser.add_argument("subscription_name") receive_with_custom_attributes_parser.add_argument( - "--timeout", default=None, type=float + "timeout", default=None, type=float, nargs="?" ) receive_with_flow_control_parser = subparsers.add_parser( "receive-flow-control", help=receive_messages_with_flow_control.__doc__ ) receive_with_flow_control_parser.add_argument("subscription_name") - receive_with_flow_control_parser.add_argument("--timeout", default=None, type=float) + receive_with_flow_control_parser.add_argument( + "timeout", default=None, type=float, nargs="?" + ) synchronous_pull_parser = subparsers.add_parser( "receive-synchronously", help=synchronous_pull.__doc__ @@ -710,7 +712,9 @@ def callback(message): "listen-for-errors", help=listen_for_errors.__doc__ ) listen_for_errors_parser.add_argument("subscription_name") - listen_for_errors_parser.add_argument("--timeout", default=None, type=float) + listen_for_errors_parser.add_argument( + "timeout", default=None, type=float, nargs="?" + ) receive_messages_with_delivery_attempts_parser = subparsers.add_parser( "receive-messages-with-delivery-attempts", @@ -718,7 +722,7 @@ def callback(message): ) receive_messages_with_delivery_attempts_parser.add_argument("subscription_name") receive_messages_with_delivery_attempts_parser.add_argument( - "--timeout", default=None, type=float + "timeout", default=None, type=float, nargs="?" ) args = parser.parse_args() From fa1c813116cb71f895355d7bcc0e0e6bd2dee7b9 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Thu, 4 Jun 2020 13:46:07 -0700 Subject: [PATCH 094/101] fix: replace name with id in samples [(#3953)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3953) --- samples/snippets/iam.py | 112 +++++++----- samples/snippets/iam_test.py | 4 +- samples/snippets/publisher.py | 72 ++++---- samples/snippets/quickstart/pub.py | 13 +- samples/snippets/quickstart/sub.py | 19 +-- samples/snippets/quickstart/sub_test.py | 8 +- samples/snippets/subscriber.py | 218 ++++++++++++------------ 7 files changed, 225 insertions(+), 221 deletions(-) diff --git a/samples/snippets/iam.py b/samples/snippets/iam.py index eb0c82463..71c55d764 100644 --- a/samples/snippets/iam.py +++ b/samples/snippets/iam.py @@ -23,14 +23,18 @@ import argparse -from google.cloud import pubsub_v1 - -def get_topic_policy(project, topic_name): +def get_topic_policy(project, topic_id): """Prints the IAM policy for the given topic.""" # [START pubsub_get_topic_policy] + from google.cloud import pubsub_v1 + + # TODO(developer) + # project_id = "your-project-id" + # topic_id = "your-topic-id" + client = pubsub_v1.PublisherClient() - topic_path = client.topic_path(project, topic_name) + topic_path = client.topic_path(project, topic_id) policy = client.get_iam_policy(topic_path) @@ -40,11 +44,17 @@ def get_topic_policy(project, topic_name): # [END pubsub_get_topic_policy] -def get_subscription_policy(project, subscription_name): +def get_subscription_policy(project, subscription_id): """Prints the IAM policy for the given subscription.""" # [START pubsub_get_subscription_policy] + from google.cloud import pubsub_v1 + + # TODO(developer) + # project_id = "your-project-id" + # subscription_id = "your-subscription-id" + client = pubsub_v1.SubscriberClient() - subscription_path = client.subscription_path(project, subscription_name) + subscription_path = client.subscription_path(project, subscription_id) policy = client.get_iam_policy(subscription_path) @@ -56,11 +66,17 @@ def get_subscription_policy(project, subscription_name): # [END pubsub_get_subscription_policy] -def set_topic_policy(project, topic_name): +def set_topic_policy(project, topic_id): """Sets the IAM policy for a topic.""" # [START pubsub_set_topic_policy] + from google.cloud import pubsub_v1 + + # TODO(developer) + # project_id = "your-project-id" + # topic_id = "your-topic-id" + client = pubsub_v1.PublisherClient() - topic_path = client.topic_path(project, topic_name) + topic_path = client.topic_path(project, topic_id) policy = client.get_iam_policy(topic_path) @@ -75,15 +91,21 @@ def set_topic_policy(project, topic_name): # Set the policy policy = client.set_iam_policy(topic_path, policy) - print("IAM policy for topic {} set: {}".format(topic_name, policy)) + print("IAM policy for topic {} set: {}".format(topic_id, policy)) # [END pubsub_set_topic_policy] -def set_subscription_policy(project, subscription_name): +def set_subscription_policy(project, subscription_id): """Sets the IAM policy for a topic.""" # [START pubsub_set_subscription_policy] + from google.cloud import pubsub_v1 + + # TODO(developer) + # project_id = "your-project-id" + # subscription_id = "your-subscription-id" + client = pubsub_v1.SubscriberClient() - subscription_path = client.subscription_path(project, subscription_name) + subscription_path = client.subscription_path(project, subscription_id) policy = client.get_iam_policy(subscription_path) @@ -91,48 +113,50 @@ def set_subscription_policy(project, subscription_name): policy.bindings.add(role="roles/pubsub.viewer", members=["allUsers"]) # Add a group as an editor. - policy.bindings.add( - role="roles/editor", members=["group:cloud-logs@google.com"] - ) + policy.bindings.add(role="roles/editor", members=["group:cloud-logs@google.com"]) # Set the policy policy = client.set_iam_policy(subscription_path, policy) - print( - "IAM policy for subscription {} set: {}".format( - subscription_name, policy - ) - ) + print("IAM policy for subscription {} set: {}".format(subscription_id, policy)) client.close() # [END pubsub_set_subscription_policy] -def check_topic_permissions(project, topic_name): +def check_topic_permissions(project, topic_id): """Checks to which permissions are available on the given topic.""" # [START pubsub_test_topic_permissions] + from google.cloud import pubsub_v1 + + # TODO(developer) + # project_id = "your-project-id" + # topic_id = "your-topic-id" + client = pubsub_v1.PublisherClient() - topic_path = client.topic_path(project, topic_name) + topic_path = client.topic_path(project, topic_id) permissions_to_check = ["pubsub.topics.publish", "pubsub.topics.update"] - allowed_permissions = client.test_iam_permissions( - topic_path, permissions_to_check - ) + allowed_permissions = client.test_iam_permissions(topic_path, permissions_to_check) print( - "Allowed permissions for topic {}: {}".format( - topic_path, allowed_permissions - ) + "Allowed permissions for topic {}: {}".format(topic_path, allowed_permissions) ) # [END pubsub_test_topic_permissions] -def check_subscription_permissions(project, subscription_name): +def check_subscription_permissions(project, subscription_id): """Checks to which permissions are available on the given subscription.""" # [START pubsub_test_subscription_permissions] + from google.cloud import pubsub_v1 + + # TODO(developer) + # project_id = "your-project-id" + # subscription_id = "your-subscription-id" + client = pubsub_v1.SubscriberClient() - subscription_path = client.subscription_path(project, subscription_name) + subscription_path = client.subscription_path(project, subscription_id) permissions_to_check = [ "pubsub.subscriptions.consume", @@ -155,8 +179,7 @@ def check_subscription_permissions(project, subscription_name): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("project", help="Your Google Cloud project ID") @@ -165,45 +188,44 @@ def check_subscription_permissions(project, subscription_name): get_topic_policy_parser = subparsers.add_parser( "get-topic-policy", help=get_topic_policy.__doc__ ) - get_topic_policy_parser.add_argument("topic_name") + get_topic_policy_parser.add_argument("topic_id") get_subscription_policy_parser = subparsers.add_parser( "get-subscription-policy", help=get_subscription_policy.__doc__ ) - get_subscription_policy_parser.add_argument("subscription_name") + get_subscription_policy_parser.add_argument("subscription_id") set_topic_policy_parser = subparsers.add_parser( "set-topic-policy", help=set_topic_policy.__doc__ ) - set_topic_policy_parser.add_argument("topic_name") + set_topic_policy_parser.add_argument("topic_id") set_subscription_policy_parser = subparsers.add_parser( "set-subscription-policy", help=set_subscription_policy.__doc__ ) - set_subscription_policy_parser.add_argument("subscription_name") + set_subscription_policy_parser.add_argument("subscription_id") check_topic_permissions_parser = subparsers.add_parser( "check-topic-permissions", help=check_topic_permissions.__doc__ ) - check_topic_permissions_parser.add_argument("topic_name") + check_topic_permissions_parser.add_argument("topic_id") check_subscription_permissions_parser = subparsers.add_parser( - "check-subscription-permissions", - help=check_subscription_permissions.__doc__, + "check-subscription-permissions", help=check_subscription_permissions.__doc__, ) - check_subscription_permissions_parser.add_argument("subscription_name") + check_subscription_permissions_parser.add_argument("subscription_id") args = parser.parse_args() if args.command == "get-topic-policy": - get_topic_policy(args.project, args.topic_name) + get_topic_policy(args.project, args.topic_id) elif args.command == "get-subscription-policy": - get_subscription_policy(args.project, args.subscription_name) + get_subscription_policy(args.project, args.subscription_id) elif args.command == "set-topic-policy": - set_topic_policy(args.project, args.topic_name) + set_topic_policy(args.project, args.topic_id) elif args.command == "set-subscription-policy": - set_subscription_policy(args.project, args.subscription_name) + set_subscription_policy(args.project, args.subscription_id) elif args.command == "check-topic-permissions": - check_topic_permissions(args.project, args.topic_name) + check_topic_permissions(args.project, args.topic_id) elif args.command == "check-subscription-permissions": - check_subscription_permissions(args.project, args.subscription_name) + check_subscription_permissions(args.project, args.subscription_id) diff --git a/samples/snippets/iam_test.py b/samples/snippets/iam_test.py index f88cde851..31764a056 100644 --- a/samples/snippets/iam_test.py +++ b/samples/snippets/iam_test.py @@ -56,9 +56,7 @@ def subscriber_client(): @pytest.fixture def subscription(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION - ) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION) try: subscriber_client.delete_subscription(subscription_path) diff --git a/samples/snippets/publisher.py b/samples/snippets/publisher.py index 9e7820fbf..477b31b9c 100644 --- a/samples/snippets/publisher.py +++ b/samples/snippets/publisher.py @@ -40,7 +40,7 @@ def list_topics(project_id): # [END pubsub_list_topics] -def create_topic(project_id, topic_name): +def create_topic(project_id, topic_id): """Create a new Pub/Sub topic.""" # [START pubsub_quickstart_create_topic] # [START pubsub_create_topic] @@ -48,10 +48,10 @@ def create_topic(project_id, topic_name): # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" + # topic_id = "your-topic-id" publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project_id, topic_name) + topic_path = publisher.topic_path(project_id, topic_id) topic = publisher.create_topic(topic_path) @@ -60,17 +60,17 @@ def create_topic(project_id, topic_name): # [END pubsub_create_topic] -def delete_topic(project_id, topic_name): +def delete_topic(project_id, topic_id): """Deletes an existing Pub/Sub topic.""" # [START pubsub_delete_topic] from google.cloud import pubsub_v1 # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" + # topic_id = "your-topic-id" publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project_id, topic_name) + topic_path = publisher.topic_path(project_id, topic_id) publisher.delete_topic(topic_path) @@ -78,7 +78,7 @@ def delete_topic(project_id, topic_name): # [END pubsub_delete_topic] -def publish_messages(project_id, topic_name): +def publish_messages(project_id, topic_id): """Publishes multiple messages to a Pub/Sub topic.""" # [START pubsub_quickstart_publisher] # [START pubsub_publish] @@ -86,12 +86,12 @@ def publish_messages(project_id, topic_name): # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" + # topic_id = "your-topic-id" publisher = pubsub_v1.PublisherClient() # The `topic_path` method creates a fully qualified identifier - # in the form `projects/{project_id}/topics/{topic_name}` - topic_path = publisher.topic_path(project_id, topic_name) + # in the form `projects/{project_id}/topics/{topic_id}` + topic_path = publisher.topic_path(project_id, topic_id) for n in range(1, 10): data = u"Message number {}".format(n) @@ -106,7 +106,7 @@ def publish_messages(project_id, topic_name): # [END pubsub_publish] -def publish_messages_with_custom_attributes(project_id, topic_name): +def publish_messages_with_custom_attributes(project_id, topic_id): """Publishes multiple messages with custom attributes to a Pub/Sub topic.""" # [START pubsub_publish_custom_attributes] @@ -114,10 +114,10 @@ def publish_messages_with_custom_attributes(project_id, topic_name): # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" + # topic_id = "your-topic-id" publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project_id, topic_name) + topic_path = publisher.topic_path(project_id, topic_id) for n in range(1, 10): data = u"Message number {}".format(n) @@ -133,7 +133,7 @@ def publish_messages_with_custom_attributes(project_id, topic_name): # [END pubsub_publish_custom_attributes] -def publish_messages_with_error_handler(project_id, topic_name): +def publish_messages_with_error_handler(project_id, topic_id): # [START pubsub_publish_messages_error_handler] """Publishes multiple messages to a Pub/Sub topic with an error handler.""" import time @@ -142,10 +142,10 @@ def publish_messages_with_error_handler(project_id, topic_name): # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" + # topic_id = "your-topic-id" publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project_id, topic_name) + topic_path = publisher.topic_path(project_id, topic_id) futures = dict() @@ -178,14 +178,14 @@ def callback(f): # [END pubsub_publish_messages_error_handler] -def publish_messages_with_batch_settings(project_id, topic_name): +def publish_messages_with_batch_settings(project_id, topic_id): """Publishes multiple messages to a Pub/Sub topic with batch settings.""" # [START pubsub_publisher_batch_settings] from google.cloud import pubsub_v1 # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" + # topic_id = "your-topic-id" # Configure the batch to publish as soon as there is ten messages, # one kilobyte of data, or one second has passed. @@ -195,7 +195,7 @@ def publish_messages_with_batch_settings(project_id, topic_name): max_latency=1, # default 10 ms ) publisher = pubsub_v1.PublisherClient(batch_settings) - topic_path = publisher.topic_path(project_id, topic_name) + topic_path = publisher.topic_path(project_id, topic_id) # Resolve the publish future in a separate thread. def callback(future): @@ -214,14 +214,14 @@ def callback(future): # [END pubsub_publisher_batch_settings] -def publish_messages_with_retry_settings(project_id, topic_name): +def publish_messages_with_retry_settings(project_id, topic_id): """Publishes messages with custom retry settings.""" # [START pubsub_publisher_retry_settings] from google.cloud import pubsub_v1 # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" + # topic_id = "your-topic-id" # Configure the retry settings. Defaults will be overwritten. retry_settings = { @@ -260,7 +260,7 @@ def publish_messages_with_retry_settings(project_id, topic_name): } publisher = pubsub_v1.PublisherClient(client_config=retry_settings) - topic_path = publisher.topic_path(project_id, topic_name) + topic_path = publisher.topic_path(project_id, topic_id) for n in range(1, 10): data = u"Message number {}".format(n) @@ -283,52 +283,52 @@ def publish_messages_with_retry_settings(project_id, topic_name): subparsers.add_parser("list", help=list_topics.__doc__) create_parser = subparsers.add_parser("create", help=create_topic.__doc__) - create_parser.add_argument("topic_name") + create_parser.add_argument("topic_id") delete_parser = subparsers.add_parser("delete", help=delete_topic.__doc__) - delete_parser.add_argument("topic_name") + delete_parser.add_argument("topic_id") publish_parser = subparsers.add_parser("publish", help=publish_messages.__doc__) - publish_parser.add_argument("topic_name") + publish_parser.add_argument("topic_id") publish_with_custom_attributes_parser = subparsers.add_parser( "publish-with-custom-attributes", help=publish_messages_with_custom_attributes.__doc__, ) - publish_with_custom_attributes_parser.add_argument("topic_name") + publish_with_custom_attributes_parser.add_argument("topic_id") publish_with_error_handler_parser = subparsers.add_parser( "publish-with-error-handler", help=publish_messages_with_error_handler.__doc__, ) - publish_with_error_handler_parser.add_argument("topic_name") + publish_with_error_handler_parser.add_argument("topic_id") publish_with_batch_settings_parser = subparsers.add_parser( "publish-with-batch-settings", help=publish_messages_with_batch_settings.__doc__, ) - publish_with_batch_settings_parser.add_argument("topic_name") + publish_with_batch_settings_parser.add_argument("topic_id") publish_with_retry_settings_parser = subparsers.add_parser( "publish-with-retry-settings", help=publish_messages_with_retry_settings.__doc__, ) - publish_with_retry_settings_parser.add_argument("topic_name") + publish_with_retry_settings_parser.add_argument("topic_id") args = parser.parse_args() if args.command == "list": list_topics(args.project_id) elif args.command == "create": - create_topic(args.project_id, args.topic_name) + create_topic(args.project_id, args.topic_id) elif args.command == "delete": - delete_topic(args.project_id, args.topic_name) + delete_topic(args.project_id, args.topic_id) elif args.command == "publish": - publish_messages(args.project_id, args.topic_name) + publish_messages(args.project_id, args.topic_id) elif args.command == "publish-with-custom-attributes": - publish_messages_with_custom_attributes(args.project_id, args.topic_name) + publish_messages_with_custom_attributes(args.project_id, args.topic_id) elif args.command == "publish-with-error-handler": - publish_messages_with_error_handler(args.project_id, args.topic_name) + publish_messages_with_error_handler(args.project_id, args.topic_id) elif args.command == "publish-with-batch-settings": - publish_messages_with_batch_settings(args.project_id, args.topic_name) + publish_messages_with_batch_settings(args.project_id, args.topic_id) elif args.command == "publish-with-retry-settings": - publish_messages_with_retry_settings(args.project_id, args.topic_name) + publish_messages_with_retry_settings(args.project_id, args.topic_id) diff --git a/samples/snippets/quickstart/pub.py b/samples/snippets/quickstart/pub.py index a3f8087ec..16432c0c3 100644 --- a/samples/snippets/quickstart/pub.py +++ b/samples/snippets/quickstart/pub.py @@ -46,15 +46,15 @@ def callback(api_future): return callback -def pub(project_id, topic_name): +def pub(project_id, topic_id): """Publishes a message to a Pub/Sub topic.""" # [START pubsub_quickstart_pub_client] # Initialize a Publisher client. client = pubsub_v1.PublisherClient() # [END pubsub_quickstart_pub_client] # Create a fully qualified identifier in the form of - # `projects/{project_id}/topics/{topic_name}` - topic_path = client.topic_path(project_id, topic_name) + # `projects/{project_id}/topics/{topic_id}` + topic_path = client.topic_path(project_id, topic_id) # Data sent to Cloud Pub/Sub must be a bytestring. data = b"Hello, World!" @@ -75,13 +75,12 @@ def pub(project_id, topic_name): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("project_id", help="Google Cloud project ID") - parser.add_argument("topic_name", help="Pub/Sub topic name") + parser.add_argument("topic_id", help="Pub/Sub topic ID") args = parser.parse_args() - pub(args.project_id, args.topic_name) + pub(args.project_id, args.topic_id) # [END pubsub_quickstart_pub_all] diff --git a/samples/snippets/quickstart/sub.py b/samples/snippets/quickstart/sub.py index 1d90726f5..efe008915 100644 --- a/samples/snippets/quickstart/sub.py +++ b/samples/snippets/quickstart/sub.py @@ -23,23 +23,19 @@ # [END pubsub_quickstart_sub_deps] -def sub(project_id, subscription_name): +def sub(project_id, subscription_id): """Receives messages from a Pub/Sub subscription.""" # [START pubsub_quickstart_sub_client] # Initialize a Subscriber client subscriber_client = pubsub_v1.SubscriberClient() # [END pubsub_quickstart_sub_client] # Create a fully qualified identifier in the form of - # `projects/{project_id}/subscriptions/{subscription_name}` - subscription_path = subscriber_client.subscription_path( - project_id, subscription_name - ) + # `projects/{project_id}/subscriptions/{subscription_id}` + subscription_path = subscriber_client.subscription_path(project_id, subscription_id) def callback(message): print( - "Received message {} of message ID {}\n".format( - message, message.message_id - ) + "Received message {} of message ID {}\n".format(message, message.message_id) ) # Acknowledge the message. Unack'ed messages will be redelivered. message.ack() @@ -62,13 +58,12 @@ def callback(message): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("project_id", help="Google Cloud project ID") - parser.add_argument("subscription_name", help="Pub/Sub subscription name") + parser.add_argument("subscription_id", help="Pub/Sub subscription ID") args = parser.parse_args() - sub(args.project_id, args.subscription_name) + sub(args.project_id, args.subscription_id) # [END pubsub_quickstart_sub_all] diff --git a/samples/snippets/quickstart/sub_test.py b/samples/snippets/quickstart/sub_test.py index 2754dc56b..65d5fa111 100644 --- a/samples/snippets/quickstart/sub_test.py +++ b/samples/snippets/quickstart/sub_test.py @@ -48,9 +48,7 @@ def topic_path(): @pytest.fixture(scope="module") def subscription_path(topic_path): - subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION - ) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION) try: subscription = subscriber_client.create_subscription( @@ -82,9 +80,7 @@ def test_sub(monkeypatch, topic_path, subscription_path, capsys): monkeypatch.setattr(pubsub_v1, "SubscriberClient", mock_client_constructor) def mock_subscribe(subscription_path, callback=None): - real_future = real_client.subscribe( - subscription_path, callback=callback - ) + real_future = real_client.subscribe(subscription_path, callback=callback) mock_future = mock.Mock(spec=real_future, wraps=real_future) def mock_result(): diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 5dc468e23..677bfe359 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -24,17 +24,17 @@ import argparse -def list_subscriptions_in_topic(project_id, topic_name): +def list_subscriptions_in_topic(project_id, topic_id): """Lists all subscriptions for a given topic.""" # [START pubsub_list_topic_subscriptions] from google.cloud import pubsub_v1 # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" + # topic_id = "your-topic-id" publisher = pubsub_v1.PublisherClient() - topic_path = publisher.topic_path(project_id, topic_name) + topic_path = publisher.topic_path(project_id, topic_id) for subscription in publisher.list_topic_subscriptions(topic_path): print(subscription) @@ -59,19 +59,19 @@ def list_subscriptions_in_project(project_id): # [END pubsub_list_subscriptions] -def create_subscription(project_id, topic_name, subscription_name): +def create_subscription(project_id, topic_id, subscription_id): """Create a new pull subscription on the given topic.""" # [START pubsub_create_pull_subscription] from google.cloud import pubsub_v1 # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" - # subscription_name = "your-subscription-id" + # topic_id = "your-topic-id" + # subscription_id = "your-subscription-id" subscriber = pubsub_v1.SubscriberClient() - topic_path = subscriber.topic_path(project_id, topic_name) - subscription_path = subscriber.subscription_path(project_id, subscription_name) + topic_path = subscriber.topic_path(project_id, topic_id) + subscription_path = subscriber.subscription_path(project_id, subscription_id) subscription = subscriber.create_subscription(subscription_path, topic_path) @@ -82,7 +82,7 @@ def create_subscription(project_id, topic_name, subscription_name): def create_subscription_with_dead_letter_topic( - project_id, topic_name, subscription_name, dead_letter_topic_name + project_id, topic_id, subscription_id, dead_letter_topic_id ): """Create a subscription with dead letter policy.""" # [START pubsub_dead_letter_create_subscription] @@ -94,17 +94,17 @@ def create_subscription_with_dead_letter_topic( # endpoint = "https://my-test-project.appspot.com/push" # TODO(developer): This is an existing topic that the subscription # with dead letter policy is attached to. - # topic_name = "your-topic-id" + # topic_id = "your-topic-id" # TODO(developer): This is an existing subscription with a dead letter policy. - # subscription_name = "your-subscription-id" + # subscription_id = "your-subscription-id" # TODO(developer): This is an existing dead letter topic that the subscription # with dead letter policy will forward dead letter messages to. - # dead_letter_topic_name = "your-dead-letter-topic-id" + # dead_letter_topic_id = "your-dead-letter-topic-id" subscriber = pubsub_v1.SubscriberClient() - topic_path = subscriber.topic_path(project_id, topic_name) - subscription_path = subscriber.subscription_path(project_id, subscription_name) - dead_letter_topic_path = subscriber.topic_path(project_id, dead_letter_topic_name) + topic_path = subscriber.topic_path(project_id, topic_id) + subscription_path = subscriber.subscription_path(project_id, subscription_id) + dead_letter_topic_path = subscriber.topic_path(project_id, dead_letter_topic_id) dead_letter_policy = DeadLetterPolicy( dead_letter_topic=dead_letter_topic_path, max_delivery_attempts=10 @@ -129,20 +129,20 @@ def create_subscription_with_dead_letter_topic( # [END pubsub_dead_letter_create_subscription] -def create_push_subscription(project_id, topic_name, subscription_name, endpoint): +def create_push_subscription(project_id, topic_id, subscription_id, endpoint): """Create a new push subscription on the given topic.""" # [START pubsub_create_push_subscription] from google.cloud import pubsub_v1 # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" - # subscription_name = "your-subscription-id" + # topic_id = "your-topic-id" + # subscription_id = "your-subscription-id" # endpoint = "https://my-test-project.appspot.com/push" subscriber = pubsub_v1.SubscriberClient() - topic_path = subscriber.topic_path(project_id, topic_name) - subscription_path = subscriber.subscription_path(project_id, subscription_name) + topic_path = subscriber.topic_path(project_id, topic_id) + subscription_path = subscriber.subscription_path(project_id, subscription_id) push_config = pubsub_v1.types.PushConfig(push_endpoint=endpoint) @@ -157,17 +157,17 @@ def create_push_subscription(project_id, topic_name, subscription_name, endpoint # [END pubsub_create_push_subscription] -def delete_subscription(project_id, subscription_name): +def delete_subscription(project_id, subscription_id): """Deletes an existing Pub/Sub topic.""" # [START pubsub_delete_subscription] from google.cloud import pubsub_v1 # TODO(developer) # project_id = "your-project-id" - # subscription_name = "your-subscription-id" + # subscription_id = "your-subscription-id" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_id) subscriber.delete_subscription(subscription_path) @@ -177,7 +177,7 @@ def delete_subscription(project_id, subscription_name): # [END pubsub_delete_subscription] -def update_push_subscription(project_id, topic_name, subscription_name, endpoint): +def update_push_subscription(project_id, topic_id, subscription_id, endpoint): """ Updates an existing Pub/Sub subscription's push endpoint URL. Note that certain properties of a subscription, such as @@ -188,17 +188,17 @@ def update_push_subscription(project_id, topic_name, subscription_name, endpoint # TODO(developer) # project_id = "your-project-id" - # topic_name = "your-topic-id" - # subscription_name = "your-subscription-id" + # topic_id = "your-topic-id" + # subscription_id = "your-subscription-id" # endpoint = "https://my-test-project.appspot.com/push" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_id) push_config = pubsub_v1.types.PushConfig(push_endpoint=endpoint) subscription = pubsub_v1.types.Subscription( - name=subscription_path, topic=topic_name, push_config=push_config + name=subscription_path, topic=topic_id, push_config=push_config ) update_mask = {"paths": {"push_config"}} @@ -213,7 +213,7 @@ def update_push_subscription(project_id, topic_name, subscription_name, endpoint def update_subscription_with_dead_letter_policy( - project_id, topic_name, subscription_name, dead_letter_topic_name + project_id, topic_id, subscription_id, dead_letter_topic_id ): """Update a subscription's dead letter policy.""" # [START pubsub_dead_letter_update_subscription] @@ -224,17 +224,17 @@ def update_subscription_with_dead_letter_policy( # project_id = "your-project-id" # TODO(developer): This is an existing topic that the subscription # with dead letter policy is attached to. - # topic_name = "your-topic-name" + # topic_id = "your-topic-id" # TODO(developer): This is an existing subscription with a dead letter policy. - # subscription_name = "your-subscription-id" + # subscription_id = "your-subscription-id" # TODO(developer): This is an existing dead letter topic that the subscription # with dead letter policy will forward dead letter messages to. - # dead_letter_topic_name = "your-dead-letter-topic-id" + # dead_letter_topic_id = "your-dead-letter-topic-id" subscriber = pubsub_v1.SubscriberClient() - topic_path = subscriber.topic_path(project_id, topic_name) - subscription_path = subscriber.subscription_path(project_id, subscription_name) - dead_letter_topic_path = subscriber.topic_path(project_id, dead_letter_topic_name) + topic_path = subscriber.topic_path(project_id, topic_id) + subscription_path = subscriber.subscription_path(project_id, subscription_id) + dead_letter_topic_path = subscriber.topic_path(project_id, dead_letter_topic_id) subscription_before_update = subscriber.get_subscription(subscription_path) print("Before the update: {}".format(subscription_before_update)) @@ -264,7 +264,7 @@ def update_subscription_with_dead_letter_policy( return subscription_after_update -def remove_dead_letter_policy(project_id, topic_name, subscription_name): +def remove_dead_letter_policy(project_id, topic_id, subscription_id): """Remove dead letter policy from a subscription.""" # [START pubsub_dead_letter_remove] from google.cloud import pubsub_v1 @@ -274,13 +274,13 @@ def remove_dead_letter_policy(project_id, topic_name, subscription_name): # project_id = "your-project-id" # TODO(developer): This is an existing topic that the subscription # with dead letter policy is attached to. - # topic_name = "your-topic-name" + # topic_id = "your-topic-id" # TODO(developer): This is an existing subscription with a dead letter policy. - # subscription_name = "your-subscription-id" + # subscription_id = "your-subscription-id" subscriber = pubsub_v1.SubscriberClient() - topic_path = subscriber.topic_path(project_id, topic_name) - subscription_path = subscriber.subscription_path(project_id, subscription_name) + topic_path = subscriber.topic_path(project_id, topic_id) + subscription_path = subscriber.subscription_path(project_id, subscription_id) subscription_before_update = subscriber.get_subscription(subscription_path) print("Before removing the policy: {}".format(subscription_before_update)) @@ -309,7 +309,7 @@ def remove_dead_letter_policy(project_id, topic_name, subscription_name): return subscription_after_update -def receive_messages(project_id, subscription_name, timeout=None): +def receive_messages(project_id, subscription_id, timeout=None): """Receives messages from a pull subscription.""" # [START pubsub_subscriber_async_pull] # [START pubsub_quickstart_subscriber] @@ -318,14 +318,14 @@ def receive_messages(project_id, subscription_name, timeout=None): # TODO(developer) # project_id = "your-project-id" - # subscription_name = "your-subscription-id" + # subscription_id = "your-subscription-id" # Number of seconds the subscriber should listen for messages # timeout = 5.0 subscriber = pubsub_v1.SubscriberClient() # The `subscription_path` method creates a fully qualified identifier - # in the form `projects/{project_id}/subscriptions/{subscription_name}` - subscription_path = subscriber.subscription_path(project_id, subscription_name) + # in the form `projects/{project_id}/subscriptions/{subscription_id}` + subscription_path = subscriber.subscription_path(project_id, subscription_id) def callback(message): print("Received message: {}".format(message)) @@ -346,9 +346,7 @@ def callback(message): # [END pubsub_quickstart_subscriber] -def receive_messages_with_custom_attributes( - project_id, subscription_name, timeout=None -): +def receive_messages_with_custom_attributes(project_id, subscription_id, timeout=None): """Receives messages from a pull subscription.""" # [START pubsub_subscriber_sync_pull_custom_attributes] # [START pubsub_subscriber_async_pull_custom_attributes] @@ -357,12 +355,12 @@ def receive_messages_with_custom_attributes( # TODO(developer) # project_id = "your-project-id" - # subscription_name = "your-subscription-id" + # subscription_id = "your-subscription-id" # Number of seconds the subscriber should listen for messages # timeout = 5.0 subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_id) def callback(message): print("Received message: {}".format(message.data)) @@ -388,7 +386,7 @@ def callback(message): # [END pubsub_subscriber_sync_pull_custom_attributes] -def receive_messages_with_flow_control(project_id, subscription_name, timeout=None): +def receive_messages_with_flow_control(project_id, subscription_id, timeout=None): """Receives messages from a pull subscription with flow control.""" # [START pubsub_subscriber_flow_settings] from concurrent.futures import TimeoutError @@ -396,12 +394,12 @@ def receive_messages_with_flow_control(project_id, subscription_name, timeout=No # TODO(developer) # project_id = "your-project-id" - # subscription_name = "your-subscription-id" + # subscription_id = "your-subscription-id" # Number of seconds the subscriber should listen for messages # timeout = 5.0 subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_id) def callback(message): print("Received message: {}".format(message.data)) @@ -426,17 +424,17 @@ def callback(message): # [END pubsub_subscriber_flow_settings] -def synchronous_pull(project_id, subscription_name): +def synchronous_pull(project_id, subscription_id): """Pulling messages synchronously.""" # [START pubsub_subscriber_sync_pull] from google.cloud import pubsub_v1 # TODO(developer) # project_id = "your-project-id" - # subscription_name = "your-subscription-id" + # subscription_id = "your-subscription-id" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_id) NUM_MESSAGES = 3 @@ -461,7 +459,7 @@ def synchronous_pull(project_id, subscription_name): # [END pubsub_subscriber_sync_pull] -def synchronous_pull_with_lease_management(project_id, subscription_name): +def synchronous_pull_with_lease_management(project_id, subscription_id): """Pulling messages synchronously with lease management""" # [START pubsub_subscriber_sync_pull_with_lease] import logging @@ -473,10 +471,10 @@ def synchronous_pull_with_lease_management(project_id, subscription_name): # TODO(developer) # project_id = "your-project-id" - # subscription_name = "your-subscription-id" + # subscription_id = "your-subscription-id" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_id) NUM_MESSAGES = 2 ACK_DEADLINE = 30 @@ -547,19 +545,19 @@ def worker(msg): # [END pubsub_subscriber_sync_pull_with_lease] -def listen_for_errors(project_id, subscription_name, timeout=None): +def listen_for_errors(project_id, subscription_id, timeout=None): """Receives messages and catches errors from a pull subscription.""" # [START pubsub_subscriber_error_listener] from google.cloud import pubsub_v1 # TODO(developer) - # project_id = "Your Google Cloud Project ID" - # subscription_name = "Your Pubsub subscription name" + # project_id = "your-project-id" + # subscription_id = "your-subscription-id" # Number of seconds the subscriber should listen for messages # timeout = 5.0 subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_id) def callback(message): print("Received message: {}".format(message)) @@ -578,25 +576,23 @@ def callback(message): streaming_pull_future.cancel() print( "Listening for messages on {} threw an exception: {}.".format( - subscription_name, e + subscription_id, e ) ) # [END pubsub_subscriber_error_listener] -def receive_messages_with_delivery_attempts( - project_id, subscription_name, timeout=None -): +def receive_messages_with_delivery_attempts(project_id, subscription_id, timeout=None): # [START pubsub_dead_letter_delivery_attempt] from concurrent.futures import TimeoutError from google.cloud import pubsub_v1 # TODO(developer) # project_id = "your-project-id" - # subscription_name = "your-subscription-id" + # subscription_id = "your-subscription-id" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_id) def callback(message): print("Received message: {}".format(message)) @@ -627,64 +623,64 @@ def callback(message): list_in_topic_parser = subparsers.add_parser( "list-in-topic", help=list_subscriptions_in_topic.__doc__ ) - list_in_topic_parser.add_argument("topic_name") + list_in_topic_parser.add_argument("topic_id") list_in_project_parser = subparsers.add_parser( "list-in-project", help=list_subscriptions_in_project.__doc__ ) create_parser = subparsers.add_parser("create", help=create_subscription.__doc__) - create_parser.add_argument("topic_name") - create_parser.add_argument("subscription_name") + create_parser.add_argument("topic_id") + create_parser.add_argument("subscription_id") create_with_dead_letter_policy_parser = subparsers.add_parser( "create-with-dead-letter-policy", help=create_subscription_with_dead_letter_topic.__doc__, ) - create_with_dead_letter_policy_parser.add_argument("topic_name") - create_with_dead_letter_policy_parser.add_argument("subscription_name") - create_with_dead_letter_policy_parser.add_argument("dead_letter_topic_name") + create_with_dead_letter_policy_parser.add_argument("topic_id") + create_with_dead_letter_policy_parser.add_argument("subscription_id") + create_with_dead_letter_policy_parser.add_argument("dead_letter_topic_id") create_push_parser = subparsers.add_parser( "create-push", help=create_push_subscription.__doc__ ) - create_push_parser.add_argument("topic_name") - create_push_parser.add_argument("subscription_name") + create_push_parser.add_argument("topic_id") + create_push_parser.add_argument("subscription_id") create_push_parser.add_argument("endpoint") delete_parser = subparsers.add_parser("delete", help=delete_subscription.__doc__) - delete_parser.add_argument("subscription_name") + delete_parser.add_argument("subscription_id") update_push_parser = subparsers.add_parser( "update-push", help=update_push_subscription.__doc__ ) - update_push_parser.add_argument("topic_name") - update_push_parser.add_argument("subscription_name") + update_push_parser.add_argument("topic_id") + update_push_parser.add_argument("subscription_id") update_push_parser.add_argument("endpoint") update_dead_letter_policy_parser = subparsers.add_parser( "update-dead-letter-policy", help=update_subscription_with_dead_letter_policy.__doc__, ) - update_dead_letter_policy_parser.add_argument("topic_name") - update_dead_letter_policy_parser.add_argument("subscription_name") - update_dead_letter_policy_parser.add_argument("dead_letter_topic_name") + update_dead_letter_policy_parser.add_argument("topic_id") + update_dead_letter_policy_parser.add_argument("subscription_id") + update_dead_letter_policy_parser.add_argument("dead_letter_topic_id") remove_dead_letter_policy_parser = subparsers.add_parser( "remove-dead-letter-policy", help=remove_dead_letter_policy.__doc__ ) - remove_dead_letter_policy_parser.add_argument("topic_name") - remove_dead_letter_policy_parser.add_argument("subscription_name") + remove_dead_letter_policy_parser.add_argument("topic_id") + remove_dead_letter_policy_parser.add_argument("subscription_id") receive_parser = subparsers.add_parser("receive", help=receive_messages.__doc__) - receive_parser.add_argument("subscription_name") + receive_parser.add_argument("subscription_id") receive_parser.add_argument("timeout", default=None, type=float, nargs="?") receive_with_custom_attributes_parser = subparsers.add_parser( "receive-custom-attributes", help=receive_messages_with_custom_attributes.__doc__, ) - receive_with_custom_attributes_parser.add_argument("subscription_name") + receive_with_custom_attributes_parser.add_argument("subscription_id") receive_with_custom_attributes_parser.add_argument( "timeout", default=None, type=float, nargs="?" ) @@ -692,7 +688,7 @@ def callback(message): receive_with_flow_control_parser = subparsers.add_parser( "receive-flow-control", help=receive_messages_with_flow_control.__doc__ ) - receive_with_flow_control_parser.add_argument("subscription_name") + receive_with_flow_control_parser.add_argument("subscription_id") receive_with_flow_control_parser.add_argument( "timeout", default=None, type=float, nargs="?" ) @@ -700,18 +696,18 @@ def callback(message): synchronous_pull_parser = subparsers.add_parser( "receive-synchronously", help=synchronous_pull.__doc__ ) - synchronous_pull_parser.add_argument("subscription_name") + synchronous_pull_parser.add_argument("subscription_id") synchronous_pull_with_lease_management_parser = subparsers.add_parser( "receive-synchronously-with-lease", help=synchronous_pull_with_lease_management.__doc__, ) - synchronous_pull_with_lease_management_parser.add_argument("subscription_name") + synchronous_pull_with_lease_management_parser.add_argument("subscription_id") listen_for_errors_parser = subparsers.add_parser( "listen-for-errors", help=listen_for_errors.__doc__ ) - listen_for_errors_parser.add_argument("subscription_name") + listen_for_errors_parser.add_argument("subscription_id") listen_for_errors_parser.add_argument( "timeout", default=None, type=float, nargs="?" ) @@ -720,7 +716,7 @@ def callback(message): "receive-messages-with-delivery-attempts", help=receive_messages_with_delivery_attempts.__doc__, ) - receive_messages_with_delivery_attempts_parser.add_argument("subscription_name") + receive_messages_with_delivery_attempts_parser.add_argument("subscription_id") receive_messages_with_delivery_attempts_parser.add_argument( "timeout", default=None, type=float, nargs="?" ) @@ -728,56 +724,54 @@ def callback(message): args = parser.parse_args() if args.command == "list-in-topic": - list_subscriptions_in_topic(args.project_id, args.topic_name) + list_subscriptions_in_topic(args.project_id, args.topic_id) elif args.command == "list-in-project": list_subscriptions_in_project(args.project_id) elif args.command == "create": - create_subscription(args.project_id, args.topic_name, args.subscription_name) + create_subscription(args.project_id, args.topic_id, args.subscription_id) elif args.command == "create-with-dead-letter-policy": create_subscription_with_dead_letter_topic( args.project_id, - args.topic_name, - args.subscription_name, - args.dead_letter_topic_name, + args.topic_id, + args.subscription_id, + args.dead_letter_topic_id, ) elif args.command == "create-push": create_push_subscription( - args.project_id, args.topic_name, args.subscription_name, args.endpoint, + args.project_id, args.topic_id, args.subscription_id, args.endpoint, ) elif args.command == "delete": - delete_subscription(args.project_id, args.subscription_name) + delete_subscription(args.project_id, args.subscription_id) elif args.command == "update-push": update_push_subscription( - args.project_id, args.topic_name, args.subscription_name, args.endpoint, + args.project_id, args.topic_id, args.subscription_id, args.endpoint, ) elif args.command == "update-dead-letter-policy": update_subscription_with_dead_letter_policy( args.project_id, - args.topic_name, - args.subscription_name, - args.dead_letter_topic_name, + args.topic_id, + args.subscription_id, + args.dead_letter_topic_id, ) elif args.command == "remove-dead-letter-policy": - remove_dead_letter_policy( - args.project_id, args.topic_name, args.subscription_name - ) + remove_dead_letter_policy(args.project_id, args.topic_id, args.subscription_id) elif args.command == "receive": - receive_messages(args.project_id, args.subscription_name, args.timeout) + receive_messages(args.project_id, args.subscription_id, args.timeout) elif args.command == "receive-custom-attributes": receive_messages_with_custom_attributes( - args.project_id, args.subscription_name, args.timeout + args.project_id, args.subscription_id, args.timeout ) elif args.command == "receive-flow-control": receive_messages_with_flow_control( - args.project_id, args.subscription_name, args.timeout + args.project_id, args.subscription_id, args.timeout ) elif args.command == "receive-synchronously": - synchronous_pull(args.project_id, args.subscription_name) + synchronous_pull(args.project_id, args.subscription_id) elif args.command == "receive-synchronously-with-lease": - synchronous_pull_with_lease_management(args.project_id, args.subscription_name) + synchronous_pull_with_lease_management(args.project_id, args.subscription_id) elif args.command == "listen-for-errors": - listen_for_errors(args.project_id, args.subscription_name, args.timeout) + listen_for_errors(args.project_id, args.subscription_id, args.timeout) elif args.command == "receive-messages-with-delivery-attempts": receive_messages_with_delivery_attempts( - args.project_id, args.subscription_name, args.timeout + args.project_id, args.subscription_id, args.timeout ) From c84240d402609e9aabfc4a65e6fe90d7efb7ee83 Mon Sep 17 00:00:00 2001 From: Kurtis Van Gent <31518063+kurtisvg@users.noreply.github.com> Date: Tue, 9 Jun 2020 14:34:27 -0700 Subject: [PATCH 095/101] Replace GCLOUD_PROJECT with GOOGLE_CLOUD_PROJECT. [(#4022)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4022) --- samples/snippets/iam_test.py | 2 +- samples/snippets/publisher_test.py | 2 +- samples/snippets/quickstart/pub_test.py | 2 +- samples/snippets/quickstart/sub_test.py | 2 +- samples/snippets/subscriber_test.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/samples/snippets/iam_test.py b/samples/snippets/iam_test.py index 31764a056..d196953f6 100644 --- a/samples/snippets/iam_test.py +++ b/samples/snippets/iam_test.py @@ -21,7 +21,7 @@ import iam UUID = uuid.uuid4().hex -PROJECT = os.environ["GCLOUD_PROJECT"] +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] TOPIC = "iam-test-topic-" + UUID SUBSCRIPTION = "iam-test-subscription-" + UUID diff --git a/samples/snippets/publisher_test.py b/samples/snippets/publisher_test.py index dc6095508..b5c2ea1ea 100644 --- a/samples/snippets/publisher_test.py +++ b/samples/snippets/publisher_test.py @@ -24,7 +24,7 @@ import publisher UUID = uuid.uuid4().hex -PROJECT = os.environ["GCLOUD_PROJECT"] +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] TOPIC_ADMIN = "publisher-test-topic-admin-" + UUID TOPIC_PUBLISH = "publisher-test-topic-publish-" + UUID diff --git a/samples/snippets/quickstart/pub_test.py b/samples/snippets/quickstart/pub_test.py index 24010c76e..6f5cc06c4 100644 --- a/samples/snippets/quickstart/pub_test.py +++ b/samples/snippets/quickstart/pub_test.py @@ -25,7 +25,7 @@ UUID = uuid.uuid4().hex -PROJECT = os.environ["GCLOUD_PROJECT"] +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] TOPIC = "quickstart-pub-test-topic-" + UUID diff --git a/samples/snippets/quickstart/sub_test.py b/samples/snippets/quickstart/sub_test.py index 65d5fa111..38047422a 100644 --- a/samples/snippets/quickstart/sub_test.py +++ b/samples/snippets/quickstart/sub_test.py @@ -25,7 +25,7 @@ UUID = uuid.uuid4().hex -PROJECT = os.environ["GCLOUD_PROJECT"] +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] TOPIC = "quickstart-sub-test-topic-" + UUID SUBSCRIPTION = "quickstart-sub-test-topic-sub-" + UUID diff --git a/samples/snippets/subscriber_test.py b/samples/snippets/subscriber_test.py index 6b90396f9..a7f7c139c 100644 --- a/samples/snippets/subscriber_test.py +++ b/samples/snippets/subscriber_test.py @@ -22,7 +22,7 @@ import subscriber UUID = uuid.uuid4().hex -PROJECT = os.environ["GCLOUD_PROJECT"] +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] TOPIC = "subscription-test-topic-" + UUID DEAD_LETTER_TOPIC = "subscription-test-dead-letter-topic-" + UUID SUBSCRIPTION_ADMIN = "subscription-test-subscription-admin-" + UUID From a40dcf03f3e0a88be2b03e95d5eed36a48f51ca1 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Thu, 11 Jun 2020 12:31:06 -0700 Subject: [PATCH 096/101] nit: remove redundant/wrong Pub/Sub region tag [(#4027)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4027) --- samples/snippets/subscriber.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index 677bfe359..a367f181c 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -348,7 +348,6 @@ def callback(message): def receive_messages_with_custom_attributes(project_id, subscription_id, timeout=None): """Receives messages from a pull subscription.""" - # [START pubsub_subscriber_sync_pull_custom_attributes] # [START pubsub_subscriber_async_pull_custom_attributes] from concurrent.futures import TimeoutError from google.cloud import pubsub_v1 @@ -383,7 +382,6 @@ def callback(message): except TimeoutError: streaming_pull_future.cancel() # [END pubsub_subscriber_async_pull_custom_attributes] - # [END pubsub_subscriber_sync_pull_custom_attributes] def receive_messages_with_flow_control(project_id, subscription_id, timeout=None): From 0686bfb5cf9bd629890d1e1b592acca9b613a4a9 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Fri, 12 Jun 2020 16:00:13 -0700 Subject: [PATCH 097/101] Pub/Sub: wrap subscriber in a with block and add comments [(#4070)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4070) Use a `with` block to wrap subscriber and describe its purpose. Internal bug: b/157401623 --- samples/snippets/subscriber.py | 72 +++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 32 deletions(-) diff --git a/samples/snippets/subscriber.py b/samples/snippets/subscriber.py index a367f181c..f079e7d42 100644 --- a/samples/snippets/subscriber.py +++ b/samples/snippets/subscriber.py @@ -52,10 +52,11 @@ def list_subscriptions_in_project(project_id): subscriber = pubsub_v1.SubscriberClient() project_path = subscriber.project_path(project_id) - for subscription in subscriber.list_subscriptions(project_path): - print(subscription.name) - - subscriber.close() + # Wrap the subscriber in a 'with' block to automatically call close() to + # close the underlying gRPC channel when done. + with subscriber: + for subscription in subscriber.list_subscriptions(project_path): + print(subscription.name) # [END pubsub_list_subscriptions] @@ -73,11 +74,12 @@ def create_subscription(project_id, topic_id, subscription_id): topic_path = subscriber.topic_path(project_id, topic_id) subscription_path = subscriber.subscription_path(project_id, subscription_id) - subscription = subscriber.create_subscription(subscription_path, topic_path) + # Wrap the subscriber in a 'with' block to automatically call close() to + # close the underlying gRPC channel when done. + with subscriber: + subscription = subscriber.create_subscription(subscription_path, topic_path) print("Subscription created: {}".format(subscription)) - - subscriber.close() # [END pubsub_create_pull_subscription] @@ -146,14 +148,15 @@ def create_push_subscription(project_id, topic_id, subscription_id, endpoint): push_config = pubsub_v1.types.PushConfig(push_endpoint=endpoint) - subscription = subscriber.create_subscription( - subscription_path, topic_path, push_config - ) + # Wrap the subscriber in a 'with' block to automatically call close() to + # close the underlying gRPC channel when done. + with subscriber: + subscription = subscriber.create_subscription( + subscription_path, topic_path, push_config + ) print("Push subscription created: {}".format(subscription)) print("Endpoint for subscription is: {}".format(endpoint)) - - subscriber.close() # [END pubsub_create_push_subscription] @@ -169,11 +172,12 @@ def delete_subscription(project_id, subscription_id): subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path(project_id, subscription_id) - subscriber.delete_subscription(subscription_path) + # Wrap the subscriber in a 'with' block to automatically call close() to + # close the underlying gRPC channel when done. + with subscriber: + subscriber.delete_subscription(subscription_path) print("Subscription deleted: {}".format(subscription_path)) - - subscriber.close() # [END pubsub_delete_subscription] @@ -203,12 +207,13 @@ def update_push_subscription(project_id, topic_id, subscription_id, endpoint): update_mask = {"paths": {"push_config"}} - result = subscriber.update_subscription(subscription, update_mask) + # Wrap the subscriber in a 'with' block to automatically call close() to + # close the underlying gRPC channel when done. + with subscriber: + result = subscriber.update_subscription(subscription, update_mask) print("Subscription updated: {}".format(subscription_path)) print("New endpoint for subscription is: {}".format(result.push_config)) - - subscriber.close() # [END pubsub_update_push_configuration] @@ -436,24 +441,25 @@ def synchronous_pull(project_id, subscription_id): NUM_MESSAGES = 3 - # The subscriber pulls a specific number of messages. - response = subscriber.pull(subscription_path, max_messages=NUM_MESSAGES) + # Wrap the subscriber in a 'with' block to automatically call close() to + # close the underlying gRPC channel when done. + with subscriber: + # The subscriber pulls a specific number of messages. + response = subscriber.pull(subscription_path, max_messages=NUM_MESSAGES) - ack_ids = [] - for received_message in response.received_messages: - print("Received: {}".format(received_message.message.data)) - ack_ids.append(received_message.ack_id) + ack_ids = [] + for received_message in response.received_messages: + print("Received: {}".format(received_message.message.data)) + ack_ids.append(received_message.ack_id) - # Acknowledges the received messages so they will not be sent again. - subscriber.acknowledge(subscription_path, ack_ids) + # Acknowledges the received messages so they will not be sent again. + subscriber.acknowledge(subscription_path, ack_ids) - print( - "Received and acknowledged {} messages. Done.".format( - len(response.received_messages) + print( + "Received and acknowledged {} messages. Done.".format( + len(response.received_messages) + ) ) - ) - - subscriber.close() # [END pubsub_subscriber_sync_pull] @@ -539,6 +545,8 @@ def worker(msg): ) ) + # Close the underlying gPRC channel. Alternatively, wrap subscriber in + # a 'with' block to automatically call close() when done. subscriber.close() # [END pubsub_subscriber_sync_pull_with_lease] From 7a68bf69f1afc5e5ff66f5d214639095fc8ae7b9 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 20 Jun 2020 06:08:08 +0200 Subject: [PATCH 098/101] Update dependency google-cloud-pubsub to v1.6.0 [(#4039)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4039) This PR contains the following updates: | Package | Update | Change | |---|---|---| | [google-cloud-pubsub](https://togithub.com/googleapis/python-pubsub) | minor | `==1.5.0` -> `==1.6.0` | --- ### Release Notes
googleapis/python-pubsub ### [`v1.6.0`](https://togithub.com/googleapis/python-pubsub/blob/master/CHANGELOG.md#​160-httpswwwgithubcomgoogleapispython-pubsubcomparev150v160-2020-06-09) [Compare Source](https://togithub.com/googleapis/python-pubsub/compare/v1.5.0...v1.6.0) ##### Features - Add flow control for message publishing ([#​96](https://www.github.com/googleapis/python-pubsub/issues/96)) ([06085c4](https://www.github.com/googleapis/python-pubsub/commit/06085c4083b9dccdd50383257799904510bbf3a0)) ##### Bug Fixes - Fix PubSub incompatibility with api-core 1.17.0+ ([#​103](https://www.github.com/googleapis/python-pubsub/issues/103)) ([c02060f](https://www.github.com/googleapis/python-pubsub/commit/c02060fbbe6e2ca4664bee08d2de10665d41dc0b)) ##### Documentation - Clarify that Schedulers shouldn't be used with multiple SubscriberClients ([#​100](https://togithub.com/googleapis/python-pubsub/pull/100)) ([cf9e87c](https://togithub.com/googleapis/python-pubsub/commit/cf9e87c80c0771f3fa6ef784a8d76cb760ad37ef)) - Fix update subscription/snapshot/topic samples ([#​113](https://togithub.com/googleapis/python-pubsub/pull/113)) ([e62c38b](https://togithub.com/googleapis/python-pubsub/commit/e62c38bb33de2434e32f866979de769382dea34a)) ##### Internal / Testing Changes - Re-generated service implementaton using synth: removed experimental notes from the RetryPolicy and filtering features in anticipation of GA, added DetachSubscription (experimental) ([#​114](https://togithub.com/googleapis/python-pubsub/pull/114)) ([0132a46](https://togithub.com/googleapis/python-pubsub/commit/0132a4680e0727ce45d5e27d98ffc9f3541a0962)) - Incorporate will_accept() checks into publish() ([#​108](https://togithub.com/googleapis/python-pubsub/pull/108)) ([6c7677e](https://togithub.com/googleapis/python-pubsub/commit/6c7677ecb259672bbb9b6f7646919e602c698570))
--- ### Renovate configuration :date: **Schedule**: At any time (no schedule defined). :vertical_traffic_light: **Automerge**: Disabled by config. Please merge this manually once you are satisfied. :recycle: **Rebasing**: Never, or you tick the rebase/retry checkbox. :no_bell: **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#GoogleCloudPlatform/python-docs-samples). --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 9cc17af84..9b496510a 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==1.5.0 +google-cloud-pubsub==1.6.0 From 39050eb220e455bd84dbde7a260ae2d6b6a7f44c Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Fri, 26 Jun 2020 09:20:30 +0200 Subject: [PATCH 099/101] chore: update templates --- .github/CODEOWNERS | 11 ++ samples/AUTHORING_GUIDE.md | 1 + samples/CONTRIBUTING.md | 1 + samples/snippets/README.rst | 48 +++++++- samples/snippets/noxfile.py | 224 ++++++++++++++++++++++++++++++++++++ synth.py | 11 +- 6 files changed, 292 insertions(+), 4 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 samples/AUTHORING_GUIDE.md create mode 100644 samples/CONTRIBUTING.md create mode 100644 samples/snippets/noxfile.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..cf01548a9 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,11 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. +# +# For syntax help see: +# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax + + +# The python-samples-owners team is the default owner for anything not +# explicitly taken by someone else. + + /samples/ @anguillanneuf @hongalex @googleapis/python-samples-owners diff --git a/samples/AUTHORING_GUIDE.md b/samples/AUTHORING_GUIDE.md new file mode 100644 index 000000000..55c97b32f --- /dev/null +++ b/samples/AUTHORING_GUIDE.md @@ -0,0 +1 @@ +See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/AUTHORING_GUIDE.md \ No newline at end of file diff --git a/samples/CONTRIBUTING.md b/samples/CONTRIBUTING.md new file mode 100644 index 000000000..34c882b6f --- /dev/null +++ b/samples/CONTRIBUTING.md @@ -0,0 +1 @@ +See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/CONTRIBUTING.md \ No newline at end of file diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index f27f9438e..2676680af 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -1,3 +1,4 @@ + .. This file is automatically generated. Do not edit this file directly. Google Cloud Pub/Sub Python Samples @@ -14,10 +15,12 @@ This directory contains samples for Google Cloud Pub/Sub. `Google Cloud Pub/Sub` .. _Google Cloud Pub/Sub: https://cloud.google.com/pubsub/docs + Setup ------------------------------------------------------------------------------- + Authentication ++++++++++++++ @@ -28,6 +31,9 @@ credentials for applications. .. _Authentication Getting Started Guide: https://cloud.google.com/docs/authentication/getting-started + + + Install Dependencies ++++++++++++++++++++ @@ -42,7 +48,7 @@ Install Dependencies .. _Python Development Environment Setup Guide: https://cloud.google.com/python/setup -#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. +#. Create a virtualenv. Samples are compatible with Python 3.6+. .. code-block:: bash @@ -58,9 +64,33 @@ Install Dependencies .. _pip: https://pip.pypa.io/ .. _virtualenv: https://virtualenv.pypa.io/ + + + + + Samples ------------------------------------------------------------------------------- + +Quickstart ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/quickstart.py,pubsub/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python quickstart.py + + + + Publisher +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -74,7 +104,8 @@ To run this sample: .. code-block:: bash - $ python publisher.py --help + $ python publisher.py + usage: publisher.py [-h] project_id @@ -111,6 +142,8 @@ To run this sample: + + Subscribers +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -124,7 +157,8 @@ To run this sample: .. code-block:: bash - $ python subscriber.py --help + $ python subscriber.py + usage: subscriber.py [-h] project_id @@ -173,6 +207,8 @@ To run this sample: + + Identity and Access Management +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -188,6 +224,7 @@ To run this sample: $ python iam.py + usage: iam.py [-h] project {get-topic-policy,get-subscription-policy,set-topic-policy,set-subscription-policy,check-topic-permissions,check-subscription-permissions} @@ -222,6 +259,10 @@ To run this sample: + + + + The client library ------------------------------------------------------------------------------- @@ -237,4 +278,5 @@ to `browse the source`_ and `report issues`_. https://github.com/GoogleCloudPlatform/google-cloud-python/issues + .. _Google Cloud SDK: https://cloud.google.com/sdk/ diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py new file mode 100644 index 000000000..ba55d7ce5 --- /dev/null +++ b/samples/snippets/noxfile.py @@ -0,0 +1,224 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +from pathlib import Path +import sys + +import nox + + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +# Copy `noxfile_config.py` to your directory and modify it instead. + + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + 'ignored_versions': ["2.7"], + + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + 'envs': {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append('.') + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars(): + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG['gcloud_project_env'] + # This should error out if not set. + ret['GOOGLE_CLOUD_PROJECT'] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG['envs']) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to tested samples. +ALL_VERSIONS = ["2.7", "3.6", "3.7", "3.8"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG['ignored_versions'] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = bool(os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False)) +# +# Style Checks +# + + +def _determine_local_import_names(start_dir): + """Determines all import names that should be considered "local". + + This is used when running the linter to insure that import order is + properly checked. + """ + file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] + return [ + basename + for basename, extension in file_ext_pairs + if extension == ".py" + or os.path.isdir(os.path.join(start_dir, basename)) + and basename not in ("__pycache__") + ] + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--import-order-style=google", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session): + session.install("flake8", "flake8-import-order") + + local_names = _determine_local_import_names(".") + args = FLAKE8_COMMON_ARGS + [ + "--application-import-names", + ",".join(local_names), + "." + ] + session.run("flake8", *args) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests(session, post_install=None): + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars() + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session): + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip("SKIPPED: {} tests are disabled for this sample.".format( + session.python + )) + + +# +# Readmegen +# + + +def _get_repo_root(): + """ Returns the root folder of the project. """ + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session, path): + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/synth.py b/synth.py index b44cc0acf..0e2c96e42 100644 --- a/synth.py +++ b/synth.py @@ -18,6 +18,7 @@ import synthtool as s from synthtool import gcp +from synthtool.languages import python gapic = gcp.GAPICBazel() common = gcp.CommonTemplates() @@ -266,8 +267,16 @@ def _merge_dict(d1, d2): # Add templated files # ---------------------------------------------------------------------------- templated_files = gcp.CommonTemplates().py_library( - unit_cov_level=97, cov_level=99, system_test_external_dependencies=["psutil"], + unit_cov_level=97, + cov_level=99, + system_test_external_dependencies=["psutil"], + samples=True, ) s.move(templated_files) +# ---------------------------------------------------------------------------- +# Samples templates +# ---------------------------------------------------------------------------- +python.py_samples() + s.shell.run(["nox", "-s", "blacken"], hide_output=False) From e48ed538603240dddb48576101c04cb82e0d6a23 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Fri, 10 Jul 2020 14:52:53 +0200 Subject: [PATCH 100/101] chore(deps): update google-cloud-pubsub to v1.6.1 This restores the chnge from 1cb6746b00 that got lost during the branch restoration. --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 9b496510a..42ab449b1 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-pubsub==1.6.0 +google-cloud-pubsub==1.6.1 From f11d069e5813882d7baf49544a465b5fa0b07db1 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Fri, 10 Jul 2020 15:58:22 +0200 Subject: [PATCH 101/101] Fix URLs, commands, etc. in samples README --- samples/snippets/README.rst | 54 +++++++++---------- samples/snippets/README.rst.in | 8 +-- scripts/readme-gen/templates/README.tmpl.rst | 4 +- .../templates/install_deps.tmpl.rst | 4 +- 4 files changed, 33 insertions(+), 37 deletions(-) diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst index 2676680af..40b2e21fc 100644 --- a/samples/snippets/README.rst +++ b/samples/snippets/README.rst @@ -1,11 +1,10 @@ - .. This file is automatically generated. Do not edit this file directly. Google Cloud Pub/Sub Python Samples =============================================================================== .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-pubsub&page=editor&open_in_editor=samples/snippets/README.rst This directory contains samples for Google Cloud Pub/Sub. `Google Cloud Pub/Sub`_ is a fully-managed real-time messaging service that allows you to send and receive messages between independent applications. @@ -16,11 +15,13 @@ This directory contains samples for Google Cloud Pub/Sub. `Google Cloud Pub/Sub` .. _Google Cloud Pub/Sub: https://cloud.google.com/pubsub/docs + + + Setup ------------------------------------------------------------------------------- - Authentication ++++++++++++++ @@ -31,9 +32,6 @@ credentials for applications. .. _Authentication Getting Started Guide: https://cloud.google.com/docs/authentication/getting-started - - - Install Dependencies ++++++++++++++++++++ @@ -41,7 +39,7 @@ Install Dependencies .. code-block:: bash - $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + $ git clone https://github.com/googleapis/python-pubsub.git #. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. @@ -64,20 +62,30 @@ Install Dependencies .. _pip: https://pip.pypa.io/ .. _virtualenv: https://virtualenv.pypa.io/ +Samples +------------------------------------------------------------------------------- +Quickstart (Publisher) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com//googleapis/python-pubsub&page=editor&open_in_editor=samples/snippets/quickstart/pub.py,samples/snippets/README.rst -Samples -------------------------------------------------------------------------------- +To run this sample: + +.. code-block:: bash -Quickstart + $ python quickstart/pub.py + + +Quickstart (Subscriber) +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/quickstart.py,pubsub/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com//googleapis/python-pubsub&page=editor&open_in_editor=samples/snippets/quickstart/sub.py,samples/snippets/README.rst @@ -86,16 +94,14 @@ To run this sample: .. code-block:: bash - $ python quickstart.py - - + $ python quickstart/sub.py Publisher +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/publisher.py,pubsub/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com//googleapis/python-pubsub&page=editor&open_in_editor=samples/snippets/publisher.py,samples/snippets/README.rst @@ -106,7 +112,6 @@ To run this sample: $ python publisher.py - usage: publisher.py [-h] project_id {list,create,delete,publish,publish-with-custom-attributes,publish-with-error-handler,publish-with-batch-settings,publish-with-retry-settings} @@ -142,13 +147,11 @@ To run this sample: - - Subscribers +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/subscriber.py,pubsub/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com//googleapis/python-pubsub&page=editor&open_in_editor=samples/snippets/subscriber.py,samples/snippets/README.rst @@ -159,7 +162,6 @@ To run this sample: $ python subscriber.py - usage: subscriber.py [-h] project_id {list-in-topic,list-in-project,create,create-with-dead-letter-policy,create-push,delete,update-push,update-dead-letter-policy,remove-dead-letter-policy,receive,receive-custom-attributes,receive-flow-control,receive-synchronously,receive-synchronously-with-lease,listen-for-errors,receive-messages-with-delivery-attempts} @@ -207,13 +209,11 @@ To run this sample: - - Identity and Access Management +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=pubsub/cloud-client/iam.py,pubsub/cloud-client/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com//googleapis/python-pubsub&page=editor&open_in_editor=samples/snippets/iam.py,samples/snippets/README.rst @@ -224,7 +224,6 @@ To run this sample: $ python iam.py - usage: iam.py [-h] project {get-topic-policy,get-subscription-policy,set-topic-policy,set-subscription-policy,check-topic-permissions,check-subscription-permissions} @@ -259,10 +258,6 @@ To run this sample: - - - - The client library ------------------------------------------------------------------------------- @@ -278,5 +273,4 @@ to `browse the source`_ and `report issues`_. https://github.com/GoogleCloudPlatform/google-cloud-python/issues - -.. _Google Cloud SDK: https://cloud.google.com/sdk/ +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/samples/snippets/README.rst.in b/samples/snippets/README.rst.in index ddbc64712..b0e98cbeb 100644 --- a/samples/snippets/README.rst.in +++ b/samples/snippets/README.rst.in @@ -13,8 +13,10 @@ setup: - install_deps samples: -- name: Quickstart - file: quickstart.py +- name: Quickstart (Publisher) + file: quickstart/pub.py +- name: Quickstart (Subscriber) + file: quickstart/sub.py - name: Publisher file: publisher.py show_help: true @@ -27,4 +29,4 @@ samples: cloud_client_library: true -folder: pubsub/cloud-client \ No newline at end of file +folder: samples/snippets \ No newline at end of file diff --git a/scripts/readme-gen/templates/README.tmpl.rst b/scripts/readme-gen/templates/README.tmpl.rst index 4fd239765..df252dd6e 100644 --- a/scripts/readme-gen/templates/README.tmpl.rst +++ b/scripts/readme-gen/templates/README.tmpl.rst @@ -6,7 +6,7 @@ =============================================================================== .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor={{folder}}/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-pubsub&page=editor&open_in_editor={{folder}}/README.rst This directory contains samples for {{product.name}}. {{product.description}} @@ -46,7 +46,7 @@ Samples {% if not sample.hide_cloudshell_button %} .. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor={{folder}}/{{sample.file}},{{folder}}/README.rst + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com//googleapis/python-pubsub&page=editor&open_in_editor={{folder}}/{{sample.file}},{{folder}}/README.rst {% endif %} diff --git a/scripts/readme-gen/templates/install_deps.tmpl.rst b/scripts/readme-gen/templates/install_deps.tmpl.rst index a0406dba8..7a47efe21 100644 --- a/scripts/readme-gen/templates/install_deps.tmpl.rst +++ b/scripts/readme-gen/templates/install_deps.tmpl.rst @@ -5,14 +5,14 @@ Install Dependencies .. code-block:: bash - $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + $ git clone https://github.com/googleapis/python-pubsub.git #. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. .. _Python Development Environment Setup Guide: https://cloud.google.com/python/setup -#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. +#. Create a virtualenv. Samples are compatible with Python 3.6+. .. code-block:: bash