From d54f501777593c8923efbe94aef303451a432c0c Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Thu, 14 Sep 2017 14:47:08 -0700 Subject: [PATCH 01/51] Add samples for Cloud Tasks [(#1068)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1068) * Add samples for Cloud Tasks * Respond to tasks sample review * Update app engine queues samples * Address review feedback * Address review issues and convert pull queue sample to not use API key auth * Reform pull queues to match appengine queues changes to auth, command line input, readme * flake8 and fix comment --- samples/snippets/README.md | 59 ++++++++ samples/snippets/pull_queue_snippets.py | 141 +++++++++++++++++++ samples/snippets/pull_queue_snippets_test.py | 35 +++++ samples/snippets/queue.yaml | 3 + samples/snippets/requirements.txt | 1 + 5 files changed, 239 insertions(+) create mode 100644 samples/snippets/README.md create mode 100644 samples/snippets/pull_queue_snippets.py create mode 100644 samples/snippets/pull_queue_snippets_test.py create mode 100644 samples/snippets/queue.yaml create mode 100644 samples/snippets/requirements.txt diff --git a/samples/snippets/README.md b/samples/snippets/README.md new file mode 100644 index 00000000..012c56ef --- /dev/null +++ b/samples/snippets/README.md @@ -0,0 +1,59 @@ +# Google Cloud Tasks Pull Queue Samples + +Sample command-line program for interacting with the Google Cloud Tasks API +using pull queues. + +Pull queues let you add tasks to a queue, then programatically remove and +interact with them. Tasks can be added or processed in any environment, +such as on Google App Engine or Google Compute Engine. + +`pull_queue_snippets.py` is a simple command-line program to demonstrate listing queues, + creating tasks, and pulling and acknowledging tasks. + +## Prerequisites to run locally: + +Please refer to [Setting Up a Python Development Environment](https://cloud.google.com/python/setup). + +## Authentication + +To set up authentication, please refer to our +[authentication getting started guide](https://cloud.google.com/docs/authentication/getting-started). + +## Creating a queue + +To create a queue using the Cloud SDK, use the following gcloud command: + + gcloud alpha tasks queues create-pull-queue my-pull-queue + +## Running the Samples + +Set the environment variables: + +Set environment variables: + +First, your project ID: + + export PROJECT_ID=my-project-id + +Then the queue ID, as specified at queue creation time. Queue IDs already +created can be listed with `gcloud alpha tasks queues list`. + + export QUEUE_ID=my-pull-queue + +And finally the location ID, which can be discovered with +`gcloud alpha tasks queues describe $QUEUE_ID`, with the location embedded in +the "name" value (for instance, if the name is +"projects/my-project/locations/us-central1/queues/my-pull-queue", then the +location is "us-central1"). + + export LOCATION_ID=us-central1 + +Create a task for a queue: + + python pull_queue_snippets.py create-task --project=$PROJECT_ID --queue=$QUEUE_ID --location=$LOCATION_ID + +Pull and acknowledge a task: + + python pull_queue_snippets.py pull-and-ack-task --project=$PROJECT_ID --queue=$QUEUE_ID --location=$LOCATION_ID + +Note that usually, there would be a processing step in between pulling a task and acknowledging it. diff --git a/samples/snippets/pull_queue_snippets.py b/samples/snippets/pull_queue_snippets.py new file mode 100644 index 00000000..8d898a58 --- /dev/null +++ b/samples/snippets/pull_queue_snippets.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sample command-line program for interacting with the Cloud Tasks API. + +See README.md for instructions on setting up your development environment +and running the scripts. +""" + +import argparse +import base64 + +from googleapiclient import discovery + + +def create_task(project, queue, location): + """Create a task for a given queue with an arbitrary payload.""" + + DISCOVERY_URL = ( + 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2') + client = discovery.build( + 'cloudtasks', 'v2beta2', discoveryServiceUrl=DISCOVERY_URL) + + payload = 'a message for the recipient' + task = { + 'task': { + 'pull_task_target': { + 'payload': base64.b64encode(payload) + } + } + } + + queue_name = 'projects/{}/locations/{}/queues/{}'.format( + project, location, queue) + + response = client.projects().locations().queues().tasks().create( + parent=queue_name, body=task).execute() + + print('Created task {}'.format(response['name'])) + return response + + +def pull_task(project, queue, location): + """Pull a single task from a given queue and lease it for 10 minutes.""" + + DISCOVERY_URL = ( + 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2') + client = discovery.build( + 'cloudtasks', 'v2beta2', discoveryServiceUrl=DISCOVERY_URL) + + duration_seconds = '600s' + pull_options = { + 'max_tasks': 1, + 'leaseDuration': duration_seconds, + 'responseView': 'FULL' + } + + queue_name = 'projects/{}/locations/{}/queues/{}'.format( + project, location, queue) + + response = client.projects().locations().queues().tasks().pull( + name=queue_name, body=pull_options).execute() + + print('Pulled task {}'.format(response)) + return response['tasks'][0] + + +def acknowledge_task(task): + """Acknowledge a given task.""" + + DISCOVERY_URL = ( + 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2') + client = discovery.build( + 'cloudtasks', 'v2beta2', discoveryServiceUrl=DISCOVERY_URL) + + body = {'scheduleTime': task['scheduleTime']} + client.projects().locations().queues().tasks().acknowledge( + name=task['name'], body=body).execute() + + print('Acknowledged task {}'.format(task['name'])) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + subparsers = parser.add_subparsers(dest='command') + + create_task_parser = subparsers.add_parser( + 'create-task', + help=create_task.__doc__) + create_task_parser.add_argument( + '--project', + help='Project of the queue to add the task to.' + ) + create_task_parser.add_argument( + '--queue', + help='ID (short name) of the queue to add the task to.' + ) + create_task_parser.add_argument( + '--location', + help='Location of the queue to add the task to.' + ) + + pull_and_ack_parser = subparsers.add_parser( + 'pull-and-ack-task', + help=create_task.__doc__) + pull_and_ack_parser.add_argument( + '--project', + help='Project of the queue to pull the task from.' + ) + pull_and_ack_parser.add_argument( + '--queue', + help='ID (short name) of the queue to pull the task from.' + ) + pull_and_ack_parser.add_argument( + '--location', + help='Location of the queue to pull the task from.' + ) + + args = parser.parse_args() + + if args.command == 'create-task': + create_task(args.project, args.queue, args.location) + if args.command == 'pull-and-ack-task': + task = pull_task(args.project, args.queue, args.location) + acknowledge_task(task) diff --git a/samples/snippets/pull_queue_snippets_test.py b/samples/snippets/pull_queue_snippets_test.py new file mode 100644 index 00000000..6905fdb3 --- /dev/null +++ b/samples/snippets/pull_queue_snippets_test.py @@ -0,0 +1,35 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pull_queue_snippets + +TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') +TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') +TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'test-queue') + + +def test_create_task(): + result = pull_queue_snippets.create_task( + TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) + assert TEST_QUEUE_NAME in result['name'] + + +def test_pull_and_ack_task(): + pull_queue_snippets.create_task( + TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) + task = pull_queue_snippets.pull_task( + TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) + pull_queue_snippets.acknowledge_task(task) diff --git a/samples/snippets/queue.yaml b/samples/snippets/queue.yaml new file mode 100644 index 00000000..08c1503b --- /dev/null +++ b/samples/snippets/queue.yaml @@ -0,0 +1,3 @@ +queue: +- name: my-pull-queue + mode: pull diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt new file mode 100644 index 00000000..b7233cdf --- /dev/null +++ b/samples/snippets/requirements.txt @@ -0,0 +1 @@ +google-api-python-client==1.6.0 From fee6b9e8fc609bcad7ad32d4f7b1ea7f2bad9fc4 Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Thu, 14 Sep 2017 15:31:55 -0700 Subject: [PATCH 02/51] Fix Tasks sample test issues. --- samples/snippets/pull_queue_snippets.py | 2 +- samples/snippets/pull_queue_snippets_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/pull_queue_snippets.py b/samples/snippets/pull_queue_snippets.py index 8d898a58..9fb08e1e 100644 --- a/samples/snippets/pull_queue_snippets.py +++ b/samples/snippets/pull_queue_snippets.py @@ -38,7 +38,7 @@ def create_task(project, queue, location): task = { 'task': { 'pull_task_target': { - 'payload': base64.b64encode(payload) + 'payload': base64.b64encode(payload.encode()).decode() } } } diff --git a/samples/snippets/pull_queue_snippets_test.py b/samples/snippets/pull_queue_snippets_test.py index 6905fdb3..e7f9f170 100644 --- a/samples/snippets/pull_queue_snippets_test.py +++ b/samples/snippets/pull_queue_snippets_test.py @@ -18,7 +18,7 @@ TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') -TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'test-queue') +TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-pull-queue') def test_create_task(): From 8ded308859ddfa2748d0fb80ee1f0cc563314541 Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Thu, 14 Sep 2017 15:49:31 -0700 Subject: [PATCH 03/51] Remove queue.yaml, now unused --- samples/snippets/queue.yaml | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 samples/snippets/queue.yaml diff --git a/samples/snippets/queue.yaml b/samples/snippets/queue.yaml deleted file mode 100644 index 08c1503b..00000000 --- a/samples/snippets/queue.yaml +++ /dev/null @@ -1,3 +0,0 @@ -queue: -- name: my-pull-queue - mode: pull From 3b2a2b56a3b0818ac7051108d2eb11e868756f65 Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Thu, 14 Sep 2017 16:20:43 -0700 Subject: [PATCH 04/51] Add required flag on certain arguments to command-line tools --- samples/snippets/pull_queue_snippets.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/samples/snippets/pull_queue_snippets.py b/samples/snippets/pull_queue_snippets.py index 9fb08e1e..1aa0f8af 100644 --- a/samples/snippets/pull_queue_snippets.py +++ b/samples/snippets/pull_queue_snippets.py @@ -105,15 +105,18 @@ def acknowledge_task(task): help=create_task.__doc__) create_task_parser.add_argument( '--project', - help='Project of the queue to add the task to.' + help='Project of the queue to add the task to.', + required=True, ) create_task_parser.add_argument( '--queue', - help='ID (short name) of the queue to add the task to.' + help='ID (short name) of the queue to add the task to.', + required=True, ) create_task_parser.add_argument( '--location', - help='Location of the queue to add the task to.' + help='Location of the queue to add the task to.', + required=True, ) pull_and_ack_parser = subparsers.add_parser( @@ -121,15 +124,18 @@ def acknowledge_task(task): help=create_task.__doc__) pull_and_ack_parser.add_argument( '--project', - help='Project of the queue to pull the task from.' + help='Project of the queue to pull the task from.', + required=True, ) pull_and_ack_parser.add_argument( '--queue', - help='ID (short name) of the queue to pull the task from.' + help='ID (short name) of the queue to pull the task from.', + required=True, ) pull_and_ack_parser.add_argument( '--location', - help='Location of the queue to pull the task from.' + help='Location of the queue to pull the task from.', + required=True, ) args = parser.parse_args() From 60f52fffa08c123195fe9658310574d8e96bde86 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Fri, 15 Sep 2017 09:39:04 -0700 Subject: [PATCH 05/51] Auto-update dependencies. [(#1116)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1116) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index b7233cdf..28ef8912 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-api-python-client==1.6.0 +google-api-python-client==1.6.3 From f8eb901872c0d7680751cf795b127b9ebcac3bc4 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Thu, 21 Sep 2017 13:40:34 -0700 Subject: [PATCH 06/51] Auto-update dependencies. [(#1133)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1133) * Auto-update dependencies. * Fix missing http library Change-Id: I99faa600f2f3f1f50f57694fc9835d7f35bda250 --- samples/snippets/requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 28ef8912..af5ec814 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1,3 @@ -google-api-python-client==1.6.3 +google-api-python-client==1.6.4 +google-auth==1.1.1 +google-auth-httplib2==0.0.2 From 35ad1ab32a28a7bbedf2545887ce9d0bf6ad9564 Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Fri, 22 Sep 2017 16:53:11 -0700 Subject: [PATCH 07/51] Update Task Queue samples for beta --- samples/snippets/pull_queue_snippets.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/samples/snippets/pull_queue_snippets.py b/samples/snippets/pull_queue_snippets.py index 1aa0f8af..800fed7a 100644 --- a/samples/snippets/pull_queue_snippets.py +++ b/samples/snippets/pull_queue_snippets.py @@ -23,21 +23,19 @@ import argparse import base64 -from googleapiclient import discovery - def create_task(project, queue, location): """Create a task for a given queue with an arbitrary payload.""" - DISCOVERY_URL = ( - 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2') + # Create a client. + from googleapiclient import discovery client = discovery.build( - 'cloudtasks', 'v2beta2', discoveryServiceUrl=DISCOVERY_URL) + 'cloudtasks', 'v2beta2') payload = 'a message for the recipient' task = { 'task': { - 'pull_task_target': { + 'pull_message': { 'payload': base64.b64encode(payload.encode()).decode() } } @@ -56,10 +54,10 @@ def create_task(project, queue, location): def pull_task(project, queue, location): """Pull a single task from a given queue and lease it for 10 minutes.""" - DISCOVERY_URL = ( - 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2') + # Create a client. + from googleapiclient import discovery client = discovery.build( - 'cloudtasks', 'v2beta2', discoveryServiceUrl=DISCOVERY_URL) + 'cloudtasks', 'v2beta2') duration_seconds = '600s' pull_options = { @@ -81,10 +79,10 @@ def pull_task(project, queue, location): def acknowledge_task(task): """Acknowledge a given task.""" - DISCOVERY_URL = ( - 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2') + # Create a client. + from googleapiclient import discovery client = discovery.build( - 'cloudtasks', 'v2beta2', discoveryServiceUrl=DISCOVERY_URL) + 'cloudtasks', 'v2beta2') body = {'scheduleTime': task['scheduleTime']} client.projects().locations().queues().tasks().acknowledge( From 64b115cb7283defc71b0aeaf63cba85a36594556 Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Mon, 25 Sep 2017 10:48:08 -0700 Subject: [PATCH 08/51] Use full import to get client --- samples/snippets/pull_queue_snippets.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/samples/snippets/pull_queue_snippets.py b/samples/snippets/pull_queue_snippets.py index 800fed7a..803849b2 100644 --- a/samples/snippets/pull_queue_snippets.py +++ b/samples/snippets/pull_queue_snippets.py @@ -27,10 +27,10 @@ def create_task(project, queue, location): """Create a task for a given queue with an arbitrary payload.""" + import googleapiclient.discovery + # Create a client. - from googleapiclient import discovery - client = discovery.build( - 'cloudtasks', 'v2beta2') + client = googleapiclient.discovery.build('cloudtasks', 'v2beta2') payload = 'a message for the recipient' task = { @@ -54,10 +54,10 @@ def create_task(project, queue, location): def pull_task(project, queue, location): """Pull a single task from a given queue and lease it for 10 minutes.""" + import googleapiclient.discovery + # Create a client. - from googleapiclient import discovery - client = discovery.build( - 'cloudtasks', 'v2beta2') + client = googleapiclient.discovery.build('cloudtasks', 'v2beta2') duration_seconds = '600s' pull_options = { @@ -79,10 +79,10 @@ def pull_task(project, queue, location): def acknowledge_task(task): """Acknowledge a given task.""" + import googleapiclient.discovery + # Create a client. - from googleapiclient import discovery - client = discovery.build( - 'cloudtasks', 'v2beta2') + client = googleapiclient.discovery.build('cloudtasks', 'v2beta2') body = {'scheduleTime': task['scheduleTime']} client.projects().locations().queues().tasks().acknowledge( From 86b77226315e5111636248c6abee7726877f7250 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Wed, 1 Nov 2017 12:30:10 -0700 Subject: [PATCH 09/51] Auto-update dependencies. [(#1186)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1186) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index af5ec814..558e42c2 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,3 @@ google-api-python-client==1.6.4 -google-auth==1.1.1 +google-auth==1.2.0 google-auth-httplib2==0.0.2 From f6e4072c28338b3ad8538fd4cd4288d544da19c9 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Wed, 15 Nov 2017 12:18:33 -0800 Subject: [PATCH 10/51] Auto-update dependencies. [(#1217)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1217) --- samples/snippets/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 558e42c2..edd6472f 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,3 @@ google-api-python-client==1.6.4 -google-auth==1.2.0 -google-auth-httplib2==0.0.2 +google-auth==1.2.1 +google-auth-httplib2==0.0.3 From c292cfeb2c85bd835101cd5cad0aa19ed4171ae7 Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Thu, 7 Dec 2017 10:34:29 -0800 Subject: [PATCH 11/51] Added "Open in Cloud Shell" buttons to README files [(#1254)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1254) --- samples/snippets/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/samples/snippets/README.md b/samples/snippets/README.md index 012c56ef..ddba20de 100644 --- a/samples/snippets/README.md +++ b/samples/snippets/README.md @@ -1,5 +1,10 @@ # Google Cloud Tasks Pull Queue Samples +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=tasks/README.md + Sample command-line program for interacting with the Google Cloud Tasks API using pull queues. From 2477d24944101bac61f8cf2324a0f23194535268 Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Tue, 12 Dec 2017 09:25:25 -0800 Subject: [PATCH 12/51] Add comments and region tags to Cloud Tasks samples [(#1271)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1271) --- samples/snippets/pull_queue_snippets.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/samples/snippets/pull_queue_snippets.py b/samples/snippets/pull_queue_snippets.py index 803849b2..c2c0c50b 100644 --- a/samples/snippets/pull_queue_snippets.py +++ b/samples/snippets/pull_queue_snippets.py @@ -24,6 +24,7 @@ import base64 +# [START cloud_tasks_create_task] def create_task(project, queue, location): """Create a task for a given queue with an arbitrary payload.""" @@ -32,25 +33,40 @@ def create_task(project, queue, location): # Create a client. client = googleapiclient.discovery.build('cloudtasks', 'v2beta2') + # Prepare the payload. payload = 'a message for the recipient' + + # The API expects base64 encoding of the payload, so encode the unicode + # `payload` object into a byte string and base64 encode it. + base64_encoded_payload = base64.b64encode(payload.encode()) + + # The request body object will be emitted in JSON, which requires + # unicode objects, so convert the byte string to unicode (still base64). + converted_payload = base64_encoded_payload.decode() + + # Construct the request body. task = { 'task': { 'pull_message': { - 'payload': base64.b64encode(payload.encode()).decode() + 'payload': converted_payload } } } + # Construct the fully qualified queue name. queue_name = 'projects/{}/locations/{}/queues/{}'.format( project, location, queue) + # Use the client to build and send the task. response = client.projects().locations().queues().tasks().create( parent=queue_name, body=task).execute() print('Created task {}'.format(response['name'])) return response +# [END cloud_tasks_create_task] +# [START cloud_tasks_pull_task] def pull_task(project, queue, location): """Pull a single task from a given queue and lease it for 10 minutes.""" @@ -74,6 +90,7 @@ def pull_task(project, queue, location): print('Pulled task {}'.format(response)) return response['tasks'][0] +# [END cloud_tasks_pull_task] def acknowledge_task(task): From adf58f8560ae5456eb57fc012212ab5602cedfef Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Tue, 19 Dec 2017 12:42:02 -0800 Subject: [PATCH 13/51] Standardize on CamelCase, reword confusing endpoint name [(#1288)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1288) --- samples/snippets/pull_queue_snippets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/pull_queue_snippets.py b/samples/snippets/pull_queue_snippets.py index c2c0c50b..360ceb2b 100644 --- a/samples/snippets/pull_queue_snippets.py +++ b/samples/snippets/pull_queue_snippets.py @@ -47,7 +47,7 @@ def create_task(project, queue, location): # Construct the request body. task = { 'task': { - 'pull_message': { + 'pullMessage': { 'payload': converted_payload } } @@ -77,7 +77,7 @@ def pull_task(project, queue, location): duration_seconds = '600s' pull_options = { - 'max_tasks': 1, + 'maxTasks': 1, 'leaseDuration': duration_seconds, 'responseView': 'FULL' } From 2dcc80e4a5332047e921ba1d74d70e65823fa08c Mon Sep 17 00:00:00 2001 From: DPE bot Date: Wed, 10 Jan 2018 09:07:00 -0800 Subject: [PATCH 14/51] Auto-update dependencies. [(#1309)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1309) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index edd6472f..8bb83f80 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,3 @@ google-api-python-client==1.6.4 -google-auth==1.2.1 +google-auth==1.3.0 google-auth-httplib2==0.0.3 From e47269cf55252b4f21bf231ac6e79f5bb3aa52ce Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Fri, 12 Jan 2018 14:44:52 -0800 Subject: [PATCH 15/51] Rename pull to lease and fix name/parent confusion [(#1311)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1311) --- samples/snippets/pull_queue_snippets.py | 38 ++++++++++---------- samples/snippets/pull_queue_snippets_test.py | 4 +-- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/samples/snippets/pull_queue_snippets.py b/samples/snippets/pull_queue_snippets.py index 360ceb2b..12ac0e95 100644 --- a/samples/snippets/pull_queue_snippets.py +++ b/samples/snippets/pull_queue_snippets.py @@ -67,8 +67,8 @@ def create_task(project, queue, location): # [START cloud_tasks_pull_task] -def pull_task(project, queue, location): - """Pull a single task from a given queue and lease it for 10 minutes.""" +def lease_task(project, queue, location): + """Lease a single task from a given queue for 10 minutes.""" import googleapiclient.discovery @@ -76,7 +76,7 @@ def pull_task(project, queue, location): client = googleapiclient.discovery.build('cloudtasks', 'v2beta2') duration_seconds = '600s' - pull_options = { + lease_options = { 'maxTasks': 1, 'leaseDuration': duration_seconds, 'responseView': 'FULL' @@ -85,10 +85,10 @@ def pull_task(project, queue, location): queue_name = 'projects/{}/locations/{}/queues/{}'.format( project, location, queue) - response = client.projects().locations().queues().tasks().pull( - name=queue_name, body=pull_options).execute() + response = client.projects().locations().queues().tasks().lease( + parent=queue_name, body=lease_options).execute() - print('Pulled task {}'.format(response)) + print('Leased task {}'.format(response)) return response['tasks'][0] # [END cloud_tasks_pull_task] @@ -120,36 +120,36 @@ def acknowledge_task(task): help=create_task.__doc__) create_task_parser.add_argument( '--project', - help='Project of the queue to add the task to.', + help='Project ID.', required=True, ) create_task_parser.add_argument( '--queue', - help='ID (short name) of the queue to add the task to.', + help='Queue ID (short name).', required=True, ) create_task_parser.add_argument( '--location', - help='Location of the queue to add the task to.', + help='Location of the queue, e.g. \'us-central1\'.', required=True, ) - pull_and_ack_parser = subparsers.add_parser( - 'pull-and-ack-task', + lease_and_ack_parser = subparsers.add_parser( + 'lease-and-ack-task', help=create_task.__doc__) - pull_and_ack_parser.add_argument( + lease_and_ack_parser.add_argument( '--project', - help='Project of the queue to pull the task from.', + help='Project ID.', required=True, ) - pull_and_ack_parser.add_argument( + lease_and_ack_parser.add_argument( '--queue', - help='ID (short name) of the queue to pull the task from.', + help='Queue ID (short name).', required=True, ) - pull_and_ack_parser.add_argument( + lease_and_ack_parser.add_argument( '--location', - help='Location of the queue to pull the task from.', + help='Location of the queue, e.g. \'us-central1\'.', required=True, ) @@ -157,6 +157,6 @@ def acknowledge_task(task): if args.command == 'create-task': create_task(args.project, args.queue, args.location) - if args.command == 'pull-and-ack-task': - task = pull_task(args.project, args.queue, args.location) + if args.command == 'lease-and-ack-task': + task = lease_task(args.project, args.queue, args.location) acknowledge_task(task) diff --git a/samples/snippets/pull_queue_snippets_test.py b/samples/snippets/pull_queue_snippets_test.py index e7f9f170..db514d67 100644 --- a/samples/snippets/pull_queue_snippets_test.py +++ b/samples/snippets/pull_queue_snippets_test.py @@ -27,9 +27,9 @@ def test_create_task(): assert TEST_QUEUE_NAME in result['name'] -def test_pull_and_ack_task(): +def test_lease_and_ack_task(): pull_queue_snippets.create_task( TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) - task = pull_queue_snippets.pull_task( + task = pull_queue_snippets.lease_task( TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) pull_queue_snippets.acknowledge_task(task) From 3e9bd49968283ebae8edd0f67db5d71fa1ff4373 Mon Sep 17 00:00:00 2001 From: ellenevans <35748459+ellenevans@users.noreply.github.com> Date: Wed, 24 Jan 2018 15:47:21 -0800 Subject: [PATCH 16/51] Updated region tags for pull queue samples [(#1329)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1329) --- samples/snippets/pull_queue_snippets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/pull_queue_snippets.py b/samples/snippets/pull_queue_snippets.py index 12ac0e95..61bcc84e 100644 --- a/samples/snippets/pull_queue_snippets.py +++ b/samples/snippets/pull_queue_snippets.py @@ -66,7 +66,7 @@ def create_task(project, queue, location): # [END cloud_tasks_create_task] -# [START cloud_tasks_pull_task] +# [START cloud_tasks_lease_and_acknowledge_task] def lease_task(project, queue, location): """Lease a single task from a given queue for 10 minutes.""" @@ -90,7 +90,6 @@ def lease_task(project, queue, location): print('Leased task {}'.format(response)) return response['tasks'][0] -# [END cloud_tasks_pull_task] def acknowledge_task(task): @@ -106,6 +105,7 @@ def acknowledge_task(task): name=task['name'], body=body).execute() print('Acknowledged task {}'.format(task['name'])) + # [END cloud_tasks_lease_and_acknowledge_task] if __name__ == '__main__': From e4288727048f48476c8f1a35121e285266406b36 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Thu, 1 Feb 2018 22:20:35 -0800 Subject: [PATCH 17/51] Auto-update dependencies. [(#1320)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1320) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 8bb83f80..4bafec30 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,3 @@ -google-api-python-client==1.6.4 +google-api-python-client==1.6.5 google-auth==1.3.0 google-auth-httplib2==0.0.3 From 2f1622926490bcd7db3523eae76c55ceedf3b4ce Mon Sep 17 00:00:00 2001 From: DPE bot Date: Fri, 9 Feb 2018 10:46:48 -0800 Subject: [PATCH 18/51] Auto-update dependencies. [(#1355)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1355) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 4bafec30..35784043 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,3 @@ google-api-python-client==1.6.5 -google-auth==1.3.0 +google-auth==1.4.0 google-auth-httplib2==0.0.3 From 04a28d59e58fbc13efaab4ba6aca9ba2512372bd Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 26 Feb 2018 09:03:37 -0800 Subject: [PATCH 19/51] Auto-update dependencies. [(#1359)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1359) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 35784043..500e732f 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,3 @@ google-api-python-client==1.6.5 -google-auth==1.4.0 +google-auth==1.4.1 google-auth-httplib2==0.0.3 From 4b004d26ac8751e3d43b3e941761bacc4363a9b0 Mon Sep 17 00:00:00 2001 From: DPE bot Date: Mon, 2 Apr 2018 02:51:10 -0700 Subject: [PATCH 20/51] Auto-update dependencies. --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 500e732f..e5f3a6c5 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,3 @@ -google-api-python-client==1.6.5 +google-api-python-client==1.6.6 google-auth==1.4.1 google-auth-httplib2==0.0.3 From 330ee9cd202195d741b981f380a2182eaa0f197a Mon Sep 17 00:00:00 2001 From: Averi Kitsch Date: Mon, 18 Jun 2018 16:31:21 -0700 Subject: [PATCH 21/51] Update Cloud Tasks Samples [(#1529)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1529) * passing create task * Passing tests * updates to region tags --- samples/snippets/pull_queue_snippets.py | 78 ++++++++------------ samples/snippets/pull_queue_snippets_test.py | 5 +- samples/snippets/requirements.txt | 4 +- 3 files changed, 35 insertions(+), 52 deletions(-) diff --git a/samples/snippets/pull_queue_snippets.py b/samples/snippets/pull_queue_snippets.py index 61bcc84e..9abcbcb3 100644 --- a/samples/snippets/pull_queue_snippets.py +++ b/samples/snippets/pull_queue_snippets.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright 2017 Google Inc. +# Copyright 2018 Google Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,91 +21,75 @@ """ import argparse -import base64 -# [START cloud_tasks_create_task] def create_task(project, queue, location): + # [START tasks_create_task] """Create a task for a given queue with an arbitrary payload.""" - import googleapiclient.discovery + from google.cloud import tasks_v2beta2 # Create a client. - client = googleapiclient.discovery.build('cloudtasks', 'v2beta2') + client = tasks_v2beta2.CloudTasksClient() - # Prepare the payload. - payload = 'a message for the recipient' - - # The API expects base64 encoding of the payload, so encode the unicode - # `payload` object into a byte string and base64 encode it. - base64_encoded_payload = base64.b64encode(payload.encode()) - - # The request body object will be emitted in JSON, which requires - # unicode objects, so convert the byte string to unicode (still base64). - converted_payload = base64_encoded_payload.decode() + # Prepare the payload of type bytes. + payload = 'a message for the recipient'.encode() # Construct the request body. task = { - 'task': { - 'pullMessage': { - 'payload': converted_payload - } + 'pull_message': { + 'payload': payload, } } # Construct the fully qualified queue name. - queue_name = 'projects/{}/locations/{}/queues/{}'.format( - project, location, queue) + parent = client.queue_path(project, location, queue) # Use the client to build and send the task. - response = client.projects().locations().queues().tasks().create( - parent=queue_name, body=task).execute() + response = client.create_task(parent, task) - print('Created task {}'.format(response['name'])) + print('Created task: {}'.format(response.name)) return response -# [END cloud_tasks_create_task] + # [END tasks_create_task] -# [START cloud_tasks_lease_and_acknowledge_task] def lease_task(project, queue, location): + # [START tasks_lease_and_acknowledge_task] """Lease a single task from a given queue for 10 minutes.""" - import googleapiclient.discovery + from google.cloud import tasks_v2beta2 # Create a client. - client = googleapiclient.discovery.build('cloudtasks', 'v2beta2') + client = tasks_v2beta2.CloudTasksClient() - duration_seconds = '600s' - lease_options = { - 'maxTasks': 1, - 'leaseDuration': duration_seconds, - 'responseView': 'FULL' - } + # Construct the fully qualified queue name. + parent = client.queue_path(project, location, queue) + + lease_duration = {'seconds': 600} - queue_name = 'projects/{}/locations/{}/queues/{}'.format( - project, location, queue) + # Send lease request to client. + response = client.lease_tasks( + parent, lease_duration, max_tasks=1, response_view='FULL') - response = client.projects().locations().queues().tasks().lease( - parent=queue_name, body=lease_options).execute() + task = response.tasks[0] - print('Leased task {}'.format(response)) - return response['tasks'][0] + print('Leased task {}'.format(task.name)) + return task def acknowledge_task(task): """Acknowledge a given task.""" - import googleapiclient.discovery + from google.cloud import tasks_v2beta2 # Create a client. - client = googleapiclient.discovery.build('cloudtasks', 'v2beta2') + client = tasks_v2beta2.CloudTasksClient() - body = {'scheduleTime': task['scheduleTime']} - client.projects().locations().queues().tasks().acknowledge( - name=task['name'], body=body).execute() + # Send request to client to acknowledge task. + client.acknowledge_task(task.name, task.schedule_time) - print('Acknowledged task {}'.format(task['name'])) - # [END cloud_tasks_lease_and_acknowledge_task] + print('Acknowledged task {}'.format(task.name)) + # [END tasks_lease_and_acknowledge_task] if __name__ == '__main__': diff --git a/samples/snippets/pull_queue_snippets_test.py b/samples/snippets/pull_queue_snippets_test.py index db514d67..69797f15 100644 --- a/samples/snippets/pull_queue_snippets_test.py +++ b/samples/snippets/pull_queue_snippets_test.py @@ -1,4 +1,4 @@ -# Copyright 2017 Google Inc. +# Copyright 2018 Google Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ def test_create_task(): result = pull_queue_snippets.create_task( TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) - assert TEST_QUEUE_NAME in result['name'] + assert TEST_QUEUE_NAME in result.name def test_lease_and_ack_task(): @@ -32,4 +32,5 @@ def test_lease_and_ack_task(): TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) task = pull_queue_snippets.lease_task( TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) + assert TEST_QUEUE_NAME in task.name pull_queue_snippets.acknowledge_task(task) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 500e732f..e46eda26 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1 @@ -google-api-python-client==1.6.5 -google-auth==1.4.1 -google-auth-httplib2==0.0.3 +google-cloud-tasks==0.2.0 From b343440c096e7657ec3c759a06869d365460705d Mon Sep 17 00:00:00 2001 From: Averi Kitsch Date: Wed, 20 Jun 2018 15:19:25 -0700 Subject: [PATCH 22/51] update region tags [(#1532)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1532) --- samples/snippets/pull_queue_snippets.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/snippets/pull_queue_snippets.py b/samples/snippets/pull_queue_snippets.py index 9abcbcb3..1f91b074 100644 --- a/samples/snippets/pull_queue_snippets.py +++ b/samples/snippets/pull_queue_snippets.py @@ -24,7 +24,7 @@ def create_task(project, queue, location): - # [START tasks_create_task] + # [START cloud_tasks_create_task] """Create a task for a given queue with an arbitrary payload.""" from google.cloud import tasks_v2beta2 @@ -50,11 +50,11 @@ def create_task(project, queue, location): print('Created task: {}'.format(response.name)) return response - # [END tasks_create_task] + # [END cloud_tasks_create_task] def lease_task(project, queue, location): - # [START tasks_lease_and_acknowledge_task] + # [START cloud_tasks_lease_and_acknowledge_task] """Lease a single task from a given queue for 10 minutes.""" from google.cloud import tasks_v2beta2 @@ -89,7 +89,7 @@ def acknowledge_task(task): client.acknowledge_task(task.name, task.schedule_time) print('Acknowledged task {}'.format(task.name)) - # [END tasks_lease_and_acknowledge_task] + # [END cloud_tasks_lease_and_acknowledge_task] if __name__ == '__main__': From f058659c67200dbfdf03a03729cf04a792ee27e9 Mon Sep 17 00:00:00 2001 From: Averi Kitsch Date: Wed, 27 Jun 2018 16:40:30 -0700 Subject: [PATCH 23/51] update Tasks Sample for App Engine [(#1541)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1541) * update gcloud command for creating queues * deploys and runs * update license * passing tests --- samples/snippets/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/README.md b/samples/snippets/README.md index ddba20de..182ee2cc 100644 --- a/samples/snippets/README.md +++ b/samples/snippets/README.md @@ -28,7 +28,7 @@ To set up authentication, please refer to our To create a queue using the Cloud SDK, use the following gcloud command: - gcloud alpha tasks queues create-pull-queue my-pull-queue + gcloud alpha tasks queues create pull my-pull-queue ## Running the Samples From 8b2e59df952a760f311f6774313686fdd9027ae6 Mon Sep 17 00:00:00 2001 From: Averi Kitsch Date: Fri, 13 Jul 2018 14:15:04 -0700 Subject: [PATCH 24/51] Fix run command [(#1563)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1563) --- samples/snippets/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/README.md b/samples/snippets/README.md index 182ee2cc..b5eeb48e 100644 --- a/samples/snippets/README.md +++ b/samples/snippets/README.md @@ -59,6 +59,6 @@ Create a task for a queue: Pull and acknowledge a task: - python pull_queue_snippets.py pull-and-ack-task --project=$PROJECT_ID --queue=$QUEUE_ID --location=$LOCATION_ID + python pull_queue_snippets.py lease-and-ack-task --project=$PROJECT_ID --queue=$QUEUE_ID --location=$LOCATION_ID Note that usually, there would be a processing step in between pulling a task and acknowledging it. From 80640846cba2d3edf82a66a8344b1bced0b6dd79 Mon Sep 17 00:00:00 2001 From: Noah Negrey Date: Fri, 13 Jul 2018 14:34:26 -0700 Subject: [PATCH 25/51] Updated library for TTS GA [(#1552)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1552) From d19a210701b731159830f8270d0b4f2e1d1d473c Mon Sep 17 00:00:00 2001 From: Averi Kitsch Date: Tue, 17 Jul 2018 09:04:37 -0700 Subject: [PATCH 26/51] update gcloud command for Cloud Tasks [(#1566)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1566) * update gcloud command * update pull queue command * update pull queue command --- samples/snippets/README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/samples/snippets/README.md b/samples/snippets/README.md index b5eeb48e..407f7db2 100644 --- a/samples/snippets/README.md +++ b/samples/snippets/README.md @@ -28,25 +28,23 @@ To set up authentication, please refer to our To create a queue using the Cloud SDK, use the following gcloud command: - gcloud alpha tasks queues create pull my-pull-queue + gcloud beta tasks queues create-pull-queue my-pull-queue ## Running the Samples Set the environment variables: -Set environment variables: - First, your project ID: export PROJECT_ID=my-project-id Then the queue ID, as specified at queue creation time. Queue IDs already -created can be listed with `gcloud alpha tasks queues list`. +created can be listed with `gcloud beta tasks queues list`. export QUEUE_ID=my-pull-queue And finally the location ID, which can be discovered with -`gcloud alpha tasks queues describe $QUEUE_ID`, with the location embedded in +`gcloud beta tasks queues describe $QUEUE_ID`, with the location embedded in the "name" value (for instance, if the name is "projects/my-project/locations/us-central1/queues/my-pull-queue", then the location is "us-central1"). From d361fc2b8545a775b8ab426fc036b2d6a56aa0bd Mon Sep 17 00:00:00 2001 From: Averi Kitsch Date: Wed, 12 Sep 2018 12:42:22 -0700 Subject: [PATCH 27/51] Update Cloud Tasks Push Queue Sample [(#1698)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1698) * deleted pull queues * updated samples * fix dependency versions --- samples/snippets/README.md | 62 -------- samples/snippets/pull_queue_snippets.py | 146 ------------------- samples/snippets/pull_queue_snippets_test.py | 36 ----- samples/snippets/requirements.txt | 1 - 4 files changed, 245 deletions(-) delete mode 100644 samples/snippets/README.md delete mode 100644 samples/snippets/pull_queue_snippets.py delete mode 100644 samples/snippets/pull_queue_snippets_test.py delete mode 100644 samples/snippets/requirements.txt diff --git a/samples/snippets/README.md b/samples/snippets/README.md deleted file mode 100644 index 407f7db2..00000000 --- a/samples/snippets/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# Google Cloud Tasks Pull Queue Samples - -[![Open in Cloud Shell][shell_img]][shell_link] - -[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png -[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=tasks/README.md - -Sample command-line program for interacting with the Google Cloud Tasks API -using pull queues. - -Pull queues let you add tasks to a queue, then programatically remove and -interact with them. Tasks can be added or processed in any environment, -such as on Google App Engine or Google Compute Engine. - -`pull_queue_snippets.py` is a simple command-line program to demonstrate listing queues, - creating tasks, and pulling and acknowledging tasks. - -## Prerequisites to run locally: - -Please refer to [Setting Up a Python Development Environment](https://cloud.google.com/python/setup). - -## Authentication - -To set up authentication, please refer to our -[authentication getting started guide](https://cloud.google.com/docs/authentication/getting-started). - -## Creating a queue - -To create a queue using the Cloud SDK, use the following gcloud command: - - gcloud beta tasks queues create-pull-queue my-pull-queue - -## Running the Samples - -Set the environment variables: - -First, your project ID: - - export PROJECT_ID=my-project-id - -Then the queue ID, as specified at queue creation time. Queue IDs already -created can be listed with `gcloud beta tasks queues list`. - - export QUEUE_ID=my-pull-queue - -And finally the location ID, which can be discovered with -`gcloud beta tasks queues describe $QUEUE_ID`, with the location embedded in -the "name" value (for instance, if the name is -"projects/my-project/locations/us-central1/queues/my-pull-queue", then the -location is "us-central1"). - - export LOCATION_ID=us-central1 - -Create a task for a queue: - - python pull_queue_snippets.py create-task --project=$PROJECT_ID --queue=$QUEUE_ID --location=$LOCATION_ID - -Pull and acknowledge a task: - - python pull_queue_snippets.py lease-and-ack-task --project=$PROJECT_ID --queue=$QUEUE_ID --location=$LOCATION_ID - -Note that usually, there would be a processing step in between pulling a task and acknowledging it. diff --git a/samples/snippets/pull_queue_snippets.py b/samples/snippets/pull_queue_snippets.py deleted file mode 100644 index 1f91b074..00000000 --- a/samples/snippets/pull_queue_snippets.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Sample command-line program for interacting with the Cloud Tasks API. - -See README.md for instructions on setting up your development environment -and running the scripts. -""" - -import argparse - - -def create_task(project, queue, location): - # [START cloud_tasks_create_task] - """Create a task for a given queue with an arbitrary payload.""" - - from google.cloud import tasks_v2beta2 - - # Create a client. - client = tasks_v2beta2.CloudTasksClient() - - # Prepare the payload of type bytes. - payload = 'a message for the recipient'.encode() - - # Construct the request body. - task = { - 'pull_message': { - 'payload': payload, - } - } - - # Construct the fully qualified queue name. - parent = client.queue_path(project, location, queue) - - # Use the client to build and send the task. - response = client.create_task(parent, task) - - print('Created task: {}'.format(response.name)) - return response - # [END cloud_tasks_create_task] - - -def lease_task(project, queue, location): - # [START cloud_tasks_lease_and_acknowledge_task] - """Lease a single task from a given queue for 10 minutes.""" - - from google.cloud import tasks_v2beta2 - - # Create a client. - client = tasks_v2beta2.CloudTasksClient() - - # Construct the fully qualified queue name. - parent = client.queue_path(project, location, queue) - - lease_duration = {'seconds': 600} - - # Send lease request to client. - response = client.lease_tasks( - parent, lease_duration, max_tasks=1, response_view='FULL') - - task = response.tasks[0] - - print('Leased task {}'.format(task.name)) - return task - - -def acknowledge_task(task): - """Acknowledge a given task.""" - - from google.cloud import tasks_v2beta2 - - # Create a client. - client = tasks_v2beta2.CloudTasksClient() - - # Send request to client to acknowledge task. - client.acknowledge_task(task.name, task.schedule_time) - - print('Acknowledged task {}'.format(task.name)) - # [END cloud_tasks_lease_and_acknowledge_task] - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - - subparsers = parser.add_subparsers(dest='command') - - create_task_parser = subparsers.add_parser( - 'create-task', - help=create_task.__doc__) - create_task_parser.add_argument( - '--project', - help='Project ID.', - required=True, - ) - create_task_parser.add_argument( - '--queue', - help='Queue ID (short name).', - required=True, - ) - create_task_parser.add_argument( - '--location', - help='Location of the queue, e.g. \'us-central1\'.', - required=True, - ) - - lease_and_ack_parser = subparsers.add_parser( - 'lease-and-ack-task', - help=create_task.__doc__) - lease_and_ack_parser.add_argument( - '--project', - help='Project ID.', - required=True, - ) - lease_and_ack_parser.add_argument( - '--queue', - help='Queue ID (short name).', - required=True, - ) - lease_and_ack_parser.add_argument( - '--location', - help='Location of the queue, e.g. \'us-central1\'.', - required=True, - ) - - args = parser.parse_args() - - if args.command == 'create-task': - create_task(args.project, args.queue, args.location) - if args.command == 'lease-and-ack-task': - task = lease_task(args.project, args.queue, args.location) - acknowledge_task(task) diff --git a/samples/snippets/pull_queue_snippets_test.py b/samples/snippets/pull_queue_snippets_test.py deleted file mode 100644 index 69797f15..00000000 --- a/samples/snippets/pull_queue_snippets_test.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2018 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import pull_queue_snippets - -TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') -TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') -TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-pull-queue') - - -def test_create_task(): - result = pull_queue_snippets.create_task( - TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) - assert TEST_QUEUE_NAME in result.name - - -def test_lease_and_ack_task(): - pull_queue_snippets.create_task( - TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) - task = pull_queue_snippets.lease_task( - TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) - assert TEST_QUEUE_NAME in task.name - pull_queue_snippets.acknowledge_task(task) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt deleted file mode 100644 index e46eda26..00000000 --- a/samples/snippets/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -google-cloud-tasks==0.2.0 From adf92187097a33c4ffac6f30140bd253437940ed Mon Sep 17 00:00:00 2001 From: Averi Kitsch Date: Tue, 16 Apr 2019 20:48:40 -0700 Subject: [PATCH 28/51] [Cloud Tasks] Move samples to new folder [(#2114)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2114) * Move samples to keep consistent with other langauges * Ad system tests as well --- samples/snippets/README.md | 111 ++++++++++++++++++++ samples/snippets/create_http_task.py | 122 ++++++++++++++++++++++ samples/snippets/create_http_task_test.py | 28 +++++ samples/snippets/requirements.txt | 3 + 4 files changed, 264 insertions(+) create mode 100644 samples/snippets/README.md create mode 100644 samples/snippets/create_http_task.py create mode 100644 samples/snippets/create_http_task_test.py create mode 100644 samples/snippets/requirements.txt diff --git a/samples/snippets/README.md b/samples/snippets/README.md new file mode 100644 index 00000000..61ac422c --- /dev/null +++ b/samples/snippets/README.md @@ -0,0 +1,111 @@ +# Google Cloud Tasks Samples + +[![Open in Cloud Shell][shell_img]][shell_link] + +[shell_img]: http://gstatic.com/cloudssh/images/open-btn.png +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/flexible/tasks/README.md + +Sample command-line programs for interacting with the Cloud Tasks API +. + +App Engine queues push tasks to an App Engine HTTP target. This directory +contains both the App Engine app to deploy, as well as the snippets to run +locally to push tasks to it, which could also be called on App Engine. + +`create_app_engine_queue_task.py` is a simple command-line program to create +tasks to be pushed to the App Engine app. + +`main.py` is the main App Engine app. This app serves as an endpoint to receive +App Engine task attempts. + +`app.yaml` configures the App Engine app. + + +## Prerequisites to run locally: + +Please refer to [Setting Up a Python Development Environment](https://cloud.google.com/python/setup). + +## Authentication + +To set up authentication, please refer to our +[authentication getting started guide](https://cloud.google.com/docs/authentication/getting-started). + +## Creating a queue + +To create a queue using the Cloud SDK, use the following gcloud command: + +``` +gcloud beta tasks queues create-app-engine-queue my-appengine-queue +``` + +Note: A newly created queue will route to the default App Engine service and +version unless configured to do otherwise. + +## Deploying the App Engine App + +Deploy the App Engine app with gcloud: + +* To deploy to the Standard environment: + ``` + gcloud app deploy app.yaml + ``` +* To deploy to the Flexible environment: + ``` + gcloud app deploy app.flexible.yaml + ``` + +Verify the index page is serving: + +``` +gcloud app browse +``` + +The App Engine app serves as a target for the push requests. It has an +endpoint `/example_task_handler` that reads the payload (i.e., the request body) +of the HTTP POST request and logs it. The log output can be viewed with: + +``` +gcloud app logs read +``` + +## Run the Sample Using the Command Line + +Set environment variables: + +First, your project ID: + +``` +export PROJECT_ID=my-project-id +``` + +Then the queue ID, as specified at queue creation time. Queue IDs already +created can be listed with `gcloud beta tasks queues list`. + +``` +export QUEUE_ID=my-appengine-queue +``` + +And finally the location ID, which can be discovered with +`gcloud beta tasks queues describe $QUEUE_ID`, with the location embedded in +the "name" value (for instance, if the name is +"projects/my-project/locations/us-central1/queues/my-appengine-queue", then the +location is "us-central1"). + +``` +export LOCATION_ID=us-central1 +``` + +### Using HTTP Push Queues + +Set an environment variable for the endpoint to your task handler. This is an +example url to send requests to the App Engine task handler: +``` +export URL=https://.appspot.com/example_task_handler +``` + +Running the sample will create a task and send the task to the specific URL +endpoint, with a payload specified: + +``` +python create_http_task.py --project=$PROJECT_ID --queue=$QUEUE_ID --location=$LOCATION_ID --url=$URL --payload=hello +``` diff --git a/samples/snippets/create_http_task.py b/samples/snippets/create_http_task.py new file mode 100644 index 00000000..6ff3a6a2 --- /dev/null +++ b/samples/snippets/create_http_task.py @@ -0,0 +1,122 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import argparse +import datetime + + +def create_http_task(project, + queue, + location, + url, + payload=None, + in_seconds=None): + # [START cloud_tasks_create_http_task] + """Create a task for a given queue with an arbitrary payload.""" + + from google.cloud import tasks_v2beta3 + from google.protobuf import timestamp_pb2 + + # Create a client. + client = tasks_v2beta3.CloudTasksClient() + + # TODO(developer): Uncomment these lines and replace with your values. + # project = 'my-project-id' + # queue = 'my-appengine-queue' + # location = 'us-central1' + # url = 'https://.appspot.com/example_task_handler' + # payload = 'hello' + + # Construct the fully qualified queue name. + parent = client.queue_path(project, location, queue) + + # Construct the request body. + task = { + 'http_request': { # Specify the type of request. + 'http_method': 'POST', + 'url': url # The full url path that the task will be sent to. + } + } + if payload is not None: + # The API expects a payload of type bytes. + converted_payload = payload.encode() + + # Add the payload to the request. + task['http_request']['body'] = converted_payload + + if in_seconds is not None: + # Convert "seconds from now" into an rfc3339 datetime string. + d = datetime.datetime.utcnow() + datetime.timedelta(seconds=in_seconds) + + # Create Timestamp protobuf. + timestamp = timestamp_pb2.Timestamp() + timestamp.FromDatetime(d) + + # Add the timestamp to the tasks. + task['schedule_time'] = timestamp + + # Use the client to build and send the task. + response = client.create_task(parent, task) + + print('Created task {}'.format(response.name)) + return response +# [END cloud_tasks_create_http_task] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=create_http_task.__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + parser.add_argument( + '--project', + help='Project of the queue to add the task to.', + required=True, + ) + + parser.add_argument( + '--queue', + help='ID (short name) of the queue to add the task to.', + required=True, + ) + + parser.add_argument( + '--location', + help='Location of the queue to add the task to.', + required=True, + ) + + parser.add_argument( + '--url', + help='The full url path that the request will be sent to.', + required=True, + ) + + parser.add_argument( + '--payload', + help='Optional payload to attach to the push queue.' + ) + + parser.add_argument( + '--in_seconds', type=int, + help='The number of seconds from now to schedule task attempt.' + ) + + args = parser.parse_args() + + create_http_task( + args.project, args.queue, args.location, args.url, + args.payload, args.in_seconds) diff --git a/samples/snippets/create_http_task_test.py b/samples/snippets/create_http_task_test.py new file mode 100644 index 00000000..6acfeaac --- /dev/null +++ b/samples/snippets/create_http_task_test.py @@ -0,0 +1,28 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import create_http_task + +TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') +TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') +TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-appengine-queue') + + +def test_create_http_task(): + url = 'https://example.appspot.com/example_task_handler' + result = create_http_task.create_http_task( + TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION, url) + assert TEST_QUEUE_NAME in result.name diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt new file mode 100644 index 00000000..9aadd8a7 --- /dev/null +++ b/samples/snippets/requirements.txt @@ -0,0 +1,3 @@ +Flask==1.0.2 +gunicorn==19.9.0 +google-cloud-tasks==0.6.0 From 775acfe8f4d6218d9061f80cd62c252398f910fe Mon Sep 17 00:00:00 2001 From: Averi Kitsch Date: Thu, 18 Apr 2019 12:54:56 -0700 Subject: [PATCH 29/51] [Cloud Tasks] Add task with authentication sample [(#2113)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2113) * Add task with authentication sample * Fix linting * Fix linting * Fix spacing * Update tests with service account * Move samples and update READMEs * Update version and linting --- samples/snippets/README.md | 34 ++------ .../snippets/create_http_task_with_token.py | 80 +++++++++++++++++++ .../create_http_task_with_token_test.py | 33 ++++++++ samples/snippets/requirements.txt | 2 +- 4 files changed, 119 insertions(+), 30 deletions(-) create mode 100644 samples/snippets/create_http_task_with_token.py create mode 100644 samples/snippets/create_http_task_with_token_test.py diff --git a/samples/snippets/README.md b/samples/snippets/README.md index 61ac422c..00503ccd 100644 --- a/samples/snippets/README.md +++ b/samples/snippets/README.md @@ -12,8 +12,11 @@ App Engine queues push tasks to an App Engine HTTP target. This directory contains both the App Engine app to deploy, as well as the snippets to run locally to push tasks to it, which could also be called on App Engine. -`create_app_engine_queue_task.py` is a simple command-line program to create -tasks to be pushed to the App Engine app. +`create_http_task.py` is a simple command-line program to create +tasks to be pushed to an URL endpoint. + +`create_http_task_with_token.py` is a simple command-line program to create +tasks to be pushed to an URL endpoint with authorization header. `main.py` is the main App Engine app. This app serves as an endpoint to receive App Engine task attempts. @@ -41,33 +44,6 @@ gcloud beta tasks queues create-app-engine-queue my-appengine-queue Note: A newly created queue will route to the default App Engine service and version unless configured to do otherwise. -## Deploying the App Engine App - -Deploy the App Engine app with gcloud: - -* To deploy to the Standard environment: - ``` - gcloud app deploy app.yaml - ``` -* To deploy to the Flexible environment: - ``` - gcloud app deploy app.flexible.yaml - ``` - -Verify the index page is serving: - -``` -gcloud app browse -``` - -The App Engine app serves as a target for the push requests. It has an -endpoint `/example_task_handler` that reads the payload (i.e., the request body) -of the HTTP POST request and logs it. The log output can be viewed with: - -``` -gcloud app logs read -``` - ## Run the Sample Using the Command Line Set environment variables: diff --git a/samples/snippets/create_http_task_with_token.py b/samples/snippets/create_http_task_with_token.py new file mode 100644 index 00000000..1b79a9b3 --- /dev/null +++ b/samples/snippets/create_http_task_with_token.py @@ -0,0 +1,80 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import datetime + + +def create_http_task(project, + queue, + location, + url, + service_account_email, + payload=None, + in_seconds=None): + # [START cloud_tasks_create_http_task_with_token] + """Create a task for a given queue with an arbitrary payload.""" + + from google.cloud import tasks_v2beta3 + from google.protobuf import timestamp_pb2 + + # Create a client. + client = tasks_v2beta3.CloudTasksClient() + + # TODO(developer): Uncomment these lines and replace with your values. + # project = 'my-project-id' + # queue = 'my-appengine-queue' + # location = 'us-central1' + # url = 'https://example.com/example_task_handler' + # payload = 'hello' + + # Construct the fully qualified queue name. + parent = client.queue_path(project, location, queue) + + # Construct the request body. + task = { + 'http_request': { # Specify the type of request. + 'http_method': 'POST', + 'url': url, # The full url path that the task will be sent to. + 'oidc_token': { + 'service_account_email': service_account_email + } + } + } + + if payload is not None: + # The API expects a payload of type bytes. + converted_payload = payload.encode() + + # Add the payload to the request. + task['http_request']['body'] = converted_payload + + if in_seconds is not None: + # Convert "seconds from now" into an rfc3339 datetime string. + d = datetime.datetime.utcnow() + datetime.timedelta(seconds=in_seconds) + + # Create Timestamp protobuf. + timestamp = timestamp_pb2.Timestamp() + timestamp.FromDatetime(d) + + # Add the timestamp to the tasks. + task['schedule_time'] = timestamp + + # Use the client to build and send the task. + response = client.create_task(parent, task) + + print('Created task {}'.format(response.name)) + return response +# [END cloud_tasks_create_http_task_with_token] diff --git a/samples/snippets/create_http_task_with_token_test.py b/samples/snippets/create_http_task_with_token_test.py new file mode 100644 index 00000000..5b98c566 --- /dev/null +++ b/samples/snippets/create_http_task_with_token_test.py @@ -0,0 +1,33 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import create_http_task_with_token + +TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') +TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') +TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-appengine-queue') +TEST_SERVICE_ACCOUNT = ( + 'test-run-invoker@python-docs-samples-tests.iam.gserviceaccount.com') + + +def test_create_http_task_with_token(): + url = 'https://example.com/example_task_handler' + result = create_http_task_with_token.create_http_task(TEST_PROJECT_ID, + TEST_QUEUE_NAME, + TEST_LOCATION, + url, + TEST_SERVICE_ACCOUNT) + assert TEST_QUEUE_NAME in result.name diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 9aadd8a7..fe50a5aa 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,3 @@ Flask==1.0.2 gunicorn==19.9.0 -google-cloud-tasks==0.6.0 +google-cloud-tasks==0.7.0 From c20580eb097474bf5ab63e13260920cab60507fa Mon Sep 17 00:00:00 2001 From: Averi Kitsch Date: Wed, 15 May 2019 14:40:31 -0700 Subject: [PATCH 30/51] Update task sample comments [(#2156)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2156) * Update task comments * Update readme * Update queue name --- samples/snippets/README.md | 36 +++++++------------ samples/snippets/create_http_task.py | 4 +-- samples/snippets/create_http_task_test.py | 4 +-- .../snippets/create_http_task_with_token.py | 4 +-- .../create_http_task_with_token_test.py | 4 +-- 5 files changed, 20 insertions(+), 32 deletions(-) diff --git a/samples/snippets/README.md b/samples/snippets/README.md index 00503ccd..2c23e482 100644 --- a/samples/snippets/README.md +++ b/samples/snippets/README.md @@ -3,14 +3,10 @@ [![Open in Cloud Shell][shell_img]][shell_link] [shell_img]: http://gstatic.com/cloudssh/images/open-btn.png -[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=appengine/flexible/tasks/README.md +[shell_link]: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=tasks/README.md -Sample command-line programs for interacting with the Cloud Tasks API -. - -App Engine queues push tasks to an App Engine HTTP target. This directory -contains both the App Engine app to deploy, as well as the snippets to run -locally to push tasks to it, which could also be called on App Engine. +This sample demonstrates how to use the +[Cloud Tasks](https://cloud.google.com/tasks/docs/) client library. `create_http_task.py` is a simple command-line program to create tasks to be pushed to an URL endpoint. @@ -18,12 +14,6 @@ tasks to be pushed to an URL endpoint. `create_http_task_with_token.py` is a simple command-line program to create tasks to be pushed to an URL endpoint with authorization header. -`main.py` is the main App Engine app. This app serves as an endpoint to receive -App Engine task attempts. - -`app.yaml` configures the App Engine app. - - ## Prerequisites to run locally: Please refer to [Setting Up a Python Development Environment](https://cloud.google.com/python/setup). @@ -35,15 +25,13 @@ To set up authentication, please refer to our ## Creating a queue -To create a queue using the Cloud SDK, use the following gcloud command: +To create a queue (named `my-queue`) using the Cloud SDK, use the following +gcloud command: ``` -gcloud beta tasks queues create-app-engine-queue my-appengine-queue +gcloud beta tasks queues create my-queue ``` -Note: A newly created queue will route to the default App Engine service and -version unless configured to do otherwise. - ## Run the Sample Using the Command Line Set environment variables: @@ -58,25 +46,25 @@ Then the queue ID, as specified at queue creation time. Queue IDs already created can be listed with `gcloud beta tasks queues list`. ``` -export QUEUE_ID=my-appengine-queue +export QUEUE_ID=my-queue ``` And finally the location ID, which can be discovered with -`gcloud beta tasks queues describe $QUEUE_ID`, with the location embedded in +`gcloud beta tasks queues describe my-queue`, with the location embedded in the "name" value (for instance, if the name is -"projects/my-project/locations/us-central1/queues/my-appengine-queue", then the +"projects/my-project/locations/us-central1/queues/my-queue", then the location is "us-central1"). ``` export LOCATION_ID=us-central1 ``` -### Using HTTP Push Queues +### Creating Tasks with HTTP Targets Set an environment variable for the endpoint to your task handler. This is an -example url to send requests to the App Engine task handler: +example url: ``` -export URL=https://.appspot.com/example_task_handler +export URL=https://example.com/task_handler ``` Running the sample will create a task and send the task to the specific URL diff --git a/samples/snippets/create_http_task.py b/samples/snippets/create_http_task.py index 6ff3a6a2..eb1a446f 100644 --- a/samples/snippets/create_http_task.py +++ b/samples/snippets/create_http_task.py @@ -35,9 +35,9 @@ def create_http_task(project, # TODO(developer): Uncomment these lines and replace with your values. # project = 'my-project-id' - # queue = 'my-appengine-queue' + # queue = 'my-queue' # location = 'us-central1' - # url = 'https://.appspot.com/example_task_handler' + # url = 'https://example.com/task_handler' # payload = 'hello' # Construct the fully qualified queue name. diff --git a/samples/snippets/create_http_task_test.py b/samples/snippets/create_http_task_test.py index 6acfeaac..b1e50cc8 100644 --- a/samples/snippets/create_http_task_test.py +++ b/samples/snippets/create_http_task_test.py @@ -18,11 +18,11 @@ TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') -TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-appengine-queue') +TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-queue') def test_create_http_task(): - url = 'https://example.appspot.com/example_task_handler' + url = 'https://example.com/task_handler' result = create_http_task.create_http_task( TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION, url) assert TEST_QUEUE_NAME in result.name diff --git a/samples/snippets/create_http_task_with_token.py b/samples/snippets/create_http_task_with_token.py index 1b79a9b3..665d0714 100644 --- a/samples/snippets/create_http_task_with_token.py +++ b/samples/snippets/create_http_task_with_token.py @@ -35,9 +35,9 @@ def create_http_task(project, # TODO(developer): Uncomment these lines and replace with your values. # project = 'my-project-id' - # queue = 'my-appengine-queue' + # queue = 'my-queue' # location = 'us-central1' - # url = 'https://example.com/example_task_handler' + # url = 'https://example.com/task_handler' # payload = 'hello' # Construct the fully qualified queue name. diff --git a/samples/snippets/create_http_task_with_token_test.py b/samples/snippets/create_http_task_with_token_test.py index 5b98c566..d95ceca3 100644 --- a/samples/snippets/create_http_task_with_token_test.py +++ b/samples/snippets/create_http_task_with_token_test.py @@ -18,13 +18,13 @@ TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') -TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-appengine-queue') +TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-queue') TEST_SERVICE_ACCOUNT = ( 'test-run-invoker@python-docs-samples-tests.iam.gserviceaccount.com') def test_create_http_task_with_token(): - url = 'https://example.com/example_task_handler' + url = 'https://example.com/task_handler' result = create_http_task_with_token.create_http_task(TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION, From fe335473a94d5fd29d9ab6f0f0195fb127043621 Mon Sep 17 00:00:00 2001 From: Averi Kitsch Date: Tue, 11 Jun 2019 10:03:03 -0700 Subject: [PATCH 31/51] update gcloud [(#2208)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2208) --- samples/snippets/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/snippets/README.md b/samples/snippets/README.md index 2c23e482..2af8ecee 100644 --- a/samples/snippets/README.md +++ b/samples/snippets/README.md @@ -29,7 +29,7 @@ To create a queue (named `my-queue`) using the Cloud SDK, use the following gcloud command: ``` -gcloud beta tasks queues create my-queue +gcloud tasks queues create my-queue ``` ## Run the Sample Using the Command Line @@ -43,14 +43,14 @@ export PROJECT_ID=my-project-id ``` Then the queue ID, as specified at queue creation time. Queue IDs already -created can be listed with `gcloud beta tasks queues list`. +created can be listed with `gcloud tasks queues list`. ``` export QUEUE_ID=my-queue ``` And finally the location ID, which can be discovered with -`gcloud beta tasks queues describe my-queue`, with the location embedded in +`gcloud tasks queues describe my-queue`, with the location embedded in the "name" value (for instance, if the name is "projects/my-project/locations/us-central1/queues/my-queue", then the location is "us-central1"). From ca75b8f43d5300e4b7ec73623b63fb570db9b49b Mon Sep 17 00:00:00 2001 From: Averi Kitsch Date: Mon, 1 Jul 2019 09:39:08 -0700 Subject: [PATCH 32/51] Add protobuf dep and install instructions [(#2250)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2250) --- samples/snippets/README.md | 12 ++++++++++++ samples/snippets/requirements.txt | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/samples/snippets/README.md b/samples/snippets/README.md index 2af8ecee..b83eae65 100644 --- a/samples/snippets/README.md +++ b/samples/snippets/README.md @@ -23,6 +23,18 @@ Please refer to [Setting Up a Python Development Environment](https://cloud.goog To set up authentication, please refer to our [authentication getting started guide](https://cloud.google.com/docs/authentication/getting-started). +## Install Dependencies + +To install the dependencies for this sample, use the following command: + +``` +pip install -r requirements.txt +``` + +This sample uses the common protos in the [googleapis](https://github.com/googleapis/googleapis) +repository. For more info, see +[Protocol Buffer Basics](https://developers.google.com/protocol-buffers/docs/pythontutorial). + ## Creating a queue To create a queue (named `my-queue`) using the Cloud SDK, use the following diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index fe50a5aa..29d55c81 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,3 +1,4 @@ Flask==1.0.2 gunicorn==19.9.0 -google-cloud-tasks==0.7.0 +google-cloud-tasks==1.1.0 +googleapis-common-protos==1.6.0 From 74e4dd6ef3d6c85d2101f849c84858d2b72f72b3 Mon Sep 17 00:00:00 2001 From: Averi Kitsch Date: Tue, 6 Aug 2019 11:15:06 -0700 Subject: [PATCH 33/51] Add Migration Guide Snippets for Cloud Tasks [(#2316)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2316) * Migration guide * remove app and update migraitonn * snippets for migration guide - tests added * lint * remove print statements * Styling changes * Travis trigger --- samples/snippets/requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 29d55c81..7abf4e33 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,2 @@ -Flask==1.0.2 -gunicorn==19.9.0 google-cloud-tasks==1.1.0 googleapis-common-protos==1.6.0 From 23532a8799a3dab2de4a4299384e6efd46308d6c Mon Sep 17 00:00:00 2001 From: Alex Voorhees Date: Tue, 1 Oct 2019 17:47:31 -0500 Subject: [PATCH 34/51] Update create_http_task.py [(#2187)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2187) Updates `create_http_task.py` to have missing `in_seconds` variable --- samples/snippets/create_http_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/create_http_task.py b/samples/snippets/create_http_task.py index eb1a446f..517ab264 100644 --- a/samples/snippets/create_http_task.py +++ b/samples/snippets/create_http_task.py @@ -39,6 +39,7 @@ def create_http_task(project, # location = 'us-central1' # url = 'https://example.com/task_handler' # payload = 'hello' + in_seconds = 10 # Construct the fully qualified queue name. parent = client.queue_path(project, location, queue) @@ -56,7 +57,6 @@ def create_http_task(project, # Add the payload to the request. task['http_request']['body'] = converted_payload - if in_seconds is not None: # Convert "seconds from now" into an rfc3339 datetime string. d = datetime.datetime.utcnow() + datetime.timedelta(seconds=in_seconds) From 5604ee4937069b7c66108e0a5b6c353459cd8837 Mon Sep 17 00:00:00 2001 From: Gus Class Date: Mon, 7 Oct 2019 15:45:22 -0700 Subject: [PATCH 35/51] Adds updates for samples profiler ... vision [(#2439)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2439) --- samples/snippets/create_http_task.py | 2 +- samples/snippets/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/create_http_task.py b/samples/snippets/create_http_task.py index 517ab264..eb1a446f 100644 --- a/samples/snippets/create_http_task.py +++ b/samples/snippets/create_http_task.py @@ -39,7 +39,6 @@ def create_http_task(project, # location = 'us-central1' # url = 'https://example.com/task_handler' # payload = 'hello' - in_seconds = 10 # Construct the fully qualified queue name. parent = client.queue_path(project, location, queue) @@ -57,6 +56,7 @@ def create_http_task(project, # Add the payload to the request. task['http_request']['body'] = converted_payload + if in_seconds is not None: # Convert "seconds from now" into an rfc3339 datetime string. d = datetime.datetime.utcnow() + datetime.timedelta(seconds=in_seconds) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 7abf4e33..21fc2a7e 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,2 +1,2 @@ -google-cloud-tasks==1.1.0 +google-cloud-tasks==1.2.1 googleapis-common-protos==1.6.0 From 3a44474ef8bcc4746c562753b40c8f3d5d1f50bb Mon Sep 17 00:00:00 2001 From: Averi Kitsch Date: Wed, 6 Nov 2019 16:16:56 -0800 Subject: [PATCH 36/51] Update Cloud Tasks library version [(#2516)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2516) * Update to new library * update library version --- samples/snippets/create_http_task.py | 4 ++-- samples/snippets/create_http_task_with_token.py | 4 ++-- samples/snippets/requirements.txt | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/samples/snippets/create_http_task.py b/samples/snippets/create_http_task.py index eb1a446f..d86fc5b6 100644 --- a/samples/snippets/create_http_task.py +++ b/samples/snippets/create_http_task.py @@ -27,11 +27,11 @@ def create_http_task(project, # [START cloud_tasks_create_http_task] """Create a task for a given queue with an arbitrary payload.""" - from google.cloud import tasks_v2beta3 + from google.cloud import tasks_v2 from google.protobuf import timestamp_pb2 # Create a client. - client = tasks_v2beta3.CloudTasksClient() + client = tasks_v2.CloudTasksClient() # TODO(developer): Uncomment these lines and replace with your values. # project = 'my-project-id' diff --git a/samples/snippets/create_http_task_with_token.py b/samples/snippets/create_http_task_with_token.py index 665d0714..0c976913 100644 --- a/samples/snippets/create_http_task_with_token.py +++ b/samples/snippets/create_http_task_with_token.py @@ -27,11 +27,11 @@ def create_http_task(project, # [START cloud_tasks_create_http_task_with_token] """Create a task for a given queue with an arbitrary payload.""" - from google.cloud import tasks_v2beta3 + from google.cloud import tasks_v2 from google.protobuf import timestamp_pb2 # Create a client. - client = tasks_v2beta3.CloudTasksClient() + client = tasks_v2.CloudTasksClient() # TODO(developer): Uncomment these lines and replace with your values. # project = 'my-project-id' diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 21fc2a7e..a945842d 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,2 +1,2 @@ -google-cloud-tasks==1.2.1 +google-cloud-tasks==1.3.0 googleapis-common-protos==1.6.0 From 2e1c777d668e314ba755a108c9c142ec4fd80b5c Mon Sep 17 00:00:00 2001 From: Sarath Kaul Date: Fri, 27 Dec 2019 13:05:48 +0530 Subject: [PATCH 37/51] Adds Task name while creating Task [(#2543)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2543) --- samples/snippets/create_http_task.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/samples/snippets/create_http_task.py b/samples/snippets/create_http_task.py index d86fc5b6..2d86549a 100644 --- a/samples/snippets/create_http_task.py +++ b/samples/snippets/create_http_task.py @@ -23,7 +23,8 @@ def create_http_task(project, location, url, payload=None, - in_seconds=None): + in_seconds=None, + task_name=None): # [START cloud_tasks_create_http_task] """Create a task for a given queue with an arbitrary payload.""" @@ -68,6 +69,10 @@ def create_http_task(project, # Add the timestamp to the tasks. task['schedule_time'] = timestamp + if task_name is not None: + # Add the name to tasks. + task['name'] = task_name + # Use the client to build and send the task. response = client.create_task(parent, task) @@ -115,8 +120,12 @@ def create_http_task(project, help='The number of seconds from now to schedule task attempt.' ) + parser.add_argument( + '--task_name', + help='Task name of the task to create' + ) args = parser.parse_args() create_http_task( args.project, args.queue, args.location, args.url, - args.payload, args.in_seconds) + args.payload, args.in_seconds, args.task_name) From 0e1161ce24576277860668141f2a4f2f36b3e83a Mon Sep 17 00:00:00 2001 From: Sarath Kaul Date: Wed, 15 Jan 2020 02:12:59 +0530 Subject: [PATCH 38/51] Task Name in Creating HTTP Task with Token [(#2700)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2700) --- samples/snippets/create_http_task_with_token.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/samples/snippets/create_http_task_with_token.py b/samples/snippets/create_http_task_with_token.py index 0c976913..3205c18c 100644 --- a/samples/snippets/create_http_task_with_token.py +++ b/samples/snippets/create_http_task_with_token.py @@ -23,7 +23,8 @@ def create_http_task(project, url, service_account_email, payload=None, - in_seconds=None): + in_seconds=None, + task_name=None): # [START cloud_tasks_create_http_task_with_token] """Create a task for a given queue with an arbitrary payload.""" @@ -72,6 +73,10 @@ def create_http_task(project, # Add the timestamp to the tasks. task['schedule_time'] = timestamp + if task_name is not None: + # Add the name to tasks. + task['name'] = task_name + # Use the client to build and send the task. response = client.create_task(parent, task) From 770a699b9fe6211d9a0b8f16d9ac6fd71fbaa1ac Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 30 Mar 2020 20:57:51 +0200 Subject: [PATCH 39/51] chore(deps): update dependency google-cloud-tasks to v1.5.0 [(#3168)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3168) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index a945842d..aa50ce69 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,2 +1,2 @@ -google-cloud-tasks==1.3.0 +google-cloud-tasks==1.5.0 googleapis-common-protos==1.6.0 From ef6bd1d7e6088a22ef3e4e4a2a42bacf1930b614 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 31 Mar 2020 00:12:19 +0200 Subject: [PATCH 40/51] chore(deps): update dependency googleapis-common-protos to v1.51.0 [(#3171)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3171) This PR contains the following updates: | Package | Update | Change | |---|---|---| | [googleapis-common-protos](https://togithub.com/googleapis/googleapis) | minor | `==1.6.0` -> `==1.51.0` | --- ### Renovate configuration :date: **Schedule**: At any time (no schedule defined). :vertical_traffic_light: **Automerge**: Disabled by config. Please merge this manually once you are satisfied. :recycle: **Rebasing**: Never, or you tick the rebase/retry checkbox. :no_bell: **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#GoogleCloudPlatform/python-docs-samples). --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index aa50ce69..e5b2c431 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,2 +1,2 @@ google-cloud-tasks==1.5.0 -googleapis-common-protos==1.6.0 +googleapis-common-protos==1.51.0 From 9bb3c4e5db3f9a3350e0c61b8e90c019031033c1 Mon Sep 17 00:00:00 2001 From: Kurtis Van Gent <31518063+kurtisvg@users.noreply.github.com> Date: Wed, 1 Apr 2020 19:11:50 -0700 Subject: [PATCH 41/51] Simplify noxfile setup. [(#2806)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2806) * chore(deps): update dependency requests to v2.23.0 * Simplify noxfile and add version control. * Configure appengine/standard to only test Python 2.7. * Update Kokokro configs to match noxfile. * Add requirements-test to each folder. * Remove Py2 versions from everything execept appengine/standard. * Remove conftest.py. * Remove appengine/standard/conftest.py * Remove 'no-sucess-flaky-report' from pytest.ini. * Add GAE SDK back to appengine/standard tests. * Fix typo. * Roll pytest to python 2 version. * Add a bunch of testing requirements. * Remove typo. * Add appengine lib directory back in. * Add some additional requirements. * Fix issue with flake8 args. * Even more requirements. * Readd appengine conftest.py. * Add a few more requirements. * Even more Appengine requirements. * Add webtest for appengine/standard/mailgun. * Add some additional requirements. * Add workaround for issue with mailjet-rest. * Add responses for appengine/standard/mailjet. Co-authored-by: Renovate Bot --- samples/snippets/requirements-test.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 samples/snippets/requirements-test.txt diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt new file mode 100644 index 00000000..781d4326 --- /dev/null +++ b/samples/snippets/requirements-test.txt @@ -0,0 +1 @@ +pytest==5.3.2 From 02870070953ac49d34add86a474552a525695406 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 4 Jun 2020 20:59:54 +0200 Subject: [PATCH 42/51] Remove dependency googleapis-common-protos [(#3955)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3955) * Update dependency googleapis-common-protos to v1.52.0 * Update requirements.txt * Update requirements.txt * Update requirements.txt Co-authored-by: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> --- samples/snippets/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index e5b2c431..b7bc58a3 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,2 +1 @@ google-cloud-tasks==1.5.0 -googleapis-common-protos==1.51.0 From 50c345e66df01b708955e4628915a07423435fdf Mon Sep 17 00:00:00 2001 From: Kurtis Van Gent <31518063+kurtisvg@users.noreply.github.com> Date: Tue, 9 Jun 2020 14:34:27 -0700 Subject: [PATCH 43/51] Replace GCLOUD_PROJECT with GOOGLE_CLOUD_PROJECT. [(#4022)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4022) --- samples/snippets/create_http_task_test.py | 2 +- samples/snippets/create_http_task_with_token_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/create_http_task_test.py b/samples/snippets/create_http_task_test.py index b1e50cc8..da61f11b 100644 --- a/samples/snippets/create_http_task_test.py +++ b/samples/snippets/create_http_task_test.py @@ -16,7 +16,7 @@ import create_http_task -TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') +TEST_PROJECT_ID = os.getenv('GOOGLE_CLOUD_PROJECT') TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-queue') diff --git a/samples/snippets/create_http_task_with_token_test.py b/samples/snippets/create_http_task_with_token_test.py index d95ceca3..931cbd0a 100644 --- a/samples/snippets/create_http_task_with_token_test.py +++ b/samples/snippets/create_http_task_with_token_test.py @@ -16,7 +16,7 @@ import create_http_task_with_token -TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') +TEST_PROJECT_ID = os.getenv('GOOGLE_CLOUD_PROJECT') TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-queue') TEST_SERVICE_ACCOUNT = ( From d7067ebe0e56943e516fe5f67e22f678235ac1c4 Mon Sep 17 00:00:00 2001 From: Takashi Matsuo Date: Wed, 10 Jun 2020 12:45:41 -0700 Subject: [PATCH 44/51] [tasks] testing: use fixtures for the queue [(#4049)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4049) fixes #4045 fixes #4044 I don't know why these tests started to fail, but anyways we'd better use fixtures and temporary queues. --- samples/snippets/create_http_task_test.py | 24 +++++++++++++++++-- .../create_http_task_with_token_test.py | 24 +++++++++++++++++-- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/samples/snippets/create_http_task_test.py b/samples/snippets/create_http_task_test.py index da61f11b..b0fb3ed7 100644 --- a/samples/snippets/create_http_task_test.py +++ b/samples/snippets/create_http_task_test.py @@ -13,15 +13,35 @@ # limitations under the License. import os +import uuid + +from google.cloud import tasks_v2 +import pytest import create_http_task TEST_PROJECT_ID = os.getenv('GOOGLE_CLOUD_PROJECT') TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') -TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-queue') +TEST_QUEUE_NAME = f'my-queue-{uuid.uuid4().hex}' + + +@pytest.fixture() +def test_queue(): + client = tasks_v2.CloudTasksClient() + parent = client.location_path(TEST_PROJECT_ID, TEST_LOCATION) + queue = { + # The fully qualified path to the queue + 'name': client.queue_path( + TEST_PROJECT_ID, TEST_LOCATION, TEST_QUEUE_NAME), + } + q = client.create_queue(parent, queue) + + yield q + + client.delete_queue(q.name) -def test_create_http_task(): +def test_create_http_task(test_queue): url = 'https://example.com/task_handler' result = create_http_task.create_http_task( TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION, url) diff --git a/samples/snippets/create_http_task_with_token_test.py b/samples/snippets/create_http_task_with_token_test.py index 931cbd0a..dd90d919 100644 --- a/samples/snippets/create_http_task_with_token_test.py +++ b/samples/snippets/create_http_task_with_token_test.py @@ -13,17 +13,37 @@ # limitations under the License. import os +import uuid + +from google.cloud import tasks_v2 +import pytest import create_http_task_with_token TEST_PROJECT_ID = os.getenv('GOOGLE_CLOUD_PROJECT') TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') -TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-queue') +TEST_QUEUE_NAME = f'my-queue-{uuid.uuid4().hex}' TEST_SERVICE_ACCOUNT = ( 'test-run-invoker@python-docs-samples-tests.iam.gserviceaccount.com') -def test_create_http_task_with_token(): +@pytest.fixture() +def test_queue(): + client = tasks_v2.CloudTasksClient() + parent = client.location_path(TEST_PROJECT_ID, TEST_LOCATION) + queue = { + # The fully qualified path to the queue + 'name': client.queue_path( + TEST_PROJECT_ID, TEST_LOCATION, TEST_QUEUE_NAME), + } + q = client.create_queue(parent, queue) + + yield q + + client.delete_queue(q.name) + + +def test_create_http_task_with_token(test_queue): url = 'https://example.com/task_handler' result = create_http_task_with_token.create_http_task(TEST_PROJECT_ID, TEST_QUEUE_NAME, From 1c5088c9006f15e3e81e1c58239b3fabaa41efef Mon Sep 17 00:00:00 2001 From: Aaron Johnson Date: Fri, 12 Jun 2020 15:53:16 -0500 Subject: [PATCH 45/51] add python snippets and tests for creating, listing, and deleting queues [(#4012)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4012) * add python snippets and tests for creating, listing, and deleting queues * fix grammar * update licenses * apply suggested fixes and format with black * refine delete_queue_test with fixture for setup * utilize fixtures and match format of create_http_task_test * utilize fixtures in list_queues_test and create_queue_test * make create_queue_test call the right function * still attempt to delete queue after test runs in case of failure * attempt to delete queue in case of failure, using try/except approach * add print when NotFound is caught * fix import Co-authored-by: Averi Kitsch Co-authored-by: Takashi Matsuo --- samples/snippets/create_queue.py | 35 +++++++++++++++++ samples/snippets/create_queue_test.py | 40 +++++++++++++++++++ samples/snippets/delete_queue.py | 30 ++++++++++++++ samples/snippets/delete_queue_test.py | 56 +++++++++++++++++++++++++++ samples/snippets/list_queues.py | 36 +++++++++++++++++ samples/snippets/list_queues_test.py | 55 ++++++++++++++++++++++++++ 6 files changed, 252 insertions(+) create mode 100644 samples/snippets/create_queue.py create mode 100644 samples/snippets/create_queue_test.py create mode 100644 samples/snippets/delete_queue.py create mode 100644 samples/snippets/delete_queue_test.py create mode 100644 samples/snippets/list_queues.py create mode 100644 samples/snippets/list_queues_test.py diff --git a/samples/snippets/create_queue.py b/samples/snippets/create_queue.py new file mode 100644 index 00000000..d8d4dfae --- /dev/null +++ b/samples/snippets/create_queue.py @@ -0,0 +1,35 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START cloud_tasks_create_queue] +def create_queue(project, queue_name, location): + """Create a task queue.""" + + from google.cloud import tasks_v2 + + # Create a client. + client = tasks_v2.CloudTasksClient() + + # Construct the fully qualified location path. + parent = client.location_path(project, location) + + # Construct the create queue request. + queue = {'name': client.queue_path(project, location, queue_name)} + + # Use the client to create the queue. + response = client.create_queue(parent, queue) + + print('Created queue {}'.format(response.name)) + return response +# [END cloud_tasks_create_queue] diff --git a/samples/snippets/create_queue_test.py b/samples/snippets/create_queue_test.py new file mode 100644 index 00000000..1f623a2a --- /dev/null +++ b/samples/snippets/create_queue_test.py @@ -0,0 +1,40 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid + +from google.cloud import tasks_v2 +import pytest + +import create_queue + +TEST_PROJECT_ID = os.environ['GOOGLE_CLOUD_PROJECT'] +TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') +TEST_QUEUE_NAME = f'my-queue-{uuid.uuid4().hex}' + + +@pytest.fixture() +def test_queue(): + client = tasks_v2.CloudTasksClient() + q = create_queue.create_queue(TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) + + yield q + + client.delete_queue(q.name) + + +def test_create_queue(capsys, test_queue): + out, _ = capsys.readouterr() + assert 'Created queue' in out diff --git a/samples/snippets/delete_queue.py b/samples/snippets/delete_queue.py new file mode 100644 index 00000000..d6951424 --- /dev/null +++ b/samples/snippets/delete_queue.py @@ -0,0 +1,30 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START cloud_tasks_delete_queue] +def delete_queue(project, queue_name, location): + """Delete a task queue.""" + + from google.cloud import tasks_v2 + + # Create a client. + client = tasks_v2.CloudTasksClient() + + # Get the fully qualified path to queue. + queue = client.queue_path(project, location, queue_name) + + # Use the client to delete the queue. + client.delete_queue(queue) + print('Deleted queue') +# [END cloud_tasks_delete_queue] diff --git a/samples/snippets/delete_queue_test.py b/samples/snippets/delete_queue_test.py new file mode 100644 index 00000000..4e1acf40 --- /dev/null +++ b/samples/snippets/delete_queue_test.py @@ -0,0 +1,56 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid + +from google.api_core import exceptions +from google.cloud import tasks_v2 +import pytest + +import delete_queue + + +TEST_PROJECT_ID = os.environ['GOOGLE_CLOUD_PROJECT'] +TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') +TEST_QUEUE_NAME = f'my-queue-{uuid.uuid4().hex}' + + +@pytest.fixture() +def test_queue(): + client = tasks_v2.CloudTasksClient() + parent = client.location_path(TEST_PROJECT_ID, TEST_LOCATION) + queue = { + # The fully qualified path to the queue + 'name': client.queue_path( + TEST_PROJECT_ID, TEST_LOCATION, TEST_QUEUE_NAME), + } + q = client.create_queue(parent, queue) + + yield q + + try: + # Attempt to delete the queue in case the sample failed. + client.delete_queue(q.name) + except exceptions.NotFound: + # The queue was already successfully deleted. + print('Queue already deleted successfully') + + +def test_delete_queue(capsys, test_queue): + delete_queue.delete_queue( + TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION + ) + out, _ = capsys.readouterr() + assert 'Deleted queue' in out diff --git a/samples/snippets/list_queues.py b/samples/snippets/list_queues.py new file mode 100644 index 00000000..fa48907f --- /dev/null +++ b/samples/snippets/list_queues.py @@ -0,0 +1,36 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START cloud_tasks_list_queues] +def list_queues(project, location): + """List all task queues.""" + + from google.cloud import tasks_v2 + + # Create a client. + client = tasks_v2.CloudTasksClient() + + # Construct the fully qualified location path. + parent = client.location_path(project, location) + + # Use the client to obtain the queues. + response = client.list_queues(parent) + + # Print the results. + for queue in response: + print(queue.name) + + if response.num_results == 0: + print('No queues found!') +# [END cloud_tasks_list_queues] diff --git a/samples/snippets/list_queues_test.py b/samples/snippets/list_queues_test.py new file mode 100644 index 00000000..df11d0bd --- /dev/null +++ b/samples/snippets/list_queues_test.py @@ -0,0 +1,55 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid + +from google.cloud import tasks_v2 +import pytest + +import list_queues + +TEST_PROJECT_ID = os.environ['GOOGLE_CLOUD_PROJECT'] +TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') +TEST_QUEUE_NAME = f'my-queue-{uuid.uuid4().hex}' + + +@pytest.fixture() +def test_queue(): + client = tasks_v2.CloudTasksClient() + parent = client.location_path(TEST_PROJECT_ID, TEST_LOCATION) + queue = { + # The fully qualified path to the queue + 'name': client.queue_path( + TEST_PROJECT_ID, TEST_LOCATION, TEST_QUEUE_NAME), + } + q = client.create_queue(parent, queue) + + yield q + + client.delete_queue(q.name) + + +def test_list_queues_not_present(capsys): + list_queues.list_queues(TEST_PROJECT_ID, TEST_LOCATION) + out, _ = capsys.readouterr() + + assert(TEST_QUEUE_NAME not in out) + + +def test_list_queues_present(capsys, test_queue): + list_queues.list_queues(TEST_PROJECT_ID, TEST_LOCATION) + out, _ = capsys.readouterr() + + assert(TEST_QUEUE_NAME in out) From 936362db09f11b262b5ceec2003a08b983dd0e5e Mon Sep 17 00:00:00 2001 From: Adam Ross Date: Fri, 26 Jun 2020 12:04:28 -0700 Subject: [PATCH 46/51] docs(tasks): service_account_email parameter example [(#4183)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4183) * docs(tasks): serviceAccountEmail parameter example * docs(tasks): parameter name camel => snake case * docs(tasks): comment order = function param order --- samples/snippets/create_http_task_with_token.py | 1 + 1 file changed, 1 insertion(+) diff --git a/samples/snippets/create_http_task_with_token.py b/samples/snippets/create_http_task_with_token.py index 3205c18c..7320ede3 100644 --- a/samples/snippets/create_http_task_with_token.py +++ b/samples/snippets/create_http_task_with_token.py @@ -39,6 +39,7 @@ def create_http_task(project, # queue = 'my-queue' # location = 'us-central1' # url = 'https://example.com/task_handler' + # service_account_email = 'service-account@my-project-id.iam.gserviceaccount.com'; # payload = 'hello' # Construct the fully qualified queue name. From 4089865495098c5545ae28303b6df4d4d5cb25dc Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 13 Jul 2020 00:46:30 +0200 Subject: [PATCH 47/51] chore(deps): update dependency pytest to v5.4.3 [(#4279)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4279) * chore(deps): update dependency pytest to v5.4.3 * specify pytest for python 2 in appengine Co-authored-by: Leah Cole --- samples/snippets/requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index 781d4326..79738af5 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1 +1 @@ -pytest==5.3.2 +pytest==5.4.3 From 05ce0c5e6ed9fae19edb451ebfee5af9578ac029 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 1 Aug 2020 21:51:00 +0200 Subject: [PATCH 48/51] Update dependency pytest to v6 [(#4390)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4390) --- samples/snippets/requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index 79738af5..7e460c8c 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1 +1 @@ -pytest==5.4.3 +pytest==6.0.1 From bc211413fca758dd6ab39370f042939b00585815 Mon Sep 17 00:00:00 2001 From: "Joab Leite S. Neto" Date: Tue, 11 Aug 2020 17:06:04 -0300 Subject: [PATCH 49/51] tasks: added json content-type request [(#4473)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4473) - added json payload compatibility - fix imports and code block used on https://cloud.google.com/tasks/docs/creating-http-target-tasks#python ## Description Fixes # Note: It's a good idea to open an issue first for discussion. ## Checklist - [ ] I have followed [Sample Guidelines from AUTHORING_GUIDE.MD](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/AUTHORING_GUIDE.md) - [ ] README is updated to include [all relevant information](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/AUTHORING_GUIDE.md#readme-file) - [ ] **Tests** pass: `nox -s py-3.6` (see [Test Environment Setup](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/AUTHORING_GUIDE.md#test-environment-setup)) - [ ] **Lint** pass: `nox -s lint` (see [Test Environment Setup](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/AUTHORING_GUIDE.md#test-environment-setup)) - [ ] These samples need a new **API enabled** in testing projects to pass (let us know which ones) - [ ] These samples need a new/updated **env vars** in testing projects set to pass (let us know which ones) - [ ] Please **merge** this PR for me once it is approved. - [ ] This sample adds a new sample directory, and I updated the [CODEOWNERS file](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/.github/CODEOWNERS) with the codeowners for this sample --- samples/snippets/create_http_task.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/samples/snippets/create_http_task.py b/samples/snippets/create_http_task.py index 2d86549a..23896209 100644 --- a/samples/snippets/create_http_task.py +++ b/samples/snippets/create_http_task.py @@ -15,7 +15,6 @@ from __future__ import print_function import argparse -import datetime def create_http_task(project, @@ -30,6 +29,8 @@ def create_http_task(project, from google.cloud import tasks_v2 from google.protobuf import timestamp_pb2 + import datetime + import json # Create a client. client = tasks_v2.CloudTasksClient() @@ -39,7 +40,7 @@ def create_http_task(project, # queue = 'my-queue' # location = 'us-central1' # url = 'https://example.com/task_handler' - # payload = 'hello' + # payload = 'hello' or {'param': 'value'} for application/json # Construct the fully qualified queue name. parent = client.queue_path(project, location, queue) @@ -52,6 +53,12 @@ def create_http_task(project, } } if payload is not None: + if isinstance(payload, dict): + # Convert dict to JSON string + payload = json.dumps(payload) + # specify http content-type to application/json + task['http_request']['headers'] = {'Content-type': 'application/json'} + # The API expects a payload of type bytes. converted_payload = payload.encode() @@ -77,8 +84,8 @@ def create_http_task(project, response = client.create_task(parent, task) print('Created task {}'.format(response.name)) + # [END cloud_tasks_create_http_task] return response -# [END cloud_tasks_create_http_task] if __name__ == '__main__': From bb3fdb26ff34eaff9e9aa869687de5a7ce5a43a2 Mon Sep 17 00:00:00 2001 From: arithmetic1728 Date: Mon, 31 Aug 2020 13:44:41 -0700 Subject: [PATCH 50/51] docs: add samples from python-docs-samples/tasks --- synth.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/synth.py b/synth.py index 74cce6d9..a9c4d92f 100644 --- a/synth.py +++ b/synth.py @@ -16,6 +16,7 @@ import synthtool as s import synthtool.gcp as gcp +from synthtool.languages import python import logging logging.basicConfig(level=logging.DEBUG) @@ -115,9 +116,14 @@ # ---------------------------------------------------------------------------- # Add templated files # ---------------------------------------------------------------------------- -templated_files = common.py_library(cov_level=86) +templated_files = common.py_library(cov_level=86, samples=True) s.move(templated_files) +# ---------------------------------------------------------------------------- +# Samples templates +# ---------------------------------------------------------------------------- +python.py_samples() + # TODO(busunkim): Use latest sphinx after microgenerator transition s.replace("noxfile.py", """['"]sphinx['"]""", '"sphinx<3.0.0"') From 64baf4b8edf26253716ead237c5af88599062c88 Mon Sep 17 00:00:00 2001 From: arithmetic1728 Date: Mon, 31 Aug 2020 13:52:35 -0700 Subject: [PATCH 51/51] chore: update templates --- .github/CODEOWNERS | 8 + .gitignore | 3 +- .kokoro/build.sh | 8 +- .kokoro/docker/docs/Dockerfile | 98 ++++++ .kokoro/docker/docs/fetch_gpg_keys.sh | 45 +++ .kokoro/docs/common.cfg | 21 +- .kokoro/docs/docs-presubmit.cfg | 17 + .kokoro/publish-docs.sh | 39 ++- .kokoro/trampoline_v2.sh | 487 ++++++++++++++++++++++++++ .trampolinerc | 51 +++ docs/conf.py | 11 +- noxfile.py | 37 ++ samples/AUTHORING_GUIDE.md | 1 + samples/CONTRIBUTING.md | 1 + samples/snippets/noxfile.py | 224 ++++++++++++ synth.metadata | 11 +- synth.py | 2 +- 17 files changed, 1037 insertions(+), 27 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .kokoro/docker/docs/Dockerfile create mode 100755 .kokoro/docker/docs/fetch_gpg_keys.sh create mode 100644 .kokoro/docs/docs-presubmit.cfg create mode 100755 .kokoro/trampoline_v2.sh create mode 100644 .trampolinerc create mode 100644 samples/AUTHORING_GUIDE.md create mode 100644 samples/CONTRIBUTING.md create mode 100644 samples/snippets/noxfile.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..fda1e837 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,8 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. +# +# For syntax help see: +# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax + + +/samples/**/*.py @averikitsch @googleapis/python-samples-owners \ No newline at end of file diff --git a/.gitignore b/.gitignore index b87e1ed5..b9daa52f 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,7 @@ pip-log.txt # Built documentation docs/_build bigquery/docs/generated +docs.metadata # Virtual environment env/ @@ -57,4 +58,4 @@ system_tests/local_test_setup # Make sure a generated file isn't accidentally committed. pylintrc -pylintrc.test \ No newline at end of file +pylintrc.test diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 2c8fcdb8..84a9a0ea 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -36,4 +36,10 @@ python3.6 -m pip uninstall --yes --quiet nox-automation python3.6 -m pip install --upgrade --quiet nox python3.6 -m nox --version -python3.6 -m nox +# If NOX_SESSION is set, it only runs the specified session, +# otherwise run all the sessions. +if [[ -n "${NOX_SESSION:-}" ]]; then + python3.6 -m nox -s "${NOX_SESSION:-}" +else + python3.6 -m nox +fi diff --git a/.kokoro/docker/docs/Dockerfile b/.kokoro/docker/docs/Dockerfile new file mode 100644 index 00000000..412b0b56 --- /dev/null +++ b/.kokoro/docker/docs/Dockerfile @@ -0,0 +1,98 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ubuntu:20.04 + +ENV DEBIAN_FRONTEND noninteractive + +# Ensure local Python is preferred over distribution Python. +ENV PATH /usr/local/bin:$PATH + +# Install dependencies. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + apt-transport-https \ + build-essential \ + ca-certificates \ + curl \ + dirmngr \ + git \ + gpg-agent \ + graphviz \ + libbz2-dev \ + libdb5.3-dev \ + libexpat1-dev \ + libffi-dev \ + liblzma-dev \ + libreadline-dev \ + libsnappy-dev \ + libssl-dev \ + libsqlite3-dev \ + portaudio19-dev \ + redis-server \ + software-properties-common \ + ssh \ + sudo \ + tcl \ + tcl-dev \ + tk \ + tk-dev \ + uuid-dev \ + wget \ + zlib1g-dev \ + && add-apt-repository universe \ + && apt-get update \ + && apt-get -y install jq \ + && apt-get clean autoclean \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* \ + && rm -f /var/cache/apt/archives/*.deb + + +COPY fetch_gpg_keys.sh /tmp +# Install the desired versions of Python. +RUN set -ex \ + && export GNUPGHOME="$(mktemp -d)" \ + && echo "disable-ipv6" >> "${GNUPGHOME}/dirmngr.conf" \ + && /tmp/fetch_gpg_keys.sh \ + && for PYTHON_VERSION in 3.7.8 3.8.5; do \ + wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \ + && wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \ + && gpg --batch --verify python-${PYTHON_VERSION}.tar.xz.asc python-${PYTHON_VERSION}.tar.xz \ + && rm -r python-${PYTHON_VERSION}.tar.xz.asc \ + && mkdir -p /usr/src/python-${PYTHON_VERSION} \ + && tar -xJC /usr/src/python-${PYTHON_VERSION} --strip-components=1 -f python-${PYTHON_VERSION}.tar.xz \ + && rm python-${PYTHON_VERSION}.tar.xz \ + && cd /usr/src/python-${PYTHON_VERSION} \ + && ./configure \ + --enable-shared \ + # This works only on Python 2.7 and throws a warning on every other + # version, but seems otherwise harmless. + --enable-unicode=ucs4 \ + --with-system-ffi \ + --without-ensurepip \ + && make -j$(nproc) \ + && make install \ + && ldconfig \ + ; done \ + && rm -rf "${GNUPGHOME}" \ + && rm -rf /usr/src/python* \ + && rm -rf ~/.cache/ + +RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ + && python3.7 /tmp/get-pip.py \ + && python3.8 /tmp/get-pip.py \ + && rm /tmp/get-pip.py + +CMD ["python3.7"] diff --git a/.kokoro/docker/docs/fetch_gpg_keys.sh b/.kokoro/docker/docs/fetch_gpg_keys.sh new file mode 100755 index 00000000..d653dd86 --- /dev/null +++ b/.kokoro/docker/docs/fetch_gpg_keys.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A script to fetch gpg keys with retry. +# Avoid jinja parsing the file. +# + +function retry { + if [[ "${#}" -le 1 ]]; then + echo "Usage: ${0} retry_count commands.." + exit 1 + fi + local retries=${1} + local command="${@:2}" + until [[ "${retries}" -le 0 ]]; do + $command && return 0 + if [[ $? -ne 0 ]]; then + echo "command failed, retrying" + ((retries--)) + fi + done + return 1 +} + +# 3.6.9, 3.7.5 (Ned Deily) +retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \ + 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D + +# 3.8.0 (Łukasz Langa) +retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \ + E3FF2839C048B25C084DEBE9B26995E310250568 + +# diff --git a/.kokoro/docs/common.cfg b/.kokoro/docs/common.cfg index a4c3dee9..832ce9a2 100644 --- a/.kokoro/docs/common.cfg +++ b/.kokoro/docs/common.cfg @@ -11,12 +11,12 @@ action { gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "python-tasks/.kokoro/trampoline.sh" +build_file: "python-tasks/.kokoro/trampoline_v2.sh" # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" + value: "gcr.io/cloud-devrel-kokoro-resources/python-lib-docs" } env_vars: { key: "TRAMPOLINE_BUILD_FILE" @@ -28,6 +28,23 @@ env_vars: { value: "docs-staging" } +env_vars: { + key: "V2_STAGING_BUCKET" + value: "docs-staging-v2-staging" +} + +# It will upload the docker image after successful builds. +env_vars: { + key: "TRAMPOLINE_IMAGE_UPLOAD" + value: "true" +} + +# It will always build the docker image. +env_vars: { + key: "TRAMPOLINE_DOCKERFILE" + value: ".kokoro/docker/docs/Dockerfile" +} + # Fetch the token needed for reporting release status to GitHub before_action { fetch_keystore { diff --git a/.kokoro/docs/docs-presubmit.cfg b/.kokoro/docs/docs-presubmit.cfg new file mode 100644 index 00000000..11181078 --- /dev/null +++ b/.kokoro/docs/docs-presubmit.cfg @@ -0,0 +1,17 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "STAGING_BUCKET" + value: "gcloud-python-test" +} + +env_vars: { + key: "V2_STAGING_BUCKET" + value: "gcloud-python-test" +} + +# We only upload the image in the main `docs` build. +env_vars: { + key: "TRAMPOLINE_IMAGE_UPLOAD" + value: "false" +} diff --git a/.kokoro/publish-docs.sh b/.kokoro/publish-docs.sh index 81c6e278..8acb14e8 100755 --- a/.kokoro/publish-docs.sh +++ b/.kokoro/publish-docs.sh @@ -18,26 +18,16 @@ set -eo pipefail # Disable buffering, so that the logs stream through. export PYTHONUNBUFFERED=1 -cd github/python-tasks - -# Remove old nox -python3.6 -m pip uninstall --yes --quiet nox-automation +export PATH="${HOME}/.local/bin:${PATH}" # Install nox -python3.6 -m pip install --upgrade --quiet nox -python3.6 -m nox --version +python3 -m pip install --user --upgrade --quiet nox +python3 -m nox --version # build docs nox -s docs -python3 -m pip install gcp-docuploader - -# install a json parser -sudo apt-get update -sudo apt-get -y install software-properties-common -sudo add-apt-repository universe -sudo apt-get update -sudo apt-get -y install jq +python3 -m pip install --user gcp-docuploader # create metadata python3 -m docuploader create-metadata \ @@ -52,4 +42,23 @@ python3 -m docuploader create-metadata \ cat docs.metadata # upload docs -python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket docs-staging +python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket "${STAGING_BUCKET}" + + +# docfx yaml files +nox -s docfx + +# create metadata. +python3 -m docuploader create-metadata \ + --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \ + --version=$(python3 setup.py --version) \ + --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \ + --distribution-name=$(python3 setup.py --name) \ + --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \ + --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \ + --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json) + +cat docs.metadata + +# upload docs +python3 -m docuploader upload docs/_build/html/docfx_yaml --metadata-file docs.metadata --destination-prefix docfx --staging-bucket "${V2_STAGING_BUCKET}" diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh new file mode 100755 index 00000000..719bcd5b --- /dev/null +++ b/.kokoro/trampoline_v2.sh @@ -0,0 +1,487 @@ +#!/usr/bin/env bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# trampoline_v2.sh +# +# This script does 3 things. +# +# 1. Prepare the Docker image for the test +# 2. Run the Docker with appropriate flags to run the test +# 3. Upload the newly built Docker image +# +# in a way that is somewhat compatible with trampoline_v1. +# +# To run this script, first download few files from gcs to /dev/shm. +# (/dev/shm is passed into the container as KOKORO_GFILE_DIR). +# +# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/secrets_viewer_service_account.json /dev/shm +# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/automl_secrets.txt /dev/shm +# +# Then run the script. +# .kokoro/trampoline_v2.sh +# +# These environment variables are required: +# TRAMPOLINE_IMAGE: The docker image to use. +# TRAMPOLINE_DOCKERFILE: The location of the Dockerfile. +# +# You can optionally change these environment variables: +# TRAMPOLINE_IMAGE_UPLOAD: +# (true|false): Whether to upload the Docker image after the +# successful builds. +# TRAMPOLINE_BUILD_FILE: The script to run in the docker container. +# TRAMPOLINE_WORKSPACE: The workspace path in the docker container. +# Defaults to /workspace. +# Potentially there are some repo specific envvars in .trampolinerc in +# the project root. + + +set -euo pipefail + +TRAMPOLINE_VERSION="2.0.5" + +if command -v tput >/dev/null && [[ -n "${TERM:-}" ]]; then + readonly IO_COLOR_RED="$(tput setaf 1)" + readonly IO_COLOR_GREEN="$(tput setaf 2)" + readonly IO_COLOR_YELLOW="$(tput setaf 3)" + readonly IO_COLOR_RESET="$(tput sgr0)" +else + readonly IO_COLOR_RED="" + readonly IO_COLOR_GREEN="" + readonly IO_COLOR_YELLOW="" + readonly IO_COLOR_RESET="" +fi + +function function_exists { + [ $(LC_ALL=C type -t $1)"" == "function" ] +} + +# Logs a message using the given color. The first argument must be one +# of the IO_COLOR_* variables defined above, such as +# "${IO_COLOR_YELLOW}". The remaining arguments will be logged in the +# given color. The log message will also have an RFC-3339 timestamp +# prepended (in UTC). You can disable the color output by setting +# TERM=vt100. +function log_impl() { + local color="$1" + shift + local timestamp="$(date -u "+%Y-%m-%dT%H:%M:%SZ")" + echo "================================================================" + echo "${color}${timestamp}:" "$@" "${IO_COLOR_RESET}" + echo "================================================================" +} + +# Logs the given message with normal coloring and a timestamp. +function log() { + log_impl "${IO_COLOR_RESET}" "$@" +} + +# Logs the given message in green with a timestamp. +function log_green() { + log_impl "${IO_COLOR_GREEN}" "$@" +} + +# Logs the given message in yellow with a timestamp. +function log_yellow() { + log_impl "${IO_COLOR_YELLOW}" "$@" +} + +# Logs the given message in red with a timestamp. +function log_red() { + log_impl "${IO_COLOR_RED}" "$@" +} + +readonly tmpdir=$(mktemp -d -t ci-XXXXXXXX) +readonly tmphome="${tmpdir}/h" +mkdir -p "${tmphome}" + +function cleanup() { + rm -rf "${tmpdir}" +} +trap cleanup EXIT + +RUNNING_IN_CI="${RUNNING_IN_CI:-false}" + +# The workspace in the container, defaults to /workspace. +TRAMPOLINE_WORKSPACE="${TRAMPOLINE_WORKSPACE:-/workspace}" + +pass_down_envvars=( + # TRAMPOLINE_V2 variables. + # Tells scripts whether they are running as part of CI or not. + "RUNNING_IN_CI" + # Indicates which CI system we're in. + "TRAMPOLINE_CI" + # Indicates the version of the script. + "TRAMPOLINE_VERSION" +) + +log_yellow "Building with Trampoline ${TRAMPOLINE_VERSION}" + +# Detect which CI systems we're in. If we're in any of the CI systems +# we support, `RUNNING_IN_CI` will be true and `TRAMPOLINE_CI` will be +# the name of the CI system. Both envvars will be passing down to the +# container for telling which CI system we're in. +if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then + # descriptive env var for indicating it's on CI. + RUNNING_IN_CI="true" + TRAMPOLINE_CI="kokoro" + if [[ "${TRAMPOLINE_USE_LEGACY_SERVICE_ACCOUNT:-}" == "true" ]]; then + if [[ ! -f "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" ]]; then + log_red "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json does not exist. Did you forget to mount cloud-devrel-kokoro-resources/trampoline? Aborting." + exit 1 + fi + # This service account will be activated later. + TRAMPOLINE_SERVICE_ACCOUNT="${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" + else + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + gcloud auth list + fi + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet + fi + pass_down_envvars+=( + # KOKORO dynamic variables. + "KOKORO_BUILD_NUMBER" + "KOKORO_BUILD_ID" + "KOKORO_JOB_NAME" + "KOKORO_GIT_COMMIT" + "KOKORO_GITHUB_COMMIT" + "KOKORO_GITHUB_PULL_REQUEST_NUMBER" + "KOKORO_GITHUB_PULL_REQUEST_COMMIT" + # For Build Cop Bot + "KOKORO_GITHUB_COMMIT_URL" + "KOKORO_GITHUB_PULL_REQUEST_URL" + ) +elif [[ "${TRAVIS:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="travis" + pass_down_envvars+=( + "TRAVIS_BRANCH" + "TRAVIS_BUILD_ID" + "TRAVIS_BUILD_NUMBER" + "TRAVIS_BUILD_WEB_URL" + "TRAVIS_COMMIT" + "TRAVIS_COMMIT_MESSAGE" + "TRAVIS_COMMIT_RANGE" + "TRAVIS_JOB_NAME" + "TRAVIS_JOB_NUMBER" + "TRAVIS_JOB_WEB_URL" + "TRAVIS_PULL_REQUEST" + "TRAVIS_PULL_REQUEST_BRANCH" + "TRAVIS_PULL_REQUEST_SHA" + "TRAVIS_PULL_REQUEST_SLUG" + "TRAVIS_REPO_SLUG" + "TRAVIS_SECURE_ENV_VARS" + "TRAVIS_TAG" + ) +elif [[ -n "${GITHUB_RUN_ID:-}" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="github-workflow" + pass_down_envvars+=( + "GITHUB_WORKFLOW" + "GITHUB_RUN_ID" + "GITHUB_RUN_NUMBER" + "GITHUB_ACTION" + "GITHUB_ACTIONS" + "GITHUB_ACTOR" + "GITHUB_REPOSITORY" + "GITHUB_EVENT_NAME" + "GITHUB_EVENT_PATH" + "GITHUB_SHA" + "GITHUB_REF" + "GITHUB_HEAD_REF" + "GITHUB_BASE_REF" + ) +elif [[ "${CIRCLECI:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="circleci" + pass_down_envvars+=( + "CIRCLE_BRANCH" + "CIRCLE_BUILD_NUM" + "CIRCLE_BUILD_URL" + "CIRCLE_COMPARE_URL" + "CIRCLE_JOB" + "CIRCLE_NODE_INDEX" + "CIRCLE_NODE_TOTAL" + "CIRCLE_PREVIOUS_BUILD_NUM" + "CIRCLE_PROJECT_REPONAME" + "CIRCLE_PROJECT_USERNAME" + "CIRCLE_REPOSITORY_URL" + "CIRCLE_SHA1" + "CIRCLE_STAGE" + "CIRCLE_USERNAME" + "CIRCLE_WORKFLOW_ID" + "CIRCLE_WORKFLOW_JOB_ID" + "CIRCLE_WORKFLOW_UPSTREAM_JOB_IDS" + "CIRCLE_WORKFLOW_WORKSPACE_ID" + ) +fi + +# Configure the service account for pulling the docker image. +function repo_root() { + local dir="$1" + while [[ ! -d "${dir}/.git" ]]; do + dir="$(dirname "$dir")" + done + echo "${dir}" +} + +# Detect the project root. In CI builds, we assume the script is in +# the git tree and traverse from there, otherwise, traverse from `pwd` +# to find `.git` directory. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + PROGRAM_PATH="$(realpath "$0")" + PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")" + PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")" +else + PROJECT_ROOT="$(repo_root $(pwd))" +fi + +log_yellow "Changing to the project root: ${PROJECT_ROOT}." +cd "${PROJECT_ROOT}" + +# To support relative path for `TRAMPOLINE_SERVICE_ACCOUNT`, we need +# to use this environment variable in `PROJECT_ROOT`. +if [[ -n "${TRAMPOLINE_SERVICE_ACCOUNT:-}" ]]; then + + mkdir -p "${tmpdir}/gcloud" + gcloud_config_dir="${tmpdir}/gcloud" + + log_yellow "Using isolated gcloud config: ${gcloud_config_dir}." + export CLOUDSDK_CONFIG="${gcloud_config_dir}" + + log_yellow "Using ${TRAMPOLINE_SERVICE_ACCOUNT} for authentication." + gcloud auth activate-service-account \ + --key-file "${TRAMPOLINE_SERVICE_ACCOUNT}" + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet +fi + +required_envvars=( + # The basic trampoline configurations. + "TRAMPOLINE_IMAGE" + "TRAMPOLINE_BUILD_FILE" +) + +if [[ -f "${PROJECT_ROOT}/.trampolinerc" ]]; then + source "${PROJECT_ROOT}/.trampolinerc" +fi + +log_yellow "Checking environment variables." +for e in "${required_envvars[@]}" +do + if [[ -z "${!e:-}" ]]; then + log "Missing ${e} env var. Aborting." + exit 1 + fi +done + +# We want to support legacy style TRAMPOLINE_BUILD_FILE used with V1 +# script: e.g. "github/repo-name/.kokoro/run_tests.sh" +TRAMPOLINE_BUILD_FILE="${TRAMPOLINE_BUILD_FILE#github/*/}" +log_yellow "Using TRAMPOLINE_BUILD_FILE: ${TRAMPOLINE_BUILD_FILE}" + +# ignore error on docker operations and test execution +set +e + +log_yellow "Preparing Docker image." +# We only download the docker image in CI builds. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + # Download the docker image specified by `TRAMPOLINE_IMAGE` + + # We may want to add --max-concurrent-downloads flag. + + log_yellow "Start pulling the Docker image: ${TRAMPOLINE_IMAGE}." + if docker pull "${TRAMPOLINE_IMAGE}"; then + log_green "Finished pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="true" + else + log_red "Failed pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="false" + fi +else + # For local run, check if we have the image. + if docker images "${TRAMPOLINE_IMAGE}:latest" | grep "${TRAMPOLINE_IMAGE}"; then + has_image="true" + else + has_image="false" + fi +fi + + +# The default user for a Docker container has uid 0 (root). To avoid +# creating root-owned files in the build directory we tell docker to +# use the current user ID. +user_uid="$(id -u)" +user_gid="$(id -g)" +user_name="$(id -un)" + +# To allow docker in docker, we add the user to the docker group in +# the host os. +docker_gid=$(cut -d: -f3 < <(getent group docker)) + +update_cache="false" +if [[ "${TRAMPOLINE_DOCKERFILE:-none}" != "none" ]]; then + # Build the Docker image from the source. + context_dir=$(dirname "${TRAMPOLINE_DOCKERFILE}") + docker_build_flags=( + "-f" "${TRAMPOLINE_DOCKERFILE}" + "-t" "${TRAMPOLINE_IMAGE}" + "--build-arg" "UID=${user_uid}" + "--build-arg" "USERNAME=${user_name}" + ) + if [[ "${has_image}" == "true" ]]; then + docker_build_flags+=("--cache-from" "${TRAMPOLINE_IMAGE}") + fi + + log_yellow "Start building the docker image." + if [[ "${TRAMPOLINE_VERBOSE:-false}" == "true" ]]; then + echo "docker build" "${docker_build_flags[@]}" "${context_dir}" + fi + + # ON CI systems, we want to suppress docker build logs, only + # output the logs when it fails. + if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + if docker build "${docker_build_flags[@]}" "${context_dir}" \ + > "${tmpdir}/docker_build.log" 2>&1; then + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + cat "${tmpdir}/docker_build.log" + fi + + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + log_yellow "Dumping the build logs:" + cat "${tmpdir}/docker_build.log" + exit 1 + fi + else + if docker build "${docker_build_flags[@]}" "${context_dir}"; then + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + exit 1 + fi + fi +else + if [[ "${has_image}" != "true" ]]; then + log_red "We do not have ${TRAMPOLINE_IMAGE} locally, aborting." + exit 1 + fi +fi + +# We use an array for the flags so they are easier to document. +docker_flags=( + # Remove the container after it exists. + "--rm" + + # Use the host network. + "--network=host" + + # Run in priviledged mode. We are not using docker for sandboxing or + # isolation, just for packaging our dev tools. + "--privileged" + + # Run the docker script with the user id. Because the docker image gets to + # write in ${PWD} you typically want this to be your user id. + # To allow docker in docker, we need to use docker gid on the host. + "--user" "${user_uid}:${docker_gid}" + + # Pass down the USER. + "--env" "USER=${user_name}" + + # Mount the project directory inside the Docker container. + "--volume" "${PROJECT_ROOT}:${TRAMPOLINE_WORKSPACE}" + "--workdir" "${TRAMPOLINE_WORKSPACE}" + "--env" "PROJECT_ROOT=${TRAMPOLINE_WORKSPACE}" + + # Mount the temporary home directory. + "--volume" "${tmphome}:/h" + "--env" "HOME=/h" + + # Allow docker in docker. + "--volume" "/var/run/docker.sock:/var/run/docker.sock" + + # Mount the /tmp so that docker in docker can mount the files + # there correctly. + "--volume" "/tmp:/tmp" + # Pass down the KOKORO_GFILE_DIR and KOKORO_KEYSTORE_DIR + # TODO(tmatsuo): This part is not portable. + "--env" "TRAMPOLINE_SECRET_DIR=/secrets" + "--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/secrets/gfile" + "--env" "KOKORO_GFILE_DIR=/secrets/gfile" + "--volume" "${KOKORO_KEYSTORE_DIR:-/dev/shm}:/secrets/keystore" + "--env" "KOKORO_KEYSTORE_DIR=/secrets/keystore" +) + +# Add an option for nicer output if the build gets a tty. +if [[ -t 0 ]]; then + docker_flags+=("-it") +fi + +# Passing down env vars +for e in "${pass_down_envvars[@]}" +do + if [[ -n "${!e:-}" ]]; then + docker_flags+=("--env" "${e}=${!e}") + fi +done + +# If arguments are given, all arguments will become the commands run +# in the container, otherwise run TRAMPOLINE_BUILD_FILE. +if [[ $# -ge 1 ]]; then + log_yellow "Running the given commands '" "${@:1}" "' in the container." + readonly commands=("${@:1}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" +else + log_yellow "Running the tests in a Docker container." + docker_flags+=("--entrypoint=${TRAMPOLINE_BUILD_FILE}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" +fi + + +test_retval=$? + +if [[ ${test_retval} -eq 0 ]]; then + log_green "Build finished with ${test_retval}" +else + log_red "Build finished with ${test_retval}" +fi + +# Only upload it when the test passes. +if [[ "${update_cache}" == "true" ]] && \ + [[ $test_retval == 0 ]] && \ + [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]]; then + log_yellow "Uploading the Docker image." + if docker push "${TRAMPOLINE_IMAGE}"; then + log_green "Finished uploading the Docker image." + else + log_red "Failed uploading the Docker image." + fi + # Call trampoline_after_upload_hook if it's defined. + if function_exists trampoline_after_upload_hook; then + trampoline_after_upload_hook + fi + +fi + +exit "${test_retval}" diff --git a/.trampolinerc b/.trampolinerc new file mode 100644 index 00000000..995ee291 --- /dev/null +++ b/.trampolinerc @@ -0,0 +1,51 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Template for .trampolinerc + +# Add required env vars here. +required_envvars+=( + "STAGING_BUCKET" + "V2_STAGING_BUCKET" +) + +# Add env vars which are passed down into the container here. +pass_down_envvars+=( + "STAGING_BUCKET" + "V2_STAGING_BUCKET" +) + +# Prevent unintentional override on the default image. +if [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]] && \ + [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + echo "Please set TRAMPOLINE_IMAGE if you want to upload the Docker image." + exit 1 +fi + +# Define the default value if it makes sense. +if [[ -z "${TRAMPOLINE_IMAGE_UPLOAD:-}" ]]; then + TRAMPOLINE_IMAGE_UPLOAD="" +fi + +if [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + TRAMPOLINE_IMAGE="" +fi + +if [[ -z "${TRAMPOLINE_DOCKERFILE:-}" ]]; then + TRAMPOLINE_DOCKERFILE="" +fi + +if [[ -z "${TRAMPOLINE_BUILD_FILE:-}" ]]; then + TRAMPOLINE_BUILD_FILE="" +fi diff --git a/docs/conf.py b/docs/conf.py index 136a058f..1b881e42 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,6 +20,10 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath("..")) +# For plugins that can not read conf.py. +# See also: https://github.com/docascode/sphinx-docfx-yaml/issues/85 +sys.path.insert(0, os.path.abspath(".")) + __version__ = "" # -- General configuration ------------------------------------------------ @@ -90,7 +94,12 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ["_build"] +exclude_patterns = [ + "_build", + "samples/AUTHORING_GUIDE.md", + "samples/CONTRIBUTING.md", + "samples/snippets/README.rst", +] # The reST default role (used for this markup: `text`) to use for all # documents. diff --git a/noxfile.py b/noxfile.py index f2b20939..b2bc0341 100644 --- a/noxfile.py +++ b/noxfile.py @@ -100,6 +100,10 @@ def system(session): """Run the system test suite.""" system_test_path = os.path.join("tests", "system.py") system_test_folder_path = os.path.join("tests", "system") + + # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. + if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false": + session.skip("RUN_SYSTEM_TESTS is set to false, skipping") # Sanity check: Only run tests if the environment variable is set. if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""): session.skip("Credentials must be set via environment variable") @@ -160,3 +164,36 @@ def docs(session): os.path.join("docs", ""), os.path.join("docs", "_build", "html", ""), ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def docfx(session): + """Build the docfx yaml files for this library.""" + + session.install("-e", ".") + session.install("sphinx<3.0.0", "alabaster", "recommonmark", "sphinx-docfx-yaml") + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-T", # show full traceback on exception + "-N", # no colors + "-D", + ( + "extensions=sphinx.ext.autodoc," + "sphinx.ext.autosummary," + "docfx_yaml.extension," + "sphinx.ext.intersphinx," + "sphinx.ext.coverage," + "sphinx.ext.napoleon," + "sphinx.ext.todo," + "sphinx.ext.viewcode," + "recommonmark" + ), + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) diff --git a/samples/AUTHORING_GUIDE.md b/samples/AUTHORING_GUIDE.md new file mode 100644 index 00000000..55c97b32 --- /dev/null +++ b/samples/AUTHORING_GUIDE.md @@ -0,0 +1 @@ +See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/AUTHORING_GUIDE.md \ No newline at end of file diff --git a/samples/CONTRIBUTING.md b/samples/CONTRIBUTING.md new file mode 100644 index 00000000..34c882b6 --- /dev/null +++ b/samples/CONTRIBUTING.md @@ -0,0 +1 @@ +See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/CONTRIBUTING.md \ No newline at end of file diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py new file mode 100644 index 00000000..ba55d7ce --- /dev/null +++ b/samples/snippets/noxfile.py @@ -0,0 +1,224 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +from pathlib import Path +import sys + +import nox + + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +# Copy `noxfile_config.py` to your directory and modify it instead. + + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + 'ignored_versions': ["2.7"], + + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + 'envs': {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append('.') + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars(): + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG['gcloud_project_env'] + # This should error out if not set. + ret['GOOGLE_CLOUD_PROJECT'] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG['envs']) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to tested samples. +ALL_VERSIONS = ["2.7", "3.6", "3.7", "3.8"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG['ignored_versions'] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = bool(os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False)) +# +# Style Checks +# + + +def _determine_local_import_names(start_dir): + """Determines all import names that should be considered "local". + + This is used when running the linter to insure that import order is + properly checked. + """ + file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] + return [ + basename + for basename, extension in file_ext_pairs + if extension == ".py" + or os.path.isdir(os.path.join(start_dir, basename)) + and basename not in ("__pycache__") + ] + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--import-order-style=google", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session): + session.install("flake8", "flake8-import-order") + + local_names = _determine_local_import_names(".") + args = FLAKE8_COMMON_ARGS + [ + "--application-import-names", + ",".join(local_names), + "." + ] + session.run("flake8", *args) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests(session, post_install=None): + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars() + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session): + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip("SKIPPED: {} tests are disabled for this sample.".format( + session.python + )) + + +# +# Readmegen +# + + +def _get_repo_root(): + """ Returns the root folder of the project. """ + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session, path): + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/synth.metadata b/synth.metadata index 8505ca1f..85423ab6 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,22 +4,21 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/python-tasks.git", - "sha": "6b8ba85de5998b0c2138bbf771fa16ba8f9bbf07" + "sha": "bb3fdb26ff34eaff9e9aa869687de5a7ce5a43a2" } }, { "git": { - "name": "googleapis", - "remote": "https://github.com/googleapis/googleapis.git", - "sha": "b882b8e6bfcd708042ff00f7adc67ce750817dd0", - "internalRef": "318028816" + "name": "synthtool", + "remote": "https://github.com/googleapis/synthtool.git", + "sha": "80f46100c047bc47efe0025ee537dc8ee413ad04" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "303271797a360f8a439203413f13a160f2f5b3b4" + "sha": "80f46100c047bc47efe0025ee537dc8ee413ad04" } } ], diff --git a/synth.py b/synth.py index a9c4d92f..f2a7d325 100644 --- a/synth.py +++ b/synth.py @@ -122,7 +122,7 @@ # ---------------------------------------------------------------------------- # Samples templates # ---------------------------------------------------------------------------- -python.py_samples() +python.py_samples(skip_readmes=True) # TODO(busunkim): Use latest sphinx after microgenerator transition s.replace("noxfile.py", """['"]sphinx['"]""", '"sphinx<3.0.0"')