Merge branch 'master' into cloudsql-refactor

SaketramDurbha · Jun 5, 2020 · b75dbcb · b75dbcb
2 parents d34b8b1 + af8d1a3
commit b75dbcb
Show file tree

Hide file tree

Showing 80 changed files with 594 additions and 399 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -51,7 +51,7 @@
 /pubsub/                        @anguillanneuf @hongalex @GoogleCloudPlatform/python-samples-owners
 /run/                           @averikitsch @GoogleCloudPlatform/python-samples-owners
 /scheduler/                     @averikitsch @GoogleCloudPlatform/python-samples-owners
-/spanner/                       @mmmarklu @GoogleCloudPlatform/python-samples-owners
+/spanner/                       @larkee @GoogleCloudPlatform/python-samples-owners
 /speech/                        @telpirion @sirtorry @GoogleCloudPlatform/python-samples-owners
 /storage/                       @GoogleCloudPlatform/storage-dpe @GoogleCloudPlatform/python-samples-owners
 /tables/automl/                 @telpirion @sirtorry @GoogleCloudPlatform/python-samples-owners

diff --git a/appengine/flexible/numpy/requirements.txt b/appengine/flexible/numpy/requirements.txt
@@ -1,3 +1,3 @@
 Flask==1.1.2
 gunicorn==20.0.4
-numpy==1.18.4
+numpy==1.18.5
diff --git a/appengine/flexible/scipy/requirements.txt b/appengine/flexible/scipy/requirements.txt
@@ -1,6 +1,6 @@
 Flask==1.1.2
 gunicorn==20.0.4
 imageio==2.8.0
-numpy==1.18.4
+numpy==1.18.5
 pillow==7.1.2
 scipy==1.4.1
diff --git a/appengine/flexible/tasks/requirements.txt b/appengine/flexible/tasks/requirements.txt
@@ -1,4 +1,3 @@
 Flask==1.1.2
 gunicorn==20.0.4
 google-cloud-tasks==1.5.0
-googleapis-common-protos==1.51.0
diff --git a/appengine/standard/firebase/firenotes/backend/requirements.txt b/appengine/standard/firebase/firenotes/backend/requirements.txt
@@ -1,6 +1,6 @@
 Flask==1.1.2
 pyjwt==1.7.1
 flask-cors==3.0.8
-google-auth==1.16.0
+google-auth==1.16.1
 requests==2.23.0
 requests-toolbelt==0.9.1
diff --git a/appengine/standard/firebase/firetactoe/requirements.txt b/appengine/standard/firebase/firetactoe/requirements.txt
@@ -1,5 +1,5 @@
 flask==1.1.2
 requests==2.23.0
 requests_toolbelt==0.9.1
-google-auth==1.16.0
+google-auth==1.16.1
 functools32==3.2.3.post2; python_version < "3"
diff --git a/appengine/standard/storage/api-client/requirements.txt b/appengine/standard/storage/api-client/requirements.txt
@@ -1,3 +1,3 @@
 google-api-python-client==1.9.1
-google-auth==1.16.0
+google-auth==1.16.1
 google-auth-httplib2==0.0.3
diff --git a/appengine/standard_python3/building-an-app/building-an-app-3/requirements.txt b/appengine/standard_python3/building-an-app/building-an-app-3/requirements.txt
@@ -1,4 +1,4 @@
 Flask==1.1.2
 google-cloud-datastore==1.12.0
-google-auth==1.16.0
+google-auth==1.16.1
 requests==2.23.0
diff --git a/appengine/standard_python3/building-an-app/building-an-app-4/requirements.txt b/appengine/standard_python3/building-an-app/building-an-app-4/requirements.txt
@@ -1,4 +1,4 @@
 Flask==1.1.2
 google-cloud-datastore==1.12.0
-google-auth==1.16.0
+google-auth==1.16.1
 requests==2.23.0
diff --git a/appengine/standard_python3/pubsub/requirements.txt b/appengine/standard_python3/pubsub/requirements.txt
@@ -1,4 +1,4 @@
 Flask==1.1.2
 google-api-python-client==1.9.1
-google-auth==1.16.0
+google-auth==1.16.1
 google-cloud-pubsub==1.5.0
diff --git a/appengine/standard_python37/building-an-app/building-an-app-3/requirements.txt b/appengine/standard_python37/building-an-app/building-an-app-3/requirements.txt
@@ -1,4 +1,4 @@
 Flask==1.1.2
 google-cloud-datastore==1.12.0
-google-auth==1.16.0
+google-auth==1.16.1
 requests==2.23.0
diff --git a/appengine/standard_python37/building-an-app/building-an-app-4/requirements.txt b/appengine/standard_python37/building-an-app/building-an-app-4/requirements.txt
@@ -1,4 +1,4 @@
 Flask==1.1.2
 google-cloud-datastore==1.12.0
-google-auth==1.16.0
+google-auth==1.16.1
 requests==2.23.0
diff --git a/appengine/standard_python37/pubsub/requirements.txt b/appengine/standard_python37/pubsub/requirements.txt
@@ -1,4 +1,4 @@
 Flask==1.1.2
 google-api-python-client==1.9.1
-google-auth==1.16.0
+google-auth==1.16.1
 google-cloud-pubsub==1.5.0
diff --git a/auth/api-client/requirements.txt b/auth/api-client/requirements.txt
@@ -1,3 +1,3 @@
 google-api-python-client==1.9.1
 google-auth-httplib2==0.0.3
-google-auth==1.16.0
+google-auth==1.16.1
diff --git a/auth/end-user/web/requirements.txt b/auth/end-user/web/requirements.txt
@@ -1,4 +1,4 @@
-google-auth==1.16.0
+google-auth==1.16.1
 google-auth-oauthlib==0.4.1
 google-auth-httplib2==0.0.3
 google-api-python-client==1.9.1

diff --git a/bigquery/pandas-gbq-migration/requirements.txt b/bigquery/pandas-gbq-migration/requirements.txt
@@ -1,5 +1,5 @@
 google-cloud-bigquery==1.24.0
-google-cloud-bigquery-storage==0.8.0
+google-cloud-bigquery-storage==1.0.0
 pandas==0.25.3; python_version > '3.0'
 pandas==0.24.2; python_version < '3.0'
 pandas-gbq==0.13.2

diff --git a/bigquery_storage/to_dataframe/requirements.txt b/bigquery_storage/to_dataframe/requirements.txt
@@ -1,5 +1,5 @@
-google-auth==1.16.0
-google-cloud-bigquery-storage==0.8.0
+google-auth==1.16.1
+google-cloud-bigquery-storage==1.0.0
 google-cloud-bigquery==1.24.0
 pyarrow==0.17.1
 ipython==7.10.2; python_version > '3.0'

diff --git a/composer/rest/requirements.txt b/composer/rest/requirements.txt
@@ -1,3 +1,3 @@
-google-auth==1.16.0
+google-auth==1.16.1
 requests==2.23.0
 six==1.15.0
diff --git a/compute/api/requirements.txt b/compute/api/requirements.txt
@@ -1,3 +1,3 @@
 google-api-python-client==1.9.1
-google-auth==1.16.0
+google-auth==1.16.1
 google-auth-httplib2==0.0.3
diff --git a/compute/auth/requirements.txt b/compute/auth/requirements.txt
@@ -1,4 +1,4 @@
 requests==2.23.0
 google-api-python-client==1.9.1
-google-auth==1.16.0
+google-auth==1.16.1
 google-auth-httplib2==0.0.3
diff --git a/compute/encryption/requirements.txt b/compute/encryption/requirements.txt
@@ -1,5 +1,5 @@
 cryptography==2.9.2
 requests==2.23.0
 google-api-python-client==1.9.1
-google-auth==1.16.0
+google-auth==1.16.1
 google-auth-httplib2==0.0.3
diff --git a/compute/oslogin/requirements.txt b/compute/oslogin/requirements.txt
@@ -1,4 +1,4 @@
 google-api-python-client==1.9.1
-google-auth==1.16.0
+google-auth==1.16.1
 google-auth-httplib2==0.0.3
 requests==2.23.0
diff --git a/dataproc/requirements.txt b/dataproc/requirements.txt
@@ -1,5 +1,5 @@
 grpcio==1.29.0
-google-auth==1.16.0
+google-auth==1.16.1
 google-auth-httplib2==0.0.3
 google-cloud==0.34.0
 google-cloud-storage==1.28.1

diff --git a/dlp/README.rst b/dlp/README.rst
@@ -14,6 +14,15 @@ This directory contains samples for Google Data Loss Prevention. `Google Data Lo
 
 .. _Google Data Loss Prevention: https://cloud.google.com/dlp/docs/
 
+To run the sample, you need to enable the API at: https://console.cloud.google.com/apis/library/dlp.googleapis.com
+
+
+To run the sample, you need to have the following roles:
+* `DLP Administrator`
+* `DLP API Service Agent`
+
+
+
 Setup
 -------------------------------------------------------------------------------
 
@@ -58,15 +67,6 @@ Install Dependencies
 .. _pip: https://pip.pypa.io/
 .. _virtualenv: https://virtualenv.pypa.io/
 
-#. For running *_test.py files, install test dependencies
-
-    .. code-block:: bash
-
-        $ pip install -r requirements-test.txt
-        $ pytest inspect_content_test.py
-
-** *_test.py files are demo wrappers and make API calls. You may get rate limited for making high number of requests. **
-
 Samples
 -------------------------------------------------------------------------------
 
@@ -83,7 +83,7 @@ To run this sample:
 
 .. code-block:: bash
 
-    $ python quickstart.py <project-id>
+    $ python quickstart.py
 
 
 Inspect Content
@@ -101,15 +101,16 @@ To run this sample:
 
     $ python inspect_content.py
 
-    usage: inspect_content.py [-h] {string,file,gcs,datastore,bigquery} ...
+    usage: inspect_content.py [-h] {string,table,file,gcs,datastore,bigquery} ...
 
     Sample app that uses the Data Loss Prevention API to inspect a string, a local
     file or a file on Google Cloud Storage.
 
     positional arguments:
-      {string,file,gcs,datastore,bigquery}
+      {string,table,file,gcs,datastore,bigquery}
                             Select how to submit content to the API.
         string              Inspect a string.
+        table               Inspect a table.
         file                Inspect a local file.
         gcs                 Inspect files on Google Cloud Storage.
         datastore           Inspect files on Google Datastore.
@@ -135,13 +136,14 @@ To run this sample:
 
     $ python redact.py
 
-    usage: redact.py [-h] [--project PROJECT] [--info_types INFO_TYPES]
+    usage: redact.py [-h] [--project PROJECT]
+                     [--info_types INFO_TYPES [INFO_TYPES ...]]
                      [--min_likelihood {LIKELIHOOD_UNSPECIFIED,VERY_UNLIKELY,UNLIKELY,POSSIBLE,LIKELY,VERY_LIKELY}]
                      [--mime_type MIME_TYPE]
                      filename output_filename
 
-    Sample app that uses the Data Loss Prevent API to redact the contents of a
-    string or an image file.
+    Sample app that uses the Data Loss Prevent API to redact the contents of an
+    image file.
 
     positional arguments:
       filename              The path to the file to inspect.
@@ -151,7 +153,7 @@ To run this sample:
       -h, --help            show this help message and exit
       --project PROJECT     The Google Cloud project id to use as a parent
                             resource.
-      --info_types INFO_TYPES
+      --info_types INFO_TYPES [INFO_TYPES ...]
                             Strings representing info types to look for. A full
                             list of info categories and types is available from
                             the API. Examples include "FIRST_NAME", "LAST_NAME",

diff --git a/dlp/README.rst.in b/dlp/README.rst.in
@@ -4,7 +4,7 @@ product:
   name: Google Data Loss Prevention
   short_name: Data Loss Prevention
   url: https://cloud.google.com/dlp/docs/
-  description: > 
+  description: >
     `Google Data Loss Prevention`_ provides programmatic access to a powerful
     detection engine for personally identifiable information and other
     privacy-sensitive data in unstructured data streams.
@@ -13,6 +13,12 @@ setup:
 - auth
 - install_deps
 
+required_api_url: https://console.cloud.google.com/apis/library/dlp.googleapis.com
+
+required_roles:
+- DLP Administrator
+- DLP API Service Agent
+
 samples:
 - name: Quickstart
   file: quickstart.py

diff --git a/dlp/conftest.py b/dlp/conftest.py
diff --git a/dlp/inspect_content.py b/dlp/inspect_content.py
@@ -459,11 +459,12 @@ def inspect_gcs_file(
     url = "gs://{}/{}".format(bucket, filename)
     storage_config = {"cloud_storage_options": {"file_set": {"url": url}}}
 
-    # Convert the project id into a full resource id.
-    parent = dlp.project_path(project)
+    # Convert the project id into full resource ids.
+    topic = google.cloud.pubsub.PublisherClient.topic_path(project, topic_id)
+    parent = dlp.location_path(project, 'global')
 
     # Tell the API where to send a notification when the job is complete.
-    actions = [{"pub_sub": {"topic": "{}/topics/{}".format(parent, topic_id)}}]
+    actions = [{"pub_sub": {"topic": topic}}]
 
     # Construct the inspect_job, which defines the entire inspect content task.
     inspect_job = {
@@ -623,11 +624,12 @@ def inspect_datastore(
         }
     }
 
-    # Convert the project id into a full resource id.
-    parent = dlp.project_path(project)
+    # Convert the project id into full resource ids.
+    topic = google.cloud.pubsub.PublisherClient.topic_path(project, topic_id)
+    parent = dlp.location_path(project, 'global')
 
     # Tell the API where to send a notification when the job is complete.
-    actions = [{"pub_sub": {"topic": "{}/topics/{}".format(parent, topic_id)}}]
+    actions = [{"pub_sub": {"topic": topic}}]
 
     # Construct the inspect_job, which defines the entire inspect content task.
     inspect_job = {
@@ -790,11 +792,12 @@ def inspect_bigquery(
         }
     }
 
-    # Convert the project id into a full resource id.
-    parent = dlp.project_path(project)
+    # Convert the project id into full resource ids.
+    topic = google.cloud.pubsub.PublisherClient.topic_path(project, topic_id)
+    parent = dlp.location_path(project, 'global')
 
     # Tell the API where to send a notification when the job is complete.
-    actions = [{"pub_sub": {"topic": "{}/topics/{}".format(parent, topic_id)}}]
+    actions = [{"pub_sub": {"topic": topic}}]
 
     # Construct the inspect_job, which defines the entire inspect content task.
     inspect_job = {