From 25500644abbcc5970afe31c2ff277d562477a667 Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Mon, 6 May 2019 21:08:26 -0300 Subject: [PATCH 01/14] Add samples for Data Catalog lookup_entry --- datacatalog/cloud-client/README.md | 0 datacatalog/cloud-client/lookup_entry.py | 92 +++++++++++++++++++++++ datacatalog/cloud-client/requirements.txt | 1 + 3 files changed, 93 insertions(+) create mode 100644 datacatalog/cloud-client/README.md create mode 100644 datacatalog/cloud-client/lookup_entry.py create mode 100644 datacatalog/cloud-client/requirements.txt diff --git a/datacatalog/cloud-client/README.md b/datacatalog/cloud-client/README.md new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/datacatalog/cloud-client/lookup_entry.py b/datacatalog/cloud-client/lookup_entry.py new file mode 100644 index 000000000000..da3d54d6a35e --- /dev/null +++ b/datacatalog/cloud-client/lookup_entry.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to perform basic operations on entries +with the Cloud Data Catalog API. + +For more information, see the README.md under /datacatalog and the documentation +at https://cloud.google.com/data-catalog/docs. +""" + +import argparse + + +def lookup_bigquery_dataset(project_id, dataset_id): + """Retrieves Data Catalog's entry for the given dataset.""" + from google.cloud import datacatalog_v1beta1 + + datacatalog = datacatalog_v1beta1.DataCatalogClient() + + resource_name = '//bigquery.googleapis.com/projects/{}/datasets/{}'\ + .format(project_id, dataset_id) + + return datacatalog.lookup_entry(linked_resource=resource_name) + + +def lookup_bigquery_table(project_id, dataset_id, table_id): + """Retrieves Data Catalog's entry for the given table.""" + from google.cloud import datacatalog_v1beta1 + + datacatalog = datacatalog_v1beta1.DataCatalogClient() + + resource_name = '//bigquery.googleapis.com/projects/{}/datasets/{}/tables/{}'\ + .format(project_id, dataset_id, table_id) + + return datacatalog.lookup_entry(linked_resource=resource_name) + + +def lookup_pubsub_topic(project_id, topic_id): + """Retrieves Data Catalog's entry for the given topic.""" + from google.cloud import datacatalog_v1beta1 + + datacatalog = datacatalog_v1beta1.DataCatalogClient() + + resource_name = '//pubsub.googleapis.com/projects/{}/topics/{}'\ + .format(project_id, topic_id) + + return datacatalog.lookup_entry(linked_resource=resource_name) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('project_id', help='Your Google Cloud project ID') + + subparsers = parser.add_subparsers(dest='command') + + bigquery_dataset_parser = subparsers.add_parser( + 'lookup-bigquery-dataset', help=lookup_bigquery_dataset.__doc__) + bigquery_dataset_parser.add_argument('dataset_id') + + bigquery_table_parser = subparsers.add_parser( + 'lookup-bigquery-table', help=lookup_bigquery_table.__doc__) + bigquery_table_parser.add_argument('dataset_id') + bigquery_table_parser.add_argument('table_id') + + pubsub_topic_parser = subparsers.add_parser( + 'lookup-pubsub-topic', help=lookup_pubsub_topic.__doc__) + pubsub_topic_parser.add_argument('topic_id') + + args = parser.parse_args() + + if args.command == 'lookup-bigquery-dataset': + lookup_bigquery_dataset(args.project_id, args.dataset_id) + elif args.command == 'lookup-bigquery-table': + lookup_bigquery_table(args.project_id, args.dataset_id, args.table_id) + elif args.command == 'lookup-pubsub-topic': + lookup_pubsub_topic(args.project_id, args.topic_id) diff --git a/datacatalog/cloud-client/requirements.txt b/datacatalog/cloud-client/requirements.txt new file mode 100644 index 000000000000..e6408e76606b --- /dev/null +++ b/datacatalog/cloud-client/requirements.txt @@ -0,0 +1 @@ +google-cloud-datacatalog From 3fcdb2bdc66bbe4efc566d1b967443514bf17912 Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Thu, 9 May 2019 10:53:28 -0300 Subject: [PATCH 02/14] Add tests for Data Catalog lookup_entry --- datacatalog/cloud-client/lookup_entry_test.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 datacatalog/cloud-client/lookup_entry_test.py diff --git a/datacatalog/cloud-client/lookup_entry_test.py b/datacatalog/cloud-client/lookup_entry_test.py new file mode 100644 index 000000000000..7a4f671526bd --- /dev/null +++ b/datacatalog/cloud-client/lookup_entry_test.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from gcp_devrel.testing import eventually_consistent + +import lookup_entry + +PROJECT = os.environ['GCLOUD_PROJECT'] +BIGQUERY_DATASET = os.environ['GCLOUD_BIGQUERY_DATASET'] +BIGQUERY_TABLE = os.environ['GCLOUD_BIGQUERY_TABLE'] +PUBSUB_TOPIC = os.environ['GCLOUD_PUBSUB_TOPIC'] + + +def test_lookup_bigquery_dataset(): + @eventually_consistent.call + def _(): + assert lookup_entry.lookup_bigquery_dataset(PROJECT, BIGQUERY_DATASET) + + +def test_lookup_bigquery_table(): + @eventually_consistent.call + def _(): + assert lookup_entry.lookup_bigquery_table(PROJECT, BIGQUERY_DATASET, BIGQUERY_TABLE) + + +def test_lookup_pubsub_topic(): + @eventually_consistent.call + def _(): + assert lookup_entry.lookup_pubsub_topic(PROJECT, PUBSUB_TOPIC) From fa90bbfb5c7d0bf743aec7f90a460ff271fc6c61 Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Thu, 9 May 2019 11:18:03 -0300 Subject: [PATCH 03/14] Add samples for lookup_entry by SQL Resource --- datacatalog/cloud-client/lookup_entry.py | 73 +++++++++++++++++-- datacatalog/cloud-client/lookup_entry_test.py | 24 +++++- 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/datacatalog/cloud-client/lookup_entry.py b/datacatalog/cloud-client/lookup_entry.py index da3d54d6a35e..db33174b0c28 100644 --- a/datacatalog/cloud-client/lookup_entry.py +++ b/datacatalog/cloud-client/lookup_entry.py @@ -33,7 +33,22 @@ def lookup_bigquery_dataset(project_id, dataset_id): resource_name = '//bigquery.googleapis.com/projects/{}/datasets/{}'\ .format(project_id, dataset_id) - return datacatalog.lookup_entry(linked_resource=resource_name) + entry = datacatalog.lookup_entry(linked_resource=resource_name) + print(entry.name) + return entry + + +def lookup_bigquery_dataset_sql_resource(project_id, dataset_id): + """Retrieves Data Catalog's entry for the given dataset by sql_resource.""" + from google.cloud import datacatalog_v1beta1 + + datacatalog = datacatalog_v1beta1.DataCatalogClient() + + sql_resource = 'bigquery.dataset.`{}`.`{}`'.format(project_id, dataset_id) + + entry = datacatalog.lookup_entry(sql_resource=sql_resource) + print(entry.name) + return entry def lookup_bigquery_table(project_id, dataset_id, table_id): @@ -45,7 +60,23 @@ def lookup_bigquery_table(project_id, dataset_id, table_id): resource_name = '//bigquery.googleapis.com/projects/{}/datasets/{}/tables/{}'\ .format(project_id, dataset_id, table_id) - return datacatalog.lookup_entry(linked_resource=resource_name) + entry = datacatalog.lookup_entry(linked_resource=resource_name) + print(entry.name) + return entry + + +def lookup_bigquery_table_sql_resource(project_id, dataset_id, table_id): + """Retrieves Data Catalog's entry for the given table by sql_resource.""" + from google.cloud import datacatalog_v1beta1 + + datacatalog = datacatalog_v1beta1.DataCatalogClient() + + sql_resource = 'bigquery.table.`{}`.`{}`.`{}`'.format( + project_id, dataset_id, table_id) + + entry = datacatalog.lookup_entry(sql_resource=sql_resource) + print(entry.name) + return entry def lookup_pubsub_topic(project_id, topic_id): @@ -57,7 +88,22 @@ def lookup_pubsub_topic(project_id, topic_id): resource_name = '//pubsub.googleapis.com/projects/{}/topics/{}'\ .format(project_id, topic_id) - return datacatalog.lookup_entry(linked_resource=resource_name) + entry = datacatalog.lookup_entry(linked_resource=resource_name) + print(entry.name) + return entry + + +def lookup_pubsub_topic_sql_resource(project_id, topic_id): + """Retrieves Data Catalog's entry for the given topic by sql_resource.""" + from google.cloud import datacatalog_v1beta1 + + datacatalog = datacatalog_v1beta1.DataCatalogClient() + + sql_resource = 'pubsub.topic.`{}`.`{}`'.format(project_id, topic_id) + + entry = datacatalog.lookup_entry(sql_resource=sql_resource) + print(entry.name) + return entry if __name__ == '__main__': @@ -66,6 +112,8 @@ def lookup_pubsub_topic(project_id, topic_id): formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument('project_id', help='Your Google Cloud project ID') + parser.add_argument('--sql-resource', action='store_true', + help='Perform lookup by SQL Resource') subparsers = parser.add_subparsers(dest='command') @@ -85,8 +133,21 @@ def lookup_pubsub_topic(project_id, topic_id): args = parser.parse_args() if args.command == 'lookup-bigquery-dataset': - lookup_bigquery_dataset(args.project_id, args.dataset_id) + methods = { + False: lookup_bigquery_dataset, + True: lookup_bigquery_dataset_sql_resource + } + methods[args.sql_resource](args.project_id, args.dataset_id) elif args.command == 'lookup-bigquery-table': - lookup_bigquery_table(args.project_id, args.dataset_id, args.table_id) + methods = { + False: lookup_bigquery_table, + True: lookup_bigquery_table_sql_resource + } + methods[args.sql_resource]( + args.project_id, args.dataset_id, args.table_id) elif args.command == 'lookup-pubsub-topic': - lookup_pubsub_topic(args.project_id, args.topic_id) + methods = { + False: lookup_pubsub_topic, + True: lookup_pubsub_topic_sql_resource + } + methods[args.sql_resource](args.project_id, args.topic_id) diff --git a/datacatalog/cloud-client/lookup_entry_test.py b/datacatalog/cloud-client/lookup_entry_test.py index 7a4f671526bd..b570ca9fca1f 100644 --- a/datacatalog/cloud-client/lookup_entry_test.py +++ b/datacatalog/cloud-client/lookup_entry_test.py @@ -32,13 +32,35 @@ def _(): assert lookup_entry.lookup_bigquery_dataset(PROJECT, BIGQUERY_DATASET) +def test_lookup_bigquery_dataset_sql_resource(): + @eventually_consistent.call + def _(): + assert lookup_entry.lookup_bigquery_dataset_sql_resource( + PROJECT, BIGQUERY_DATASET) + + def test_lookup_bigquery_table(): @eventually_consistent.call def _(): - assert lookup_entry.lookup_bigquery_table(PROJECT, BIGQUERY_DATASET, BIGQUERY_TABLE) + assert lookup_entry.lookup_bigquery_table( + PROJECT, BIGQUERY_DATASET, BIGQUERY_TABLE) + + +def test_lookup_bigquery_table_sql_resource(): + @eventually_consistent.call + def _(): + assert lookup_entry.lookup_bigquery_table_sql_resource( + PROJECT, BIGQUERY_DATASET, BIGQUERY_TABLE) def test_lookup_pubsub_topic(): @eventually_consistent.call def _(): assert lookup_entry.lookup_pubsub_topic(PROJECT, PUBSUB_TOPIC) + + +def test_lookup_pubsub_topic_sql_resource(): + @eventually_consistent.call + def _(): + assert lookup_entry.lookup_pubsub_topic_sql_resource( + PROJECT, PUBSUB_TOPIC) From e30befbba6a41790d5a3a109812d1dc7238bdfd5 Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Thu, 9 May 2019 11:42:43 -0300 Subject: [PATCH 04/14] Add README.rst --- datacatalog/cloud-client/README.md | 0 datacatalog/cloud-client/README.rst | 123 +++++++++++++++++++++++++ datacatalog/cloud-client/README.rst.in | 23 +++++ 3 files changed, 146 insertions(+) delete mode 100644 datacatalog/cloud-client/README.md create mode 100644 datacatalog/cloud-client/README.rst create mode 100644 datacatalog/cloud-client/README.rst.in diff --git a/datacatalog/cloud-client/README.md b/datacatalog/cloud-client/README.md deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/datacatalog/cloud-client/README.rst b/datacatalog/cloud-client/README.rst new file mode 100644 index 000000000000..972f3fa9c6e8 --- /dev/null +++ b/datacatalog/cloud-client/README.rst @@ -0,0 +1,123 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Data Catalog Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=datacatalog/cloud-client/README.rst + + +This directory contains samples for Google Cloud Data Catalog. `Data Catalog`_ is a fully managed and scalable metadata management service that empowers organizations to quickly discover, manage, and understand all their data in Google Cloud.. + + + + +.. _Google Cloud Data Catalog: https://cloud.google.com/data-catalog/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Lookup entry ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=datacatalog/cloud-client/lookup_entry.py,datacatalog/cloud-client/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python lookup_entry.py + + usage: lookup_entry.py [-h] [--sql-resource] + project_id + {lookup-bigquery-dataset,lookup-bigquery-table,lookup-pubsub-topic} + ... + + This application demonstrates how to perform basic operations on entries + with the Cloud Data Catalog API. + + For more information, see the README.md under /datacatalog and the documentation + at https://cloud.google.com/data-catalog/docs. + + positional arguments: + project_id Your Google Cloud project ID + {lookup-bigquery-dataset,lookup-bigquery-table,lookup-pubsub-topic} + lookup-bigquery-dataset + Retrieves Data Catalog's entry for the given dataset. + lookup-bigquery-table + Retrieves Data Catalog's entry for the given table. + lookup-pubsub-topic + Retrieves Data Catalog's entry for the given topic. + + optional arguments: + -h, --help show this help message and exit + --sql-resource Perform lookup by SQL Resource + + + + + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/datacatalog/cloud-client/README.rst.in b/datacatalog/cloud-client/README.rst.in new file mode 100644 index 000000000000..718351b83618 --- /dev/null +++ b/datacatalog/cloud-client/README.rst.in @@ -0,0 +1,23 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Data Catalog + short_name: Data Catalog + url: https://cloud.google.com/data-catalog/docs + description: > + `Data Catalog`_ is a fully managed and scalable metadata management service + that empowers organizations to quickly discover, manage, and understand all + their data in Google Cloud.. + +setup: +- auth +- install_deps + +samples: +- name: Lookup entry + file: lookup_entry.py + show_help: true + +cloud_client_library: true + +folder: datacatalog/cloud-client From 9ab7b1d37588706cb7045373fc9f6e03d8d1d6d7 Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Thu, 9 May 2019 11:46:51 -0300 Subject: [PATCH 05/14] Fix README.rst --- datacatalog/cloud-client/README.rst | 2 +- datacatalog/cloud-client/README.rst.in | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/datacatalog/cloud-client/README.rst b/datacatalog/cloud-client/README.rst index 972f3fa9c6e8..b9b08f1129f3 100644 --- a/datacatalog/cloud-client/README.rst +++ b/datacatalog/cloud-client/README.rst @@ -7,7 +7,7 @@ Google Cloud Data Catalog Python Samples :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=datacatalog/cloud-client/README.rst -This directory contains samples for Google Cloud Data Catalog. `Data Catalog`_ is a fully managed and scalable metadata management service that empowers organizations to quickly discover, manage, and understand all their data in Google Cloud.. +This directory contains samples for Google Cloud Data Catalog. `Google Cloud Data Catalog`_ is a fully managed and scalable metadata management service that empowers organizations to quickly discover, manage, and understand all their data in Google Cloud. diff --git a/datacatalog/cloud-client/README.rst.in b/datacatalog/cloud-client/README.rst.in index 718351b83618..704d55a5f9f0 100644 --- a/datacatalog/cloud-client/README.rst.in +++ b/datacatalog/cloud-client/README.rst.in @@ -5,9 +5,9 @@ product: short_name: Data Catalog url: https://cloud.google.com/data-catalog/docs description: > - `Data Catalog`_ is a fully managed and scalable metadata management service - that empowers organizations to quickly discover, manage, and understand all - their data in Google Cloud.. + `Google Cloud Data Catalog`_ is a fully managed and scalable metadata + management service that empowers organizations to quickly discover, manage, + and understand all their data in Google Cloud. setup: - auth From 061c18a46a4e169061c4cd70d89c2fbed7775e0e Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Thu, 9 May 2019 11:49:38 -0300 Subject: [PATCH 06/14] Update comments to improve README.rst formatting --- datacatalog/cloud-client/README.rst | 6 +++--- datacatalog/cloud-client/lookup_entry.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/datacatalog/cloud-client/README.rst b/datacatalog/cloud-client/README.rst index b9b08f1129f3..8c8c1ed28425 100644 --- a/datacatalog/cloud-client/README.rst +++ b/datacatalog/cloud-client/README.rst @@ -91,11 +91,11 @@ To run this sample: project_id Your Google Cloud project ID {lookup-bigquery-dataset,lookup-bigquery-table,lookup-pubsub-topic} lookup-bigquery-dataset - Retrieves Data Catalog's entry for the given dataset. + Retrieves Data Catalog entry for the given dataset. lookup-bigquery-table - Retrieves Data Catalog's entry for the given table. + Retrieves Data Catalog entry for the given table. lookup-pubsub-topic - Retrieves Data Catalog's entry for the given topic. + Retrieves Data Catalog entry for the given topic. optional arguments: -h, --help show this help message and exit diff --git a/datacatalog/cloud-client/lookup_entry.py b/datacatalog/cloud-client/lookup_entry.py index db33174b0c28..60ee1d650cb8 100644 --- a/datacatalog/cloud-client/lookup_entry.py +++ b/datacatalog/cloud-client/lookup_entry.py @@ -25,7 +25,7 @@ def lookup_bigquery_dataset(project_id, dataset_id): - """Retrieves Data Catalog's entry for the given dataset.""" + """Retrieves Data Catalog entry for the given dataset.""" from google.cloud import datacatalog_v1beta1 datacatalog = datacatalog_v1beta1.DataCatalogClient() @@ -39,7 +39,7 @@ def lookup_bigquery_dataset(project_id, dataset_id): def lookup_bigquery_dataset_sql_resource(project_id, dataset_id): - """Retrieves Data Catalog's entry for the given dataset by sql_resource.""" + """Retrieves Data Catalog entry for the given dataset by sql_resource.""" from google.cloud import datacatalog_v1beta1 datacatalog = datacatalog_v1beta1.DataCatalogClient() @@ -52,7 +52,7 @@ def lookup_bigquery_dataset_sql_resource(project_id, dataset_id): def lookup_bigquery_table(project_id, dataset_id, table_id): - """Retrieves Data Catalog's entry for the given table.""" + """Retrieves Data Catalog entry for the given table.""" from google.cloud import datacatalog_v1beta1 datacatalog = datacatalog_v1beta1.DataCatalogClient() @@ -66,7 +66,7 @@ def lookup_bigquery_table(project_id, dataset_id, table_id): def lookup_bigquery_table_sql_resource(project_id, dataset_id, table_id): - """Retrieves Data Catalog's entry for the given table by sql_resource.""" + """Retrieves Data Catalog entry for the given table by sql_resource.""" from google.cloud import datacatalog_v1beta1 datacatalog = datacatalog_v1beta1.DataCatalogClient() @@ -80,7 +80,7 @@ def lookup_bigquery_table_sql_resource(project_id, dataset_id, table_id): def lookup_pubsub_topic(project_id, topic_id): - """Retrieves Data Catalog's entry for the given topic.""" + """Retrieves Data Catalog entry for the given topic.""" from google.cloud import datacatalog_v1beta1 datacatalog = datacatalog_v1beta1.DataCatalogClient() @@ -94,7 +94,7 @@ def lookup_pubsub_topic(project_id, topic_id): def lookup_pubsub_topic_sql_resource(project_id, topic_id): - """Retrieves Data Catalog's entry for the given topic by sql_resource.""" + """Retrieves Data Catalog entry for the given topic by sql_resource.""" from google.cloud import datacatalog_v1beta1 datacatalog = datacatalog_v1beta1.DataCatalogClient() From d843626103b4ef2557d2ffd26691b79a53e73aa9 Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Thu, 9 May 2019 11:55:14 -0300 Subject: [PATCH 07/14] Update comments to improve README.rst formatting --- datacatalog/cloud-client/README.rst | 9 ++++++--- datacatalog/cloud-client/lookup_entry.py | 18 ++++++++++++------ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/datacatalog/cloud-client/README.rst b/datacatalog/cloud-client/README.rst index 8c8c1ed28425..979b2a41ce91 100644 --- a/datacatalog/cloud-client/README.rst +++ b/datacatalog/cloud-client/README.rst @@ -91,11 +91,14 @@ To run this sample: project_id Your Google Cloud project ID {lookup-bigquery-dataset,lookup-bigquery-table,lookup-pubsub-topic} lookup-bigquery-dataset - Retrieves Data Catalog entry for the given dataset. + Retrieves Data Catalog entry for the given BigQuery + Dataset. lookup-bigquery-table - Retrieves Data Catalog entry for the given table. + Retrieves Data Catalog entry for the given BigQuery + Table. lookup-pubsub-topic - Retrieves Data Catalog entry for the given topic. + Retrieves Data Catalog entry for the given Pub/Sub + Topic. optional arguments: -h, --help show this help message and exit diff --git a/datacatalog/cloud-client/lookup_entry.py b/datacatalog/cloud-client/lookup_entry.py index 60ee1d650cb8..a751f5c4c60d 100644 --- a/datacatalog/cloud-client/lookup_entry.py +++ b/datacatalog/cloud-client/lookup_entry.py @@ -25,7 +25,7 @@ def lookup_bigquery_dataset(project_id, dataset_id): - """Retrieves Data Catalog entry for the given dataset.""" + """Retrieves Data Catalog entry for the given BigQuery Dataset.""" from google.cloud import datacatalog_v1beta1 datacatalog = datacatalog_v1beta1.DataCatalogClient() @@ -39,7 +39,9 @@ def lookup_bigquery_dataset(project_id, dataset_id): def lookup_bigquery_dataset_sql_resource(project_id, dataset_id): - """Retrieves Data Catalog entry for the given dataset by sql_resource.""" + """Retrieves Data Catalog entry for the given BigQuery Dataset by + sql_resource. + """ from google.cloud import datacatalog_v1beta1 datacatalog = datacatalog_v1beta1.DataCatalogClient() @@ -52,7 +54,7 @@ def lookup_bigquery_dataset_sql_resource(project_id, dataset_id): def lookup_bigquery_table(project_id, dataset_id, table_id): - """Retrieves Data Catalog entry for the given table.""" + """Retrieves Data Catalog entry for the given BigQuery Table.""" from google.cloud import datacatalog_v1beta1 datacatalog = datacatalog_v1beta1.DataCatalogClient() @@ -66,7 +68,9 @@ def lookup_bigquery_table(project_id, dataset_id, table_id): def lookup_bigquery_table_sql_resource(project_id, dataset_id, table_id): - """Retrieves Data Catalog entry for the given table by sql_resource.""" + """Retrieves Data Catalog entry for the given BigQuery Table by + sql_resource. + """ from google.cloud import datacatalog_v1beta1 datacatalog = datacatalog_v1beta1.DataCatalogClient() @@ -80,7 +84,7 @@ def lookup_bigquery_table_sql_resource(project_id, dataset_id, table_id): def lookup_pubsub_topic(project_id, topic_id): - """Retrieves Data Catalog entry for the given topic.""" + """Retrieves Data Catalog entry for the given Pub/Sub Topic.""" from google.cloud import datacatalog_v1beta1 datacatalog = datacatalog_v1beta1.DataCatalogClient() @@ -94,7 +98,9 @@ def lookup_pubsub_topic(project_id, topic_id): def lookup_pubsub_topic_sql_resource(project_id, topic_id): - """Retrieves Data Catalog entry for the given topic by sql_resource.""" + """Retrieves Data Catalog entry for the given Pub/Sub Topic by + sql_resource. + """ from google.cloud import datacatalog_v1beta1 datacatalog = datacatalog_v1beta1.DataCatalogClient() From 4fc72875b17df59a2939145de5a7960a478bbf90 Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Thu, 9 May 2019 14:04:50 -0300 Subject: [PATCH 08/14] Fix max line length issues --- datacatalog/cloud-client/lookup_entry.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datacatalog/cloud-client/lookup_entry.py b/datacatalog/cloud-client/lookup_entry.py index a751f5c4c60d..e71910a38e2b 100644 --- a/datacatalog/cloud-client/lookup_entry.py +++ b/datacatalog/cloud-client/lookup_entry.py @@ -59,7 +59,8 @@ def lookup_bigquery_table(project_id, dataset_id, table_id): datacatalog = datacatalog_v1beta1.DataCatalogClient() - resource_name = '//bigquery.googleapis.com/projects/{}/datasets/{}/tables/{}'\ + resource_name = '//bigquery.googleapis.com/projects/{}/datasets/{}' \ + '/tables/{}'\ .format(project_id, dataset_id, table_id) entry = datacatalog.lookup_entry(linked_resource=resource_name) From 685fcbbb7bf2851eb571d58ff8ee9840f2accfa0 Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Thu, 9 May 2019 14:09:07 -0300 Subject: [PATCH 09/14] Fix max line length issues --- datacatalog/cloud-client/README.rst | 4 ++-- datacatalog/cloud-client/lookup_entry.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/datacatalog/cloud-client/README.rst b/datacatalog/cloud-client/README.rst index 979b2a41ce91..e4ed58c67258 100644 --- a/datacatalog/cloud-client/README.rst +++ b/datacatalog/cloud-client/README.rst @@ -84,8 +84,8 @@ To run this sample: This application demonstrates how to perform basic operations on entries with the Cloud Data Catalog API. - For more information, see the README.md under /datacatalog and the documentation - at https://cloud.google.com/data-catalog/docs. + For more information, see the README.md under /datacatalog and the + documentation at https://cloud.google.com/data-catalog/docs. positional arguments: project_id Your Google Cloud project ID diff --git a/datacatalog/cloud-client/lookup_entry.py b/datacatalog/cloud-client/lookup_entry.py index e71910a38e2b..8f0858c220a0 100644 --- a/datacatalog/cloud-client/lookup_entry.py +++ b/datacatalog/cloud-client/lookup_entry.py @@ -17,8 +17,8 @@ """This application demonstrates how to perform basic operations on entries with the Cloud Data Catalog API. -For more information, see the README.md under /datacatalog and the documentation -at https://cloud.google.com/data-catalog/docs. +For more information, see the README.md under /datacatalog and the +documentation at https://cloud.google.com/data-catalog/docs. """ import argparse From df864954e53453cacf29e033fa9ff68c2decb78a Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Thu, 9 May 2019 16:36:06 -0300 Subject: [PATCH 10/14] Pin google-cloud-datacatalog version number --- datacatalog/cloud-client/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datacatalog/cloud-client/requirements.txt b/datacatalog/cloud-client/requirements.txt index e6408e76606b..7882537c470c 100644 --- a/datacatalog/cloud-client/requirements.txt +++ b/datacatalog/cloud-client/requirements.txt @@ -1 +1 @@ -google-cloud-datacatalog +google-cloud-datacatalog==0.1.0 From fb77f4d041205d3355655f25b9b6349c8350162b Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Fri, 10 May 2019 15:26:52 -0300 Subject: [PATCH 11/14] Readability improvements --- datacatalog/cloud-client/lookup_entry.py | 53 +++++++++--------------- 1 file changed, 19 insertions(+), 34 deletions(-) diff --git a/datacatalog/cloud-client/lookup_entry.py b/datacatalog/cloud-client/lookup_entry.py index 8f0858c220a0..672273274749 100644 --- a/datacatalog/cloud-client/lookup_entry.py +++ b/datacatalog/cloud-client/lookup_entry.py @@ -33,9 +33,7 @@ def lookup_bigquery_dataset(project_id, dataset_id): resource_name = '//bigquery.googleapis.com/projects/{}/datasets/{}'\ .format(project_id, dataset_id) - entry = datacatalog.lookup_entry(linked_resource=resource_name) - print(entry.name) - return entry + return datacatalog.lookup_entry(linked_resource=resource_name) def lookup_bigquery_dataset_sql_resource(project_id, dataset_id): @@ -48,9 +46,7 @@ def lookup_bigquery_dataset_sql_resource(project_id, dataset_id): sql_resource = 'bigquery.dataset.`{}`.`{}`'.format(project_id, dataset_id) - entry = datacatalog.lookup_entry(sql_resource=sql_resource) - print(entry.name) - return entry + return datacatalog.lookup_entry(sql_resource=sql_resource) def lookup_bigquery_table(project_id, dataset_id, table_id): @@ -63,9 +59,7 @@ def lookup_bigquery_table(project_id, dataset_id, table_id): '/tables/{}'\ .format(project_id, dataset_id, table_id) - entry = datacatalog.lookup_entry(linked_resource=resource_name) - print(entry.name) - return entry + return datacatalog.lookup_entry(linked_resource=resource_name) def lookup_bigquery_table_sql_resource(project_id, dataset_id, table_id): @@ -79,9 +73,7 @@ def lookup_bigquery_table_sql_resource(project_id, dataset_id, table_id): sql_resource = 'bigquery.table.`{}`.`{}`.`{}`'.format( project_id, dataset_id, table_id) - entry = datacatalog.lookup_entry(sql_resource=sql_resource) - print(entry.name) - return entry + return datacatalog.lookup_entry(sql_resource=sql_resource) def lookup_pubsub_topic(project_id, topic_id): @@ -93,9 +85,7 @@ def lookup_pubsub_topic(project_id, topic_id): resource_name = '//pubsub.googleapis.com/projects/{}/topics/{}'\ .format(project_id, topic_id) - entry = datacatalog.lookup_entry(linked_resource=resource_name) - print(entry.name) - return entry + return datacatalog.lookup_entry(linked_resource=resource_name) def lookup_pubsub_topic_sql_resource(project_id, topic_id): @@ -108,9 +98,7 @@ def lookup_pubsub_topic_sql_resource(project_id, topic_id): sql_resource = 'pubsub.topic.`{}`.`{}`'.format(project_id, topic_id) - entry = datacatalog.lookup_entry(sql_resource=sql_resource) - print(entry.name) - return entry + return datacatalog.lookup_entry(sql_resource=sql_resource) if __name__ == '__main__': @@ -139,22 +127,19 @@ def lookup_pubsub_topic_sql_resource(project_id, topic_id): args = parser.parse_args() + entry = None + if args.command == 'lookup-bigquery-dataset': - methods = { - False: lookup_bigquery_dataset, - True: lookup_bigquery_dataset_sql_resource - } - methods[args.sql_resource](args.project_id, args.dataset_id) + lookup_method = lookup_bigquery_dataset_sql_resource \ + if args.sql_resource else lookup_bigquery_dataset + entry = lookup_method(args.project_id, args.dataset_id) elif args.command == 'lookup-bigquery-table': - methods = { - False: lookup_bigquery_table, - True: lookup_bigquery_table_sql_resource - } - methods[args.sql_resource]( - args.project_id, args.dataset_id, args.table_id) + lookup_method = lookup_bigquery_table_sql_resource \ + if args.sql_resource else lookup_bigquery_table + entry = lookup_method(args.project_id, args.dataset_id, args.table_id) elif args.command == 'lookup-pubsub-topic': - methods = { - False: lookup_pubsub_topic, - True: lookup_pubsub_topic_sql_resource - } - methods[args.sql_resource](args.project_id, args.topic_id) + lookup_method = lookup_pubsub_topic_sql_resource \ + if args.sql_resource else lookup_pubsub_topic + entry = lookup_method(args.project_id, args.topic_id) + + print(entry.name) From 531cbf022eefc98614a76effdeda6ce066ee207e Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Fri, 10 May 2019 18:06:01 -0300 Subject: [PATCH 12/14] Remove @eventually_consistent decorator --- datacatalog/cloud-client/lookup_entry_test.py | 39 ++++++------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/datacatalog/cloud-client/lookup_entry_test.py b/datacatalog/cloud-client/lookup_entry_test.py index b570ca9fca1f..f1336476966a 100644 --- a/datacatalog/cloud-client/lookup_entry_test.py +++ b/datacatalog/cloud-client/lookup_entry_test.py @@ -16,51 +16,36 @@ import os -from gcp_devrel.testing import eventually_consistent - import lookup_entry PROJECT = os.environ['GCLOUD_PROJECT'] -BIGQUERY_DATASET = os.environ['GCLOUD_BIGQUERY_DATASET'] -BIGQUERY_TABLE = os.environ['GCLOUD_BIGQUERY_TABLE'] -PUBSUB_TOPIC = os.environ['GCLOUD_PUBSUB_TOPIC'] +BIGQUERY_DATASET = os.environ['BIGQUERY_DATASET'] +BIGQUERY_TABLE = os.environ['BIGQUERY_TABLE'] +PUBSUB_TOPIC = os.environ['PUBSUB_TOPIC'] def test_lookup_bigquery_dataset(): - @eventually_consistent.call - def _(): - assert lookup_entry.lookup_bigquery_dataset(PROJECT, BIGQUERY_DATASET) + assert lookup_entry.lookup_bigquery_dataset(PROJECT, BIGQUERY_DATASET) def test_lookup_bigquery_dataset_sql_resource(): - @eventually_consistent.call - def _(): - assert lookup_entry.lookup_bigquery_dataset_sql_resource( - PROJECT, BIGQUERY_DATASET) + assert lookup_entry.lookup_bigquery_dataset_sql_resource( + PROJECT, BIGQUERY_DATASET) def test_lookup_bigquery_table(): - @eventually_consistent.call - def _(): - assert lookup_entry.lookup_bigquery_table( - PROJECT, BIGQUERY_DATASET, BIGQUERY_TABLE) + assert lookup_entry.lookup_bigquery_table( + PROJECT, BIGQUERY_DATASET, BIGQUERY_TABLE) def test_lookup_bigquery_table_sql_resource(): - @eventually_consistent.call - def _(): - assert lookup_entry.lookup_bigquery_table_sql_resource( - PROJECT, BIGQUERY_DATASET, BIGQUERY_TABLE) + assert lookup_entry.lookup_bigquery_table_sql_resource( + PROJECT, BIGQUERY_DATASET, BIGQUERY_TABLE) def test_lookup_pubsub_topic(): - @eventually_consistent.call - def _(): - assert lookup_entry.lookup_pubsub_topic(PROJECT, PUBSUB_TOPIC) + assert lookup_entry.lookup_pubsub_topic(PROJECT, PUBSUB_TOPIC) def test_lookup_pubsub_topic_sql_resource(): - @eventually_consistent.call - def _(): - assert lookup_entry.lookup_pubsub_topic_sql_resource( - PROJECT, PUBSUB_TOPIC) + assert lookup_entry.lookup_pubsub_topic_sql_resource(PROJECT, PUBSUB_TOPIC) From d42fbf2d0f7922d9c2576c34ce4f5213abefeca7 Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Fri, 10 May 2019 18:10:24 -0300 Subject: [PATCH 13/14] Improve command line interface * Removed the "lookup-" prefix from commands * Handle the --sql-resource optional argument by subparsers --- datacatalog/cloud-client/README.rst | 17 ++++++----------- datacatalog/cloud-client/lookup_entry.py | 21 +++++++++++++-------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/datacatalog/cloud-client/README.rst b/datacatalog/cloud-client/README.rst index e4ed58c67258..ba214203558c 100644 --- a/datacatalog/cloud-client/README.rst +++ b/datacatalog/cloud-client/README.rst @@ -76,10 +76,9 @@ To run this sample: $ python lookup_entry.py - usage: lookup_entry.py [-h] [--sql-resource] + usage: lookup_entry.py [-h] project_id - {lookup-bigquery-dataset,lookup-bigquery-table,lookup-pubsub-topic} - ... + {bigquery-dataset,bigquery-table,pubsub-topic} ... This application demonstrates how to perform basic operations on entries with the Cloud Data Catalog API. @@ -89,20 +88,16 @@ To run this sample: positional arguments: project_id Your Google Cloud project ID - {lookup-bigquery-dataset,lookup-bigquery-table,lookup-pubsub-topic} - lookup-bigquery-dataset - Retrieves Data Catalog entry for the given BigQuery + {bigquery-dataset,bigquery-table,pubsub-topic} + bigquery-dataset Retrieves Data Catalog entry for the given BigQuery Dataset. - lookup-bigquery-table - Retrieves Data Catalog entry for the given BigQuery + bigquery-table Retrieves Data Catalog entry for the given BigQuery Table. - lookup-pubsub-topic - Retrieves Data Catalog entry for the given Pub/Sub + pubsub-topic Retrieves Data Catalog entry for the given Pub/Sub Topic. optional arguments: -h, --help show this help message and exit - --sql-resource Perform lookup by SQL Resource diff --git a/datacatalog/cloud-client/lookup_entry.py b/datacatalog/cloud-client/lookup_entry.py index 672273274749..a186408fe560 100644 --- a/datacatalog/cloud-client/lookup_entry.py +++ b/datacatalog/cloud-client/lookup_entry.py @@ -106,38 +106,43 @@ def lookup_pubsub_topic_sql_resource(project_id, topic_id): description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) + parser.add_argument('project_id', help='Your Google Cloud project ID') - parser.add_argument('--sql-resource', action='store_true', - help='Perform lookup by SQL Resource') subparsers = parser.add_subparsers(dest='command') bigquery_dataset_parser = subparsers.add_parser( - 'lookup-bigquery-dataset', help=lookup_bigquery_dataset.__doc__) + 'bigquery-dataset', help=lookup_bigquery_dataset.__doc__) bigquery_dataset_parser.add_argument('dataset_id') + bigquery_dataset_parser.add_argument('--sql-resource', action='store_true', + help='Perform lookup by SQL Resource') bigquery_table_parser = subparsers.add_parser( - 'lookup-bigquery-table', help=lookup_bigquery_table.__doc__) + 'bigquery-table', help=lookup_bigquery_table.__doc__) bigquery_table_parser.add_argument('dataset_id') bigquery_table_parser.add_argument('table_id') + bigquery_table_parser.add_argument('--sql-resource', action='store_true', + help='Perform lookup by SQL Resource') pubsub_topic_parser = subparsers.add_parser( - 'lookup-pubsub-topic', help=lookup_pubsub_topic.__doc__) + 'pubsub-topic', help=lookup_pubsub_topic.__doc__) pubsub_topic_parser.add_argument('topic_id') + pubsub_topic_parser.add_argument('--sql-resource', action='store_true', + help='Perform lookup by SQL Resource') args = parser.parse_args() entry = None - if args.command == 'lookup-bigquery-dataset': + if args.command == 'bigquery-dataset': lookup_method = lookup_bigquery_dataset_sql_resource \ if args.sql_resource else lookup_bigquery_dataset entry = lookup_method(args.project_id, args.dataset_id) - elif args.command == 'lookup-bigquery-table': + elif args.command == 'bigquery-table': lookup_method = lookup_bigquery_table_sql_resource \ if args.sql_resource else lookup_bigquery_table entry = lookup_method(args.project_id, args.dataset_id, args.table_id) - elif args.command == 'lookup-pubsub-topic': + elif args.command == 'pubsub-topic': lookup_method = lookup_pubsub_topic_sql_resource \ if args.sql_resource else lookup_pubsub_topic entry = lookup_method(args.project_id, args.topic_id) From 10450b2313d38f995179179aaa35d8a6091bdbe9 Mon Sep 17 00:00:00 2001 From: Ricardo Mendes Date: Fri, 10 May 2019 18:20:24 -0300 Subject: [PATCH 14/14] Refer to GCP public assets in tests --- datacatalog/cloud-client/lookup_entry_test.py | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/datacatalog/cloud-client/lookup_entry_test.py b/datacatalog/cloud-client/lookup_entry_test.py index f1336476966a..2030cb072197 100644 --- a/datacatalog/cloud-client/lookup_entry_test.py +++ b/datacatalog/cloud-client/lookup_entry_test.py @@ -14,38 +14,40 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - import lookup_entry -PROJECT = os.environ['GCLOUD_PROJECT'] -BIGQUERY_DATASET = os.environ['BIGQUERY_DATASET'] -BIGQUERY_TABLE = os.environ['BIGQUERY_TABLE'] -PUBSUB_TOPIC = os.environ['PUBSUB_TOPIC'] +BIGQUERY_PROJECT = 'bigquery-public-data' +BIGQUERY_DATASET = 'new_york_taxi_trips' +BIGQUERY_TABLE = 'taxi_zone_geom' + +PUBSUB_PROJECT = 'pubsub-public-data' +PUBSUB_TOPIC = 'taxirides-realtime' def test_lookup_bigquery_dataset(): - assert lookup_entry.lookup_bigquery_dataset(PROJECT, BIGQUERY_DATASET) + assert lookup_entry.lookup_bigquery_dataset( + BIGQUERY_PROJECT, BIGQUERY_DATASET) def test_lookup_bigquery_dataset_sql_resource(): assert lookup_entry.lookup_bigquery_dataset_sql_resource( - PROJECT, BIGQUERY_DATASET) + BIGQUERY_PROJECT, BIGQUERY_DATASET) def test_lookup_bigquery_table(): assert lookup_entry.lookup_bigquery_table( - PROJECT, BIGQUERY_DATASET, BIGQUERY_TABLE) + BIGQUERY_PROJECT, BIGQUERY_DATASET, BIGQUERY_TABLE) def test_lookup_bigquery_table_sql_resource(): assert lookup_entry.lookup_bigquery_table_sql_resource( - PROJECT, BIGQUERY_DATASET, BIGQUERY_TABLE) + BIGQUERY_PROJECT, BIGQUERY_DATASET, BIGQUERY_TABLE) def test_lookup_pubsub_topic(): - assert lookup_entry.lookup_pubsub_topic(PROJECT, PUBSUB_TOPIC) + assert lookup_entry.lookup_pubsub_topic(PUBSUB_PROJECT, PUBSUB_TOPIC) def test_lookup_pubsub_topic_sql_resource(): - assert lookup_entry.lookup_pubsub_topic_sql_resource(PROJECT, PUBSUB_TOPIC) + assert lookup_entry.lookup_pubsub_topic_sql_resource( + PUBSUB_PROJECT, PUBSUB_TOPIC)