Switch from oauth2client to google-auth #2726

theacodes · 2016-11-11T22:21:49Z

Removes all use of oauth2client from every package and tests.
Updates core to use google-auth's default credentials, project ID, and scoping logic.
Updates bigtable to use google-auth's scoping logic.

dhermes

Really great changes! Why do we don't merge this?

core/google/cloud/_helpers.py

    import grpc
 except ImportError:  # pragma: NO COVER
    grpc = None
 import six
 from six.moves import http_client
-from six.moves import configparser
-
-from google.cloud.environment_vars import PROJECT


core/google/cloud/_helpers.py

 def _determine_default_project(project=None):
    """Determine default project ID explicitly or implicitly as fall-back.

    In implicit case, supports three environments. In order of precedence, the
    implicit environments are:

-    * GOOGLE_CLOUD_PROJECT environment variable
+    * GOOGLE_CLOUD_PROJECT and GCLOUD_PROJECT environment variable


core/google/cloud/_helpers.py

-    if project is None:
-        project = _compute_engine_id()
-
+        _, project = google.auth.default()


core/google/cloud/client.py

            json_credentials_path)
        kwargs['credentials'] = credentials
        return cls(*args, **kwargs)

-    @classmethod
-    def from_service_account_p12(cls, client_email, private_key_path,


core/google/cloud/credentials.py

@@ -84,7 +85,8 @@ def get_credentials():
    :returns: A new credentials instance corresponding to the implicit
              environment.
    """
-    return client.GoogleCredentials.get_application_default()
+    credentials, _ = google.auth.default()


core/tox.ini

@@ -4,6 +4,7 @@ envlist =

 [testing]
 deps =
+    mock


core/unit_tests/test__helpers.py

-            returned_project = self._call_fut(project)
-
-        return returned_project, _callers
+    def test(self):


core/unit_tests/test__helpers.py

+    def test(self):
+        with mock.patch('google.auth.default', autospec=True) as default:
+            default.return_value = (
+                mock.sentinel.credentials, mock.sentinel.project)


core/unit_tests/test__helpers.py

+            project = self._call_fut(mock.sentinel.project)
+
+        self.assertEqual(project, mock.sentinel.project)
+        self.assertFalse(default.called)


core/unit_tests/test__http.py

        conn = self._make_one(credentials)
-        self.assertIs(conn.credentials, credentials)
+        self.assertIs(conn.credentials, mock.sentinel.credentials)


core/google/cloud/_helpers.py

-try:
+import google_auth_httplib2
+
+try:  # pragma: NO COVER


core/unit_tests/test__helpers.py

-        with _Monkey(MUT, grpc=grpc_mod,
-                     MetadataPlugin=mock_plugin):
+
+        grpc_patch = mock.patch.object(MUT, 'grpc', new=grpc_mod)


core/unit_tests/test__helpers.py

+        request_patch = mock.patch('google_auth_httplib2.Request')
+        plugin_patch = mock.patch.object(
+            MUT, 'AuthMetadataPlugin', create=True)
+        with grpc_patch, request_patch as request_mock, plugin_patch as plugin:


core/google/cloud/credentials.py

@@ -30,58 +30,9 @@
 def get_credentials():
    """Gets credentials implicitly from the current environment.

-    .. note::


core/google/cloud/_helpers.py

    import grpc
-except ImportError:  # pragma: NO COVER
+    from google.auth.transport.grpc import AuthMetadataPlugin


bigquery/unit_tests/test_client.py

-    def create_scoped(self, scope):
-        self._scopes = scope
-        return self
+    pass


core/google/cloud/_helpers.py

-      ``$ gcloud beta auth application-default login``
-    * Google App Engine application ID
-    * Google Compute Engine project ID (from metadata server)
+    See :func:`google.auth.default` for details on how the default project


core/google/cloud/_helpers.py

-    custom_metadata_plugin = MetadataPlugin(credentials)
+    http = httplib2.Http()
+    custom_metadata_plugin = AuthMetadataPlugin(
+        credentials, google_auth_httplib2.Request(http=http))


core/google/cloud/streaming/http_wrapper.py

@@ -90,7 +90,7 @@ def _httplib2_debug_level(http_request, level, http=None):
    old_level = httplib2.debuglevel
    http_levels = {}
    httplib2.debuglevel = level
-    if http is not None:
+    if http is not None and hasattr(http, 'connections'):


core/setup.py

@@ -52,8 +52,9 @@
 REQUIREMENTS = [
    'httplib2 >= 0.9.1',
    'googleapis-common-protos >= 1.3.4',
-    'oauth2client >= 3.0.0, < 4.0.0dev',


core/unit_tests/test__helpers.py

+            project = self._call_fut(mock.sentinel.project)
+
+        self.assertEqual(project, mock.sentinel.project)
+        self.assertFalse(default.called)


theacodes

@dhermes I think I addressed all of the comments - let me know if there's anything left.

Do not merge remains until I get all system tests to pass.

bigquery/unit_tests/test_client.py

-    def create_scoped(self, scope):
-        self._scopes = scope
-        return self
+    pass


core/google/cloud/_helpers.py

-      ``$ gcloud beta auth application-default login``
-    * Google App Engine application ID
-    * Google Compute Engine project ID (from metadata server)
+    See :func:`google.auth.default` for details on how the default project


core/google/cloud/streaming/http_wrapper.py

@@ -90,7 +90,7 @@ def _httplib2_debug_level(http_request, level, http=None):
    old_level = httplib2.debuglevel
    http_levels = {}
    httplib2.debuglevel = level
-    if http is not None:
+    if http is not None and hasattr(http, 'connections'):


theacodes · 2016-12-02T20:57:12Z

Unit tests all pass on Circle. Going to try to run through the system tests locally now.

theacodes · 2016-12-02T21:06:12Z

bigquery, bigtable, and datastore are passing.

theacodes · 2016-12-02T21:23:08Z

language, logging, monitoring, pubsub, speech, and storage are passing.

Translate seems to be failing for unrelated reasons?

======================================================================
ERROR: test_translate (translate.TestTranslate)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/Users/jonwayne/workspace/gcloud-python/system_tests/translate.py", line 59, in test_translate
    values, target_language='de', model=translate.NMT)
  File "/Users/jonwayne/workspace/gcloud-python/.tox/system-tests/lib/python2.7/site-packages/google/cloud/translate/client.py", line 246, in translate
    method='GET', path='', query_params=query_params)
  File "/Users/jonwayne/workspace/gcloud-python/.tox/system-tests/lib/python2.7/site-packages/google/cloud/_http.py", line 335, in api_request
    error_info=method + ' ' + url)
BadRequest: 400 Invalid JSON payload received. Unknown name "model": Cannot bind 'nmt'. Field 'model' could not be found in request message. (GET https://translation.googleapis.com/language/translate/v2?target=de&q=hvala+ti&q=dankon&q=Me+llamo+Jeff&q=My+name+is+Jeff&model=nmt)

That's everything. All seems good?

dhermes · 2016-12-02T22:27:29Z

@jonparrott Your translate failure may be a rebase issue? Or an old tox env?

dhermes

Approving with malice

theacodes · 2016-12-02T23:01:54Z

Merging with ill intent.

…kground-update) There is no specific reason to fix version and to manage this dependency differently (get last version), and it solved the issue below: for recent image push I have this error with gcloud 0.21.0: Traceback (most recent call last): File "./src/orchestrator/launch_backgroundupdate.py", line 8, in <module> init_gcloud_log(GCLOUD_PROJECT, u'background_update', IS_TEST_ENV) File "/usr/local/lib/python2.7/site-packages/common/log.py", line 14, in init_gcloud_log handler = CloudLoggingHandler(client, logger_name) File "/usr/local/lib/python2.7/site-packages/google/cloud/logging/handlers/handlers.py", line 80, in __init__ self.transport = transport(client, name) File "/usr/local/lib/python2.7/site-packages/google/cloud/logging/handlers/transports/background_thread.py", line 154, in __init__ http = client._connection.credentials.authorize(http) AttributeError: 'Credentials' object has no attribute 'authorize' It seems related to change: googleapis/google-cloud-python#2726 Upgrade to 0.21.1 fix the issue.

* Removes all use of oauth2client from every package and tests. * Updates core to use google-auth's default credentials, project ID, and scoping logic. * Updates bigtable to use google-auth's scoping logic.

theacodes added the do not merge Indicates a pull request not ready for merge, due to either quality or timing. label Nov 11, 2016

theacodes assigned dhermes Nov 11, 2016

googlebot added the cla: yes This human has signed the Contributor License Agreement. label Nov 11, 2016

theacodes force-pushed the google-auth branch from 4563140 to 2581686 Compare November 11, 2016 22:25

dhermes reviewed Nov 11, 2016

View reviewed changes

dhermes reviewed Nov 12, 2016

View reviewed changes

core/google/cloud/_helpers.py

import grpc

except ImportError: # pragma: NO COVER

from google.auth.transport.grpc import AuthMetadataPlugin

This comment was marked as spam.

Sign in to view

This comment was marked as spam.

Sign in to view

This comment was marked as spam.

Sign in to view

Jon Wayne Parrott added 5 commits December 1, 2016 14:31

Switch core to google-auth.

a6d2846

Update gRPC helpers

bb487ee

Address review comments

eae7799

Remove references to oauth2client

3b908c8

Get storage system tests to pass

46f83da

theacodes force-pushed the google-auth branch from 4820062 to 46f83da Compare December 1, 2016 22:46

Jon Wayne Parrott added 10 commits December 1, 2016 14:46

Swap NO COVER location

bf09a31

Update to latest google-auth

ddb75c8

Update docstring to point to google auth.

4ef18d4

Fix bigquery test coverage

464460b

Fix datastore tests

86fb2ab

Fix language tests

2feb33c

Fix logging tests

91bfa18

Fix monitoring tests

1f4977d

Fix pubsub tests

5b27859

Fix resource manager tests

e7f29da

dhermes reviewed Dec 2, 2016

View reviewed changes

Jon Wayne Parrott added 5 commits December 2, 2016 10:45

Fix runtimeconfig tests

2b12b27

Fix speech samples

7d47bdf

Fix storage tests

f8929f4

Fix vision tests

bf03afc

Remove all _Credentials classes

dba6136

Use getattr over hasattr

b63a67f

theacodes commented Dec 2, 2016

View reviewed changes

Update bigtable to use google-auth

0b5667a

theacodes changed the title ~~[DO NOT MERGE] Switch core to google-auth.~~ Switch core to google-auth. Dec 2, 2016

theacodes removed the do not merge Indicates a pull request not ready for merge, due to either quality or timing. label Dec 2, 2016

theacodes changed the title ~~Switch core to google-auth.~~ Switch from oauth2client to google-auth Dec 2, 2016

Jon Wayne Parrott added 2 commits December 2, 2016 14:34

Update minimum google-auth version

4f0feeb

Use credentials.signer_email

f32d639

dhermes approved these changes Dec 2, 2016

View reviewed changes

theacodes merged commit 0581c28 into googleapis:master Dec 2, 2016

theacodes deleted the google-auth branch December 2, 2016 23:02

theacodes mentioned this pull request Dec 2, 2016

Drop oauth2client in favor of google-auth #2639

Closed

This was referenced Dec 5, 2016

Cutting new releases for BigQuery, Language and Core. #2815

Merged

Update versions for mega-release. #2846

Merged

yebrahim mentioned this pull request Sep 5, 2017

Bumping google.cloud from 0.19 to 0.27 googledatalab/pydatalab#483

Closed

theacodes unassigned dhermes Sep 28, 2018

Switch from oauth2client to google-auth #2726

Switch from oauth2client to google-auth #2726

Conversation

theacodes commented Nov 11, 2016 • edited Loading

dhermes left a comment

Choose a reason for hiding this comment

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

theacodes left a comment

Choose a reason for hiding this comment

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

theacodes commented Dec 2, 2016

theacodes commented Dec 2, 2016

theacodes commented Dec 2, 2016

dhermes commented Dec 2, 2016

dhermes left a comment

Choose a reason for hiding this comment

theacodes commented Dec 2, 2016

theacodes commented Nov 11, 2016 •

edited

Loading