From dfd5ecf70d30a768779286fe365409d43c5da1dd Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 19 May 2017 14:52:10 -0400 Subject: [PATCH 1/4] Move '_catch_remap_gax_error' to 'core.exceptions'. Toward #3175. --- core/google/cloud/exceptions.py | 58 +++++++++++++++++++-- core/setup.py | 1 + core/tests/unit/test_exceptions.py | 65 ++++++++++++++++++++++++ datastore/google/cloud/datastore/_gax.py | 49 +----------------- datastore/tests/unit/test__gax.py | 63 ----------------------- 5 files changed, 121 insertions(+), 115 deletions(-) diff --git a/core/google/cloud/exceptions.py b/core/google/cloud/exceptions.py index ab0ede688ef3..9b573a7ea4a6 100644 --- a/core/google/cloud/exceptions.py +++ b/core/google/cloud/exceptions.py @@ -20,18 +20,27 @@ # Avoid the grpc and google.cloud.grpc collision. from __future__ import absolute_import +import contextlib import copy import json -import six - -from google.cloud._helpers import _to_bytes +import sys -_HTTP_CODE_TO_EXCEPTION = {} # populated at end of module +import six try: + from google.gax.errors import GaxError + from google.gax.grpc import exc_to_code + from grpc import StatusCode from grpc._channel import _Rendezvous except ImportError: # pragma: NO COVER + _HAVE_GRPC = False _Rendezvous = None +else: + _HAVE_GRPC = True + +from google.cloud._helpers import _to_bytes + +_HTTP_CODE_TO_EXCEPTION = {} # populated at end of module # pylint: disable=invalid-name @@ -250,3 +259,44 @@ def _walk_subclasses(klass): code = getattr(_eklass, 'code', None) if code is not None: _HTTP_CODE_TO_EXCEPTION[code] = _eklass + +_GRPC_ERROR_MAPPING = { + StatusCode.UNKNOWN: InternalServerError, + StatusCode.INVALID_ARGUMENT: BadRequest, + StatusCode.DEADLINE_EXCEEDED: GatewayTimeout, + StatusCode.NOT_FOUND: NotFound, + StatusCode.ALREADY_EXISTS: Conflict, + StatusCode.PERMISSION_DENIED: Forbidden, + StatusCode.UNAUTHENTICATED: Unauthorized, + StatusCode.RESOURCE_EXHAUSTED: TooManyRequests, + StatusCode.FAILED_PRECONDITION: PreconditionFailed, + StatusCode.ABORTED: Conflict, + StatusCode.OUT_OF_RANGE: BadRequest, + StatusCode.UNIMPLEMENTED: MethodNotImplemented, + StatusCode.INTERNAL: InternalServerError, + StatusCode.UNAVAILABLE: ServiceUnavailable, + StatusCode.DATA_LOSS: InternalServerError, +} + + +@contextlib.contextmanager +def _catch_remap_gax_error(): + """Remap GAX exceptions that happen in context. + + .. _code.proto: https://github.com/googleapis/googleapis/blob/\ + master/google/rpc/code.proto + + Remaps gRPC exceptions to the classes defined in + :mod:`~google.cloud.exceptions` (according to the description + in `code.proto`_). + """ + try: + yield + except GaxError as exc: + error_code = exc_to_code(exc.cause) + error_class = _GRPC_ERROR_MAPPING.get(error_code) + if error_class is None: + raise + else: + new_exc = error_class(exc.cause.details()) + six.reraise(error_class, new_exc, sys.exc_info()[2]) diff --git a/core/setup.py b/core/setup.py index 3dfa13ef5284..372fa725868e 100644 --- a/core/setup.py +++ b/core/setup.py @@ -56,6 +56,7 @@ 'protobuf >= 3.0.0', 'google-auth >= 0.4.0, < 2.0.0dev', 'google-auth-httplib2', + 'google-gax>=0.15.7, <0.16dev', 'six', ] diff --git a/core/tests/unit/test_exceptions.py b/core/tests/unit/test_exceptions.py index b3488296eff4..5e79a7463b6d 100644 --- a/core/tests/unit/test_exceptions.py +++ b/core/tests/unit/test_exceptions.py @@ -14,6 +14,8 @@ import unittest +from google.cloud.exceptions import _HAVE_GRPC + class Test_GoogleCloudError(unittest.TestCase): @@ -145,6 +147,69 @@ def test_without_use_json(self): self.assertEqual(list(exception.errors), []) +@unittest.skipUnless(_HAVE_GRPC, 'No gRPC') +class Test__catch_remap_gax_error(unittest.TestCase): + + def _call_fut(self): + from google.cloud.exceptions import _catch_remap_gax_error + + return _catch_remap_gax_error() + + @staticmethod + def _fake_method(exc, result=None): + if exc is None: + return result + else: + raise exc + + @staticmethod + def _make_rendezvous(status_code, details): + from grpc._channel import _RPCState + from google.cloud.exceptions import GrpcRendezvous + + exc_state = _RPCState((), None, None, status_code, details) + return GrpcRendezvous(exc_state, None, None, None) + + def test_success(self): + expected = object() + with self._call_fut(): + result = self._fake_method(None, expected) + self.assertIs(result, expected) + + def test_non_grpc_err(self): + exc = RuntimeError('Not a gRPC error') + with self.assertRaises(RuntimeError): + with self._call_fut(): + self._fake_method(exc) + + def test_gax_error(self): + from google.gax.errors import GaxError + from grpc import StatusCode + from google.cloud.exceptions import Forbidden + + # First, create low-level GrpcRendezvous exception. + details = 'Some error details.' + cause = self._make_rendezvous(StatusCode.PERMISSION_DENIED, details) + # Then put it into a high-level GaxError. + msg = 'GAX Error content.' + exc = GaxError(msg, cause=cause) + + with self.assertRaises(Forbidden): + with self._call_fut(): + self._fake_method(exc) + + def test_gax_error_not_mapped(self): + from google.gax.errors import GaxError + from grpc import StatusCode + + cause = self._make_rendezvous(StatusCode.CANCELLED, None) + exc = GaxError(None, cause=cause) + + with self.assertRaises(GaxError): + with self._call_fut(): + self._fake_method(exc) + + class _Response(object): def __init__(self, status): self.status = status diff --git a/datastore/google/cloud/datastore/_gax.py b/datastore/google/cloud/datastore/_gax.py index e1d0e57a7737..f480da3ac7f4 100644 --- a/datastore/google/cloud/datastore/_gax.py +++ b/datastore/google/cloud/datastore/_gax.py @@ -14,21 +14,14 @@ """Helpers for making API requests via GAX / gRPC.""" - -import contextlib -import sys - from google.cloud.gapic.datastore.v1 import datastore_client -from google.gax.errors import GaxError -from google.gax.grpc import exc_to_code from google.gax.utils import metrics from grpc import insecure_channel -from grpc import StatusCode import six from google.cloud._helpers import make_secure_channel from google.cloud._http import DEFAULT_USER_AGENT -from google.cloud import exceptions +from google.cloud.exceptions import _catch_remap_gax_error from google.cloud.datastore import __version__ @@ -40,46 +33,6 @@ _GRPC_EXTRA_OPTIONS = ( ('x-goog-api-client', _HEADER_STR), ) -_GRPC_ERROR_MAPPING = { - StatusCode.UNKNOWN: exceptions.InternalServerError, - StatusCode.INVALID_ARGUMENT: exceptions.BadRequest, - StatusCode.DEADLINE_EXCEEDED: exceptions.GatewayTimeout, - StatusCode.NOT_FOUND: exceptions.NotFound, - StatusCode.ALREADY_EXISTS: exceptions.Conflict, - StatusCode.PERMISSION_DENIED: exceptions.Forbidden, - StatusCode.UNAUTHENTICATED: exceptions.Unauthorized, - StatusCode.RESOURCE_EXHAUSTED: exceptions.TooManyRequests, - StatusCode.FAILED_PRECONDITION: exceptions.PreconditionFailed, - StatusCode.ABORTED: exceptions.Conflict, - StatusCode.OUT_OF_RANGE: exceptions.BadRequest, - StatusCode.UNIMPLEMENTED: exceptions.MethodNotImplemented, - StatusCode.INTERNAL: exceptions.InternalServerError, - StatusCode.UNAVAILABLE: exceptions.ServiceUnavailable, - StatusCode.DATA_LOSS: exceptions.InternalServerError, -} - - -@contextlib.contextmanager -def _catch_remap_gax_error(): - """Remap GAX exceptions that happen in context. - - .. _code.proto: https://github.com/googleapis/googleapis/blob/\ - master/google/rpc/code.proto - - Remaps gRPC exceptions to the classes defined in - :mod:`~google.cloud.exceptions` (according to the description - in `code.proto`_). - """ - try: - yield - except GaxError as exc: - error_code = exc_to_code(exc.cause) - error_class = _GRPC_ERROR_MAPPING.get(error_code) - if error_class is None: - raise - else: - new_exc = error_class(exc.cause.details()) - six.reraise(error_class, new_exc, sys.exc_info()[2]) class GAPICDatastoreAPI(datastore_client.DatastoreClient): diff --git a/datastore/tests/unit/test__gax.py b/datastore/tests/unit/test__gax.py index 2dd7f8d0e3d5..bc7560be4491 100644 --- a/datastore/tests/unit/test__gax.py +++ b/datastore/tests/unit/test__gax.py @@ -19,69 +19,6 @@ from google.cloud.datastore.client import _HAVE_GRPC -@unittest.skipUnless(_HAVE_GRPC, 'No gRPC') -class Test__catch_remap_gax_error(unittest.TestCase): - - def _call_fut(self): - from google.cloud.datastore._gax import _catch_remap_gax_error - - return _catch_remap_gax_error() - - @staticmethod - def _fake_method(exc, result=None): - if exc is None: - return result - else: - raise exc - - @staticmethod - def _make_rendezvous(status_code, details): - from grpc._channel import _RPCState - from google.cloud.exceptions import GrpcRendezvous - - exc_state = _RPCState((), None, None, status_code, details) - return GrpcRendezvous(exc_state, None, None, None) - - def test_success(self): - expected = object() - with self._call_fut(): - result = self._fake_method(None, expected) - self.assertIs(result, expected) - - def test_non_grpc_err(self): - exc = RuntimeError('Not a gRPC error') - with self.assertRaises(RuntimeError): - with self._call_fut(): - self._fake_method(exc) - - def test_gax_error(self): - from google.gax.errors import GaxError - from grpc import StatusCode - from google.cloud.exceptions import Forbidden - - # First, create low-level GrpcRendezvous exception. - details = 'Some error details.' - cause = self._make_rendezvous(StatusCode.PERMISSION_DENIED, details) - # Then put it into a high-level GaxError. - msg = 'GAX Error content.' - exc = GaxError(msg, cause=cause) - - with self.assertRaises(Forbidden): - with self._call_fut(): - self._fake_method(exc) - - def test_gax_error_not_mapped(self): - from google.gax.errors import GaxError - from grpc import StatusCode - - cause = self._make_rendezvous(StatusCode.CANCELLED, None) - exc = GaxError(None, cause=cause) - - with self.assertRaises(GaxError): - with self._call_fut(): - self._fake_method(exc) - - @unittest.skipUnless(_HAVE_GRPC, 'No gRPC') class TestGAPICDatastoreAPI(unittest.TestCase): From 69a8fb7e4ccac4ee70c0e2cc2e84d6269a789025 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 19 May 2017 17:16:42 -0400 Subject: [PATCH 2/4] Remove conditional imports of GAX/gRPC-related modules. We depend on explicitly in core. --- core/google/cloud/exceptions.py | 15 +++++---------- core/tests/unit/test_exceptions.py | 3 --- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/core/google/cloud/exceptions.py b/core/google/cloud/exceptions.py index 9b573a7ea4a6..55310f11f9f2 100644 --- a/core/google/cloud/exceptions.py +++ b/core/google/cloud/exceptions.py @@ -27,16 +27,10 @@ import six -try: - from google.gax.errors import GaxError - from google.gax.grpc import exc_to_code - from grpc import StatusCode - from grpc._channel import _Rendezvous -except ImportError: # pragma: NO COVER - _HAVE_GRPC = False - _Rendezvous = None -else: - _HAVE_GRPC = True +from google.gax.errors import GaxError +from google.gax.grpc import exc_to_code +from grpc import StatusCode +from grpc._channel import _Rendezvous from google.cloud._helpers import _to_bytes @@ -260,6 +254,7 @@ def _walk_subclasses(klass): if code is not None: _HTTP_CODE_TO_EXCEPTION[code] = _eklass + _GRPC_ERROR_MAPPING = { StatusCode.UNKNOWN: InternalServerError, StatusCode.INVALID_ARGUMENT: BadRequest, diff --git a/core/tests/unit/test_exceptions.py b/core/tests/unit/test_exceptions.py index 5e79a7463b6d..73d3637e0995 100644 --- a/core/tests/unit/test_exceptions.py +++ b/core/tests/unit/test_exceptions.py @@ -14,8 +14,6 @@ import unittest -from google.cloud.exceptions import _HAVE_GRPC - class Test_GoogleCloudError(unittest.TestCase): @@ -147,7 +145,6 @@ def test_without_use_json(self): self.assertEqual(list(exception.errors), []) -@unittest.skipUnless(_HAVE_GRPC, 'No gRPC') class Test__catch_remap_gax_error(unittest.TestCase): def _call_fut(self): From 2eb6fcc257da914e97cb3830945e5abb48a8f8a0 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 19 May 2017 17:18:43 -0400 Subject: [PATCH 3/4] Remove explicit 'google-gax' dep in non-core subprojects. 'core' now depends on it explicitly. --- bigtable/setup.py | 1 - datastore/setup.py | 1 - videointelligence/setup.py | 1 - 3 files changed, 3 deletions(-) diff --git a/bigtable/setup.py b/bigtable/setup.py index 212feda21758..35af084df33f 100644 --- a/bigtable/setup.py +++ b/bigtable/setup.py @@ -52,7 +52,6 @@ REQUIREMENTS = [ 'google-cloud-core >= 0.24.0, < 0.25dev', - 'google-gax>=0.15.7, <0.16dev', ] setup( diff --git a/datastore/setup.py b/datastore/setup.py index cc82802315ae..e94e3d508d0a 100644 --- a/datastore/setup.py +++ b/datastore/setup.py @@ -52,7 +52,6 @@ REQUIREMENTS = [ 'google-cloud-core >= 0.24.0, < 0.25dev', - 'google-gax>=0.15.7, <0.16dev', 'gapic-google-cloud-datastore-v1 >= 0.15.0, < 0.16dev', ] diff --git a/videointelligence/setup.py b/videointelligence/setup.py index 9325a8ffb09a..c6c688e313f3 100644 --- a/videointelligence/setup.py +++ b/videointelligence/setup.py @@ -43,7 +43,6 @@ packages=find_packages(exclude=('tests*',)), install_requires=( 'googleapis-common-protos >= 1.5.2, < 2.0dev', - 'google-gax >= 0.15.12, < 0.16dev', 'six >= 1.10.0', ), url='https://github.com/GoogleCloudPlatform/google-cloud-python', From 457c207b30f3eb3f22d939e26ea47808667e51d5 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 5 Jun 2017 14:23:54 -0400 Subject: [PATCH 4/4] Move '_catch_remap_gax_error()' before end-of-module constants. Addresses: https://github.com/GoogleCloudPlatform/google-cloud-python/pull/3444#discussion_r120169860 --- core/google/cloud/exceptions.py | 46 ++++++++++++++++----------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/core/google/cloud/exceptions.py b/core/google/cloud/exceptions.py index 55310f11f9f2..37e32e3092a0 100644 --- a/core/google/cloud/exceptions.py +++ b/core/google/cloud/exceptions.py @@ -248,6 +248,29 @@ def _walk_subclasses(klass): yield subsub +@contextlib.contextmanager +def _catch_remap_gax_error(): + """Remap GAX exceptions that happen in context. + + .. _code.proto: https://github.com/googleapis/googleapis/blob/\ + master/google/rpc/code.proto + + Remaps gRPC exceptions to the classes defined in + :mod:`~google.cloud.exceptions` (according to the description + in `code.proto`_). + """ + try: + yield + except GaxError as exc: + error_code = exc_to_code(exc.cause) + error_class = _GRPC_ERROR_MAPPING.get(error_code) + if error_class is None: + raise + else: + new_exc = error_class(exc.cause.details()) + six.reraise(error_class, new_exc, sys.exc_info()[2]) + + # Build the code->exception class mapping. for _eklass in _walk_subclasses(GoogleCloudError): code = getattr(_eklass, 'code', None) @@ -272,26 +295,3 @@ def _walk_subclasses(klass): StatusCode.UNAVAILABLE: ServiceUnavailable, StatusCode.DATA_LOSS: InternalServerError, } - - -@contextlib.contextmanager -def _catch_remap_gax_error(): - """Remap GAX exceptions that happen in context. - - .. _code.proto: https://github.com/googleapis/googleapis/blob/\ - master/google/rpc/code.proto - - Remaps gRPC exceptions to the classes defined in - :mod:`~google.cloud.exceptions` (according to the description - in `code.proto`_). - """ - try: - yield - except GaxError as exc: - error_code = exc_to_code(exc.cause) - error_class = _GRPC_ERROR_MAPPING.get(error_code) - if error_class is None: - raise - else: - new_exc = error_class(exc.cause.details()) - six.reraise(error_class, new_exc, sys.exc_info()[2])