From d82a2bde39a4c4d26102bfc8cf10e5716d9ef3ce Mon Sep 17 00:00:00 2001 From: Jacob Geiger Date: Fri, 5 Aug 2016 15:40:51 -0700 Subject: [PATCH] Add support for Python 3 (#120) * Add support for Python 3 Add testing environments for Python 3.4, 3.5. Modify custom iterator objects to be Python 3-compatible. --- .travis.yml | 8 ++++-- google/gax/__init__.py | 12 ++++++-- google/gax/api_callable.py | 6 ++-- google/gax/bundling.py | 6 ++-- requirements.txt | 3 +- setup.py | 3 +- test/test_api_callable.py | 27 +++++++++--------- test/test_bundling.py | 56 +++++++++++++++++++------------------- test/test_grpc.py | 6 ++-- tox.ini | 8 ++++-- 10 files changed, 75 insertions(+), 60 deletions(-) diff --git a/.travis.yml b/.travis.yml index d3bd0bf..a87fcd6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,11 @@ sudo: false env: - - BREW_HOME=$HOME/.linuxbrew + - BREW_HOME=$HOME/.linuxbrew PATH=$BREW_HOME/bin:$PATH # Protobuf is very expensive to install, so we cache it between builds -before_install: ./install-protobuf3.sh +before_install: + - pip install --upgrade pip + - ./install-protobuf3.sh cache: directories: - $BREW_HOME @@ -11,6 +13,8 @@ cache: language: python python: - "2.7" + - "3.4" + - "3.5" install: pip install codecov tox-travis script: tox after_success: diff --git a/google/gax/__init__.py b/google/gax/__init__.py index bb8224d..79650e5 100644 --- a/google/gax/__init__.py +++ b/google/gax/__init__.py @@ -431,6 +431,10 @@ def __iter__(self): return self def next(self): + """For Python 2.7 compatibility; see __next__.""" + return self.__next__() + + def __next__(self): """Retrieves the next page.""" if self._done: raise StopIteration @@ -456,7 +460,7 @@ def __init__(self, page_iterator): Args: page_iterator (PageIterator): the base iterator of getting pages. """ - self._iterator = page_iterator + self._page_iterator = page_iterator self._current = None self._index = -1 @@ -464,10 +468,14 @@ def __iter__(self): return self def next(self): + """For Python 2.7 compatibility; see __next__.""" + return self.__next__() + + def __next__(self): """Retrieves the next resource.""" # pylint: disable=next-method-called while not self._current: - self._current = self._iterator.next() + self._current = next(self._page_iterator) self._index = 0 resource = self._current[self._index] self._index += 1 diff --git a/google/gax/api_callable.py b/google/gax/api_callable.py index 96c4892..086b630 100644 --- a/google/gax/api_callable.py +++ b/google/gax/api_callable.py @@ -31,9 +31,10 @@ from __future__ import absolute_import, division import random -import sys import time +from future.utils import raise_with_traceback + from . import (BackoffSettings, BundleOptions, bundling, CallSettings, config, PageIterator, ResourceIterator, RetryOptions) from .errors import GaxError, RetryError @@ -429,8 +430,7 @@ def inner(*args, **kwargs): return a_func(*args, **kwargs) # pylint: disable=catching-non-exception except tuple(errors) as exception: - raise (GaxError('RPC failed', cause=exception), None, - sys.exc_info()[2]) + raise_with_traceback(GaxError('RPC failed', cause=exception)) return inner diff --git a/google/gax/bundling.py b/google/gax/bundling.py index d2fd06f..23aa245 100644 --- a/google/gax/bundling.py +++ b/google/gax/bundling.py @@ -179,8 +179,8 @@ def _run_with_subresponses(self, req, subresponse_field, kwargs): in_sizes = [len(elts) for elts in self._in_deque] all_subresponses = getattr(resp, subresponse_field) if len(all_subresponses) != sum(in_sizes): - _LOG.warn(_WARN_DEMUX_MISMATCH, len(all_subresponses), - sum(in_sizes)) + _LOG.warning(_WARN_DEMUX_MISMATCH, len(all_subresponses), + sum(in_sizes)) for event in self._event_deque: event.result = resp event.set() @@ -251,7 +251,7 @@ def canceller(): return canceller -TIMER_FACTORY = threading.Timer +TIMER_FACTORY = threading.Timer # pylint: disable=invalid-name """A class with an interface similar to threading.Timer. Defaults to threading.Timer. This makes it easy to plug-in alternate diff --git a/requirements.txt b/requirements.txt index 5cf62b5..a131055 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ -grpcio>=0.15.0 +future>=0.15.2 +grpcio>=1.0rc1 oauth2client>=1.5.2 ply==3.8 protobuf>=3.0.0b3 diff --git a/setup.py b/setup.py index 00dc5a7..b3b1a20 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,8 @@ raise RuntimeError("No version number found!") install_requires = [ - 'grpcio>=0.15.0', + 'future>=0.15.2', + 'grpcio>=1.0rc1', 'ply==3.8', 'protobuf>=3.0.0b3', 'oauth2client>=1.5.2', diff --git a/test/test_api_callable.py b/test/test_api_callable.py index d8c2fb3..1d3ccce 100644 --- a/test/test_api_callable.py +++ b/test/test_api_callable.py @@ -131,9 +131,9 @@ def test_call_kwargs(self): settings = CallSettings(kwargs={'key': 'value'}) my_callable = api_callable.create_api_call( lambda _req, _timeout, **kwargs: kwargs['key'], settings) - self.assertEquals(my_callable(None), 'value') - self.assertEquals(my_callable(None, CallOptions(key='updated')), - 'updated') + self.assertEqual(my_callable(None), 'value') + self.assertEqual(my_callable(None, CallOptions(key='updated')), + 'updated') @mock.patch('time.time') @mock.patch('google.gax.config.exc_to_code') @@ -313,8 +313,7 @@ def grpc_return_value(request, *dummy_args, **dummy_kwargs): return PageStreamingResponse(nums=list(range(page_size)), next_page_token=page_size) - with mock.patch('grpc.framework.crust.implementations.' - '_UnaryUnaryMultiCallable') as mock_grpc: + with mock.patch('grpc.UnaryUnaryMultiCallable') as mock_grpc: mock_grpc.side_effect = grpc_return_value settings = CallSettings( page_descriptor=fake_grpc_func_descriptor, timeout=0) @@ -371,7 +370,7 @@ def my_func(request, dummy_timeout): self.assertIsInstance(first, bundling.Event) self.assertIsNone(first.result) # pylint: disable=no-member second = my_callable(BundlingRequest([0] * 5)) - self.assertEquals(second.result, 8) # pylint: disable=no-member + self.assertEqual(second.result, 8) # pylint: disable=no-member def test_construct_settings(self): defaults = api_callable.construct_settings( @@ -385,14 +384,14 @@ def test_construct_settings(self): self.assertIsInstance(settings.bundle_descriptor, BundleDescriptor) self.assertIsNone(settings.page_descriptor) self.assertIsInstance(settings.retry, RetryOptions) - self.assertEquals(settings.kwargs, {'key1': 'value1'}) + self.assertEqual(settings.kwargs, {'key1': 'value1'}) settings = defaults['page_streaming_method'] self.assertAlmostEqual(settings.timeout, 12.0) self.assertIsNone(settings.bundler) self.assertIsNone(settings.bundle_descriptor) self.assertIsInstance(settings.page_descriptor, PageDescriptor) self.assertIsInstance(settings.retry, RetryOptions) - self.assertEquals(settings.kwargs, {'key1': 'value1'}) + self.assertEqual(settings.kwargs, {'key1': 'value1'}) def test_construct_settings_override(self): _override = { @@ -455,8 +454,8 @@ def test_construct_settings_override2(self): page_descriptors=_PAGE_DESCRIPTORS) settings = defaults['bundling_method'] backoff = settings.retry.backoff_settings - self.assertEquals(backoff.initial_retry_delay_millis, 1000) - self.assertEquals(settings.retry.retry_codes, [_RETRY_DICT['code_a']]) + self.assertEqual(backoff.initial_retry_delay_millis, 1000) + self.assertEqual(settings.retry.retry_codes, [_RETRY_DICT['code_a']]) self.assertIsInstance(settings.bundler, bundling.Executor) self.assertIsInstance(settings.bundle_descriptor, BundleDescriptor) @@ -465,10 +464,10 @@ def test_construct_settings_override2(self): # not affect the methods which are not in the overrides. settings = defaults['page_streaming_method'] backoff = settings.retry.backoff_settings - self.assertEquals(backoff.initial_retry_delay_millis, 100) - self.assertEquals(backoff.retry_delay_multiplier, 1.2) - self.assertEquals(backoff.max_retry_delay_millis, 1000) - self.assertEquals(settings.retry.retry_codes, [_RETRY_DICT['code_c']]) + self.assertEqual(backoff.initial_retry_delay_millis, 100) + self.assertEqual(backoff.retry_delay_multiplier, 1.2) + self.assertEqual(backoff.max_retry_delay_millis, 1000) + self.assertEqual(settings.retry.retry_codes, [_RETRY_DICT['code_c']]) @mock.patch('google.gax.config.API_ERRORS', (CustomException, )) def test_catch_error(self): diff --git a/test/test_bundling.py b/test/test_bundling.py index af224ba..f0c4d83 100644 --- a/test/test_bundling.py +++ b/test/test_bundling.py @@ -89,7 +89,7 @@ def test_computes_bundle_ids_ok(self): for t in tests: got = bundling.compute_bundle_id(t['object'], t['fields']) message = 'failed while making an id for {}'.format(t['message']) - self.assertEquals(got, t['want'], message) + self.assertEqual(got, t['want'], message) def test_should_raise_if_fields_are_missing(self): tests = [ @@ -169,7 +169,7 @@ def test_extend_increases_the_element_count(self): t['update'](test_task) got = test_task.element_count message = 'bad message count when {}'.format(t['message']) - self.assertEquals(got, t['want'], message) + self.assertEqual(got, t['want'], message) def test_extend_increases_the_request_byte_count(self): simple_msg = 'a simple msg' @@ -193,7 +193,7 @@ def test_extend_increases_the_request_byte_count(self): t['update'](test_task) got = test_task.request_bytesize message = 'bad message count when {}'.format(t['message']) - self.assertEquals(got, t['want'], message) + self.assertEqual(got, t['want'], message) def test_run_sends_the_bundle_elements(self): simple_msg = 'a simple msg' @@ -221,26 +221,26 @@ def test_run_sends_the_bundle_elements(self): for t in tests: test_task = _make_a_test_task() event = t['update'](test_task) - self.assertEquals(test_task.element_count, t['count_before_run']) + self.assertEqual(test_task.element_count, t['count_before_run']) test_task.run() - self.assertEquals(test_task.element_count, 0) - self.assertEquals(test_task.request_bytesize, 0) + self.assertEqual(test_task.element_count, 0) + self.assertEqual(test_task.request_bytesize, 0) if t['has_event']: self.assertIsNotNone( event, 'expected event for {}'.format(t['message'])) got = event.result message = 'bad output when run with {}'.format(t['message']) - self.assertEquals(got, t['want'], message) + self.assertEqual(got, t['want'], message) def test_run_adds_an_error_if_execution_fails(self): simple_msg = 'a simple msg' test_task = _make_a_test_task(api_call=_raise_exc) event = test_task.extend([simple_msg]) - self.assertEquals(test_task.element_count, 1) + self.assertEqual(test_task.element_count, 1) test_task.run() - self.assertEquals(test_task.element_count, 0) - self.assertEquals(test_task.request_bytesize, 0) + self.assertEqual(test_task.element_count, 0) + self.assertEqual(test_task.request_bytesize, 0) self.assertTrue(isinstance(event.result, ValueError)) def test_calling_the_canceller_stops_the_element_from_getting_sent(self): @@ -249,14 +249,14 @@ def test_calling_the_canceller_stops_the_element_from_getting_sent(self): test_task = _make_a_test_task() an_event = test_task.extend([an_elt]) another_event = test_task.extend([another_msg]) - self.assertEquals(test_task.element_count, 2) + self.assertEqual(test_task.element_count, 2) self.assertTrue(an_event.cancel()) - self.assertEquals(test_task.element_count, 1) + self.assertEqual(test_task.element_count, 1) self.assertFalse(an_event.cancel()) - self.assertEquals(test_task.element_count, 1) + self.assertEqual(test_task.element_count, 1) test_task.run() - self.assertEquals(test_task.element_count, 0) - self.assertEquals(_Bundled([another_msg]), another_event.result) + self.assertEqual(test_task.element_count, 0) + self.assertEqual(_Bundled([another_msg]), another_event.result) self.assertFalse(an_event.is_set()) self.assertIsNone(an_event.result) @@ -301,8 +301,8 @@ def test_api_calls_are_grouped_by_bundle_id(self): self.assertTrue( got_event.is_set(), 'event is not set after triggering element') - self.assertEquals(_Bundled([an_elt] * threshold), - got_event.result) + self.assertEqual(_Bundled([an_elt] * threshold), + got_event.result) def test_each_event_has_exception_when_demuxed_api_call_fails(self): an_elt = 'dummy message' @@ -366,8 +366,8 @@ def test_each_event_has_its_result_from_a_demuxed_api_call(self): self.assertTrue(previous_event != event) self.assertTrue(event.is_set(), 'event is not set after triggering element') - self.assertEquals(event.result, - _Bundled(['%s%d' % (an_elt, index)] * index)) + self.assertEqual(event.result, + _Bundled(['%s%d' % (an_elt, index)] * index)) previous_event = event def test_each_event_has_same_result_from_mismatched_demuxed_api_call(self): @@ -394,7 +394,7 @@ def test_each_event_has_same_result_from_mismatched_demuxed_api_call(self): self.assertTrue(previous_event != event) self.assertTrue(event.is_set(), 'event is not set after triggering element') - self.assertEquals(event.result, mismatched_result) + self.assertEqual(event.result, mismatched_result) previous_event = event def test_schedule_passes_kwargs(self): @@ -409,8 +409,8 @@ def test_schedule_passes_kwargs(self): _Bundled([an_elt]), {'an_option': 'a_value'} ) - self.assertEquals('a_value', - event.result['an_option']) + self.assertEqual('a_value', + event.result['an_option']) class TestExecutor_ElementCountTrigger(unittest2.TestCase): @@ -437,8 +437,8 @@ def test_api_call_not_invoked_until_threshold(self): self.assertIsNone(got_event.result) else: self.assertTrue(got_event.is_set()) - self.assertEquals(_Bundled([an_elt] * threshold), - got_event.result) + self.assertEqual(_Bundled([an_elt] * threshold), + got_event.result) class TestExecutor_RequestByteTrigger(unittest2.TestCase): @@ -466,8 +466,8 @@ def test_api_call_not_invoked_until_threshold(self): self.assertIsNone(got_event.result) else: self.assertTrue(got_event.is_set()) - self.assertEquals(_Bundled([an_elt] * elts_for_threshold), - got_event.result) + self.assertEqual(_Bundled([an_elt] * elts_for_threshold), + got_event.result) class TestExecutor_DelayThreshold(unittest2.TestCase): @@ -490,8 +490,8 @@ def test_api_call_is_scheduled_on_timer(self, timer_class): self.assertIsNone(got_event.result) self.assertTrue(timer_class.called) timer_args, timer_kwargs = timer_class.call_args_list[0] - self.assertEquals(delay_threshold, timer_args[0]) - self.assertEquals({'args': [an_id]}, timer_kwargs) + self.assertEqual(delay_threshold, timer_args[0]) + self.assertEqual({'args': [an_id]}, timer_kwargs) timer_class.return_value.start.assert_called_once_with() diff --git a/test/test_grpc.py b/test/test_grpc.py index e727d9c..56329d9 100644 --- a/test/test_grpc.py +++ b/test/test_grpc.py @@ -58,7 +58,7 @@ def test_creates_a_stub_ok_with_no_scopes( chan.assert_called_once_with(self.FAKE_SERVICE_PATH, self.FAKE_PORT, comp.return_value) auth.assert_called_once_with([]) - self.assertEquals(got_channel, chan.return_value) + self.assertEqual(got_channel, chan.return_value) @mock.patch('grpc.beta.implementations.composite_channel_credentials') @mock.patch('grpc.beta.implementations.ssl_channel_credentials') @@ -86,7 +86,7 @@ def test_creates_a_stub_with_given_channel( got_channel = grpc.create_stub( _fake_create_stub, self.FAKE_SERVICE_PATH, self.FAKE_PORT, channel=fake_channel) - self.assertEquals(got_channel, fake_channel) + self.assertEqual(got_channel, fake_channel) self.assertFalse(auth.called) self.assertFalse(chan_creds.called) self.assertFalse(chan.called) @@ -111,7 +111,7 @@ def test_creates_a_stub_ok_with_given_creds(self, auth, chan, chan_creds, self.assertFalse(chan_creds.called) self.assertTrue(comp.called) self.assertTrue(md.called) - self.assertEquals(got_channel, chan.return_value) + self.assertEqual(got_channel, chan.return_value) @mock.patch('grpc.beta.implementations.composite_channel_credentials') @mock.patch('grpc.beta.implementations.ssl_channel_credentials') diff --git a/tox.ini b/tox.ini index 193501e..fad8ccb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,13 +1,14 @@ [tox] -envlist = py27,pep8,pylint-errors,pylint-full +envlist = py27,py34,py35,pep8,pylint-errors,pylint-full [tox:travis] -2.7 = py27, pep8, pylint-full, docs +2.7 = py27,pep8,pylint-full,docs +3.4 = py34,pep8,pylint-full +3.5 = py35,pep8,pylint-full [testenv] setenv = PYTHONPATH = {toxinidir}:{toxinidir}/src-gen/test - deps = -r{toxinidir}/test-requirements.txt -r{toxinidir}/requirements.txt whitelist_externals = mkdir @@ -55,5 +56,6 @@ commands = sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html sphinx-build -b latex -D language=en -d _build/doctrees docs _build/latex deps = + -r{toxinidir}/requirements.txt Sphinx sphinx_rtd_theme