Unifying datetime->timestamp conversions.

Using a method to convert to microseconds (which handles
both naive datetime objects and ones with timezones other
than UTC) and then converts them to microseconds. All
consumers then convert to the desired granularity.

gcloud/_helpers.py

@@ -16,10 +16,10 @@
 This module is not part of the public API surface of `gcloud`.
 """
 
+import calendar
 import datetime
 import os
 import socket
-import sys
 
 try:
     from threading import local as Local
@@ -208,32 +208,6 @@ def _determine_default_project(project=None):
     return project
 
 
-def _manual_total_seconds(offset):
-    """Backport of timedelta.total_seconds() from python 2.7+.
-
-    :type offset: :class:`datetime.timedelta`
-    :param offset: The value to convert into seconds.
-
-    :rtype: float
-    :returns: The time offset, converted to total seconds.
-    """
-    seconds = offset.days * 24 * 60 * 60 + offset.seconds
-    microseconds = seconds * 10**6 + offset.microseconds
-    return microseconds / (10**6 * 1.0)
-
-
-def _total_seconds_from_type(offset):
-    """Basic wrapper around timedelta.total_seconds().
-
-    :type offset: :class:`datetime.timedelta`
-    :param offset: The value to convert into seconds.
-
-    :rtype: float
-    :returns: The time offset, converted to total seconds.
-    """
-    return offset.total_seconds()
-
-
 def _millis(when):
     """Convert a zone-aware datetime to integer milliseconds.
 
@@ -243,7 +217,9 @@ def _millis(when):
     :rtype: integer
     :returns: milliseconds since epoch for ``when``
     """
-    return int(_TOTAL_SECONDS(when - _EPOCH) * 1000)
+    micros = _microseconds_from_datetime(when)
+    millis, _ = divmod(micros, 1000)
+    return millis
 
 
 def _datetime_from_millis(value):
@@ -261,6 +237,23 @@ def _datetime_from_millis(value):
         )
 
 
+def _microseconds_from_datetime(value):
+    """Convert non-none datetime to microseconds.
+
+    :type value: :class:`datetime.datetime`
+    :param value: The timestamp to convert.
+
+    :rtype: integer
+    :returns: The timestamp, in microseconds.
+    """
+    if not value.tzinfo:
+        value = value.replace(tzinfo=UTC)
+    # Regardless of what timezone is on the value, convert it to UTC.
+    value = value.astimezone(UTC)
+    # Convert the datetime to a microsecond timestamp.
+    return int(calendar.timegm(value.timetuple()) * 1e6) + value.microsecond
+
+
 def _millis_from_datetime(value):
     """Convert non-none datetime to timestamp, assuming UTC.
 
@@ -271,10 +264,6 @@ def _millis_from_datetime(value):
     :returns: the timestamp, in milliseconds, or None
     """
     if value is not None:
-        if value.tzinfo is None:
-            # Assume UTC
-            value = value.replace(tzinfo=UTC)
-        # back-end wants timestamps as milliseconds since the epoch
         return _millis(value)
 
 
@@ -285,9 +274,3 @@ def _millis_from_datetime(value):
 
 # Need to define _EPOCH at the end of module since it relies on UTC.
 _EPOCH = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=UTC)
-
-
-if sys.version_info[:2] < (2, 7):
-    _TOTAL_SECONDS = _manual_total_seconds  # pragma: NO COVER
-else:
-    _TOTAL_SECONDS = _total_seconds_from_type

gcloud/credentials.py

@@ -15,7 +15,6 @@
 """A simple wrapper around the OAuth2 credentials library."""
 
 import base64
-import calendar
 import datetime
 import six
 from six.moves.urllib.parse import urlencode  # pylint: disable=F0401
@@ -40,6 +39,7 @@ class _GAECreds(object):
         """Dummy class if not in App Engine environment."""
 
 from gcloud._helpers import UTC
+from gcloud._helpers import _microseconds_from_datetime
 
 
 def get_credentials():
@@ -280,15 +280,8 @@ def _get_expiration_seconds(expiration):
 
     # If it's a datetime, convert to a timestamp.
     if isinstance(expiration, datetime.datetime):
-        # Make sure the timezone on the value is UTC
-        # (either by converting or replacing the value).
-        if expiration.tzinfo:
-            expiration = expiration.astimezone(UTC)
-        else:
-            expiration = expiration.replace(tzinfo=UTC)
-
-        # Turn the datetime into a timestamp (seconds, not microseconds).
-        expiration = int(calendar.timegm(expiration.timetuple()))
+        micros = _microseconds_from_datetime(expiration)
+        expiration, _ = divmod(micros, 10**6)
 
     if not isinstance(expiration, six.integer_types):
         raise TypeError('Expected an integer timestamp, datetime, or '

gcloud/datastore/helpers.py

@@ -17,13 +17,13 @@
 The non-private functions are part of the API.
 """
 
-import calendar
 import datetime
 
 from google.protobuf.internal.type_checkers import Int64ValueChecker
 import six
 
 from gcloud._helpers import UTC
+from gcloud._helpers import _microseconds_from_datetime
 from gcloud.datastore import _datastore_v1_pb2 as datastore_pb
 from gcloud.datastore.entity import Entity
 from gcloud.datastore.key import Key
@@ -182,14 +182,7 @@ def _pb_attr_value(val):
 
     if isinstance(val, datetime.datetime):
         name = 'timestamp_microseconds'
-        # If the datetime is naive (no timezone), consider that it was
-        # intended to be UTC and replace the tzinfo to that effect.
-        if not val.tzinfo:
-            val = val.replace(tzinfo=UTC)
-        # Regardless of what timezone is on the value, convert it to UTC.
-        val = val.astimezone(UTC)
-        # Convert the datetime to a microsecond timestamp.
-        value = int(calendar.timegm(val.timetuple()) * 1e6) + val.microsecond
+        value = _microseconds_from_datetime(val)
     elif isinstance(val, Key):
         name, value = 'key', val.to_protobuf()
     elif isinstance(val, bool):

gcloud/test__helpers.py

@@ -251,41 +251,6 @@ def test_prod(self):
         self.assertEqual(callers, ['prod_mock'])
 
 
-class Test__manual_total_seconds(unittest2.TestCase):
-
-    def _callFUT(self, value):
-        from gcloud._helpers import _manual_total_seconds
-        return _manual_total_seconds(value)
-
-    def test_it(self):
-        import datetime
-
-        delta = datetime.timedelta(minutes=1, seconds=1,
-                                   microseconds=10**(6 - 1))
-        result = self._callFUT(delta)
-        self.assertEqual(result, 61.1)
-
-
-class Test__total_seconds_from_type(unittest2.TestCase):
-
-    def _callFUT(self, value):
-        from gcloud._helpers import _total_seconds_from_type
-        return _total_seconds_from_type(value)
-
-    def test_it(self):
-        class FakeDelta(object):
-
-            def __init__(self, total_seconds):
-                self._total_seconds = total_seconds
-
-            def total_seconds(self):
-                return self._total_seconds
-
-        value = object()
-        fake_delta = FakeDelta(value)
-        self.assertEqual(self._callFUT(fake_delta), value)
-
-
 class Test__millis(unittest2.TestCase):
 
     def _callFUT(self, value):
@@ -300,25 +265,21 @@ def test_one_second_from_epoch(self):
         self.assertEqual(self._callFUT(WHEN), 1000)
 
 
-class Test__datetime_from_millis(unittest2.TestCase):
+class Test__microseconds_from_datetime(unittest2.TestCase):
 
     def _callFUT(self, value):
-        from gcloud._helpers import _datetime_from_millis
-        return _datetime_from_millis(value)
-
-    def test_w_none(self):
-        self.assertTrue(self._callFUT(None) is None)
+        from gcloud._helpers import _microseconds_from_datetime
+        return _microseconds_from_datetime(value)
 
-    def test_w_millis(self):
+    def test_it(self):
         import datetime
-        from gcloud._helpers import UTC
-        from gcloud._helpers import _TOTAL_SECONDS
 
-        NOW = datetime.datetime(2015, 7, 29, 17, 45, 21, 123456,
-                                tzinfo=UTC)
-        EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC)
-        MILLIS = _TOTAL_SECONDS(NOW - EPOCH) * 1000
-        self.assertEqual(self._callFUT(MILLIS), NOW)
+        microseconds = 314159
+        timestamp = datetime.datetime(1970, 1, 1, hour=0,
+                                      minute=0, second=0,
+                                      microsecond=microseconds)
+        result = self._callFUT(timestamp)
+        self.assertEqual(result, microseconds)
 
 
 class Test__millis_from_datetime(unittest2.TestCase):
@@ -333,47 +294,67 @@ def test_w_none(self):
     def test_w_utc_datetime(self):
         import datetime
         from gcloud._helpers import UTC
-        from gcloud._helpers import _TOTAL_SECONDS
+        from gcloud._helpers import _microseconds_from_datetime
 
         NOW = datetime.datetime.utcnow().replace(tzinfo=UTC)
-        EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC)
-        MILLIS = int(_TOTAL_SECONDS(NOW - EPOCH) * 1000)
+        NOW_MICROS = _microseconds_from_datetime(NOW)
+        MILLIS, _ = divmod(NOW_MICROS, 1000)
         result = self._callFUT(NOW)
         self.assertTrue(isinstance(result, int))
         self.assertEqual(result, MILLIS)
 
     def test_w_non_utc_datetime(self):
         import datetime
-        from gcloud._helpers import UTC
         from gcloud._helpers import _UTC
-        from gcloud._helpers import _TOTAL_SECONDS
+        from gcloud._helpers import _microseconds_from_datetime
 
         class CET(_UTC):
             _tzname = 'CET'
             _utcoffset = datetime.timedelta(hours=-1)
 
         zone = CET()
         NOW = datetime.datetime(2015, 7, 28, 16, 34, 47, tzinfo=zone)
-        EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC)
-        MILLIS = int(_TOTAL_SECONDS(NOW - EPOCH) * 1000)
+        NOW_MICROS = _microseconds_from_datetime(NOW)
+        MILLIS, _ = divmod(NOW_MICROS, 1000)
         result = self._callFUT(NOW)
         self.assertTrue(isinstance(result, int))
         self.assertEqual(result, MILLIS)
 
     def test_w_naive_datetime(self):
         import datetime
         from gcloud._helpers import UTC
-        from gcloud._helpers import _TOTAL_SECONDS
+        from gcloud._helpers import _microseconds_from_datetime
 
         NOW = datetime.datetime.utcnow()
         UTC_NOW = NOW.replace(tzinfo=UTC)
-        EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC)
-        MILLIS = int(_TOTAL_SECONDS(UTC_NOW - EPOCH) * 1000)
+        UTC_NOW_MICROS = _microseconds_from_datetime(UTC_NOW)
+        MILLIS, _ = divmod(UTC_NOW_MICROS, 1000)
         result = self._callFUT(NOW)
         self.assertTrue(isinstance(result, int))
         self.assertEqual(result, MILLIS)
 
 
+class Test__datetime_from_millis(unittest2.TestCase):
+
+    def _callFUT(self, value):
+        from gcloud._helpers import _datetime_from_millis
+        return _datetime_from_millis(value)
+
+    def test_w_none(self):
+        self.assertTrue(self._callFUT(None) is None)
+
+    def test_w_millis(self):
+        import datetime
+        from gcloud._helpers import UTC
+        from gcloud._helpers import _microseconds_from_datetime
+
+        NOW = datetime.datetime(2015, 7, 29, 17, 45, 21, 123456,
+                                tzinfo=UTC)
+        NOW_MICROS = _microseconds_from_datetime(NOW)
+        MILLIS = NOW_MICROS / 1000.0
+        self.assertEqual(self._callFUT(MILLIS), NOW)
+
+
 class _AppIdentity(object):
 
     def __init__(self, app_id):