Parse timestamps in query parameters using canonical format (#3945)

* Parse timestamps in query parameters according to BigQuery canonical timestamp format.

The timestamp format in query parameters follows the canonical format
specified at
https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#timestamp-type

This fixes a system test error which was happening in the bigquery-b2
branch.

* Support more possible timestamp formats.

Any of these formats may be returned from the BigQuery API.

* Chop and string-replace timestamps into a canonical format.

* BQ: fix lint errors. Remove references to table.name

bigquery/google/cloud/bigquery/_helpers.py

@@ -71,6 +71,39 @@ def _timestamp_from_json(value, field):
         return _datetime_from_microseconds(1e6 * float(value))
 
 
+def _timestamp_query_param_from_json(value, field):
+    """Coerce 'value' to a datetime, if set or not nullable.
+
+    Args:
+        value (str): The timestamp.
+        field (.SchemaField): The field corresponding to the value.
+
+    Returns:
+        Optional[datetime.datetime]: The parsed datetime object from
+        ``value`` if the ``field`` is not null (otherwise it is
+        :data:`None`).
+    """
+    if _not_null(value, field):
+        # Canonical formats for timestamps in BigQuery are flexible. See:
+        # g.co/cloud/bigquery/docs/reference/standard-sql/data-types#timestamp-type
+        # The separator between the date and time can be 'T' or ' '.
+        value = value.replace(' ', 'T', 1)
+        # The UTC timezone may be formatted as Z or +00:00.
+        value = value.replace('Z', '')
+        value = value.replace('+00:00', '')
+
+        if '.' in value:
+            # YYYY-MM-DDTHH:MM:SS.ffffff
+            return datetime.datetime.strptime(
+                value, _RFC3339_MICROS_NO_ZULU).replace(tzinfo=UTC)
+        else:
+            # YYYY-MM-DDTHH:MM:SS
+            return datetime.datetime.strptime(
+                value, _RFC3339_NO_FRACTION).replace(tzinfo=UTC)
+    else:
+        return None
+
+
 def _datetime_from_json(value, field):
     """Coerce 'value' to a datetime, if set or not nullable.
 
@@ -139,6 +172,9 @@ def _record_from_json(value, field):
     'RECORD': _record_from_json,
 }
 
+_QUERY_PARAMS_FROM_JSON = dict(_CELLDATA_FROM_JSON)
+_QUERY_PARAMS_FROM_JSON['TIMESTAMP'] = _timestamp_query_param_from_json
+
 
 def _row_from_json(row, schema):
     """Convert JSON row data to row with appropriate types.
@@ -454,7 +490,7 @@ def from_api_repr(cls, resource):
         name = resource.get('name')
         type_ = resource['parameterType']['type']
         value = resource['parameterValue']['value']
-        converted = _CELLDATA_FROM_JSON[type_](value, None)
+        converted = _QUERY_PARAMS_FROM_JSON[type_](value, None)
         return cls(name, type_, converted)
 
     def to_api_repr(self):
@@ -576,7 +612,9 @@ def _from_api_repr_scalar(cls, resource):
             for value
             in resource['parameterValue']['arrayValues']]
         converted = [
-            _CELLDATA_FROM_JSON[array_type](value, None) for value in values]
+            _QUERY_PARAMS_FROM_JSON[array_type](value, None)
+            for value in values
+        ]
         return cls(name, array_type, converted)
 
     @classmethod
@@ -732,7 +770,7 @@ def from_api_repr(cls, resource):
                 converted = ArrayQueryParameter.from_api_repr(struct_resource)
             else:
                 value = value['value']
-                converted = _CELLDATA_FROM_JSON[type_](value, None)
+                converted = _QUERY_PARAMS_FROM_JSON[type_](value, None)
             instance.struct_values[key] = converted
         return instance
 

bigquery/tests/system.py

@@ -1081,6 +1081,14 @@ def test_dbapi_w_query_parameters(self):
                 },
                 'expected': datetime.datetime(2012, 3, 4, 5, 6, 0, tzinfo=UTC),
             },
+            {
+                'sql': 'SELECT TIMESTAMP_TRUNC(%(zoned)s, MINUTE)',
+                'query_parameters': {
+                    'zoned': datetime.datetime(
+                        2012, 3, 4, 5, 6, 7, 250000, tzinfo=UTC),
+                },
+                'expected': datetime.datetime(2012, 3, 4, 5, 6, 0, tzinfo=UTC),
+            },
         ]
         for example in examples:
             msg = 'sql: {} query_parameters: {}'.format(

bigquery/tests/unit/test__helpers.py

@@ -158,6 +158,63 @@ def test_w_base64_encoded_text(self):
         self.assertEqual(coerced, expected)
 
 
+class Test_timestamp_query_param_from_json(unittest.TestCase):
+
+    def _call_fut(self, value, field):
+        from google.cloud.bigquery import _helpers
+
+        return _helpers._timestamp_query_param_from_json(value, field)
+
+    def test_w_none_nullable(self):
+        self.assertIsNone(self._call_fut(None, _Field('NULLABLE')))
+
+    def test_w_timestamp_valid(self):
+        from google.cloud._helpers import UTC
+
+        samples = [
+            (
+                '2016-12-20 15:58:27.339328+00:00',
+                datetime.datetime(2016, 12, 20, 15, 58, 27, 339328, tzinfo=UTC)
+            ),
+            (
+                '2016-12-20 15:58:27+00:00',
+                datetime.datetime(2016, 12, 20, 15, 58, 27, tzinfo=UTC)
+            ),
+            (
+                '2016-12-20T15:58:27.339328+00:00',
+                datetime.datetime(2016, 12, 20, 15, 58, 27, 339328, tzinfo=UTC)
+            ),
+            (
+                '2016-12-20T15:58:27+00:00',
+                datetime.datetime(2016, 12, 20, 15, 58, 27, tzinfo=UTC)
+            ),
+            (
+                '2016-12-20 15:58:27.339328Z',
+                datetime.datetime(2016, 12, 20, 15, 58, 27, 339328, tzinfo=UTC)
+            ),
+            (
+                '2016-12-20 15:58:27Z',
+                datetime.datetime(2016, 12, 20, 15, 58, 27, tzinfo=UTC)
+            ),
+            (
+                '2016-12-20T15:58:27.339328Z',
+                datetime.datetime(2016, 12, 20, 15, 58, 27, 339328, tzinfo=UTC)
+            ),
+            (
+                '2016-12-20T15:58:27Z',
+                datetime.datetime(2016, 12, 20, 15, 58, 27, tzinfo=UTC)
+            ),
+        ]
+        for timestamp_str, expected_result in samples:
+            self.assertEqual(
+                self._call_fut(timestamp_str, _Field('NULLABLE')),
+                expected_result)
+
+    def test_w_timestamp_invalid(self):
+        with self.assertRaises(ValueError):
+            self._call_fut('definitely-not-a-timestamp', _Field('NULLABLE'))
+
+
 class Test_timestamp_from_json(unittest.TestCase):
 
     def _call_fut(self, value, field):
@@ -1820,6 +1877,44 @@ def test_w_scalar(self):
         self.assertEqual(parameter.type_, 'INT64')
         self.assertEqual(parameter.value, 123)
 
+    def test_w_scalar_timestamp(self):
+        from google.cloud.bigquery._helpers import ScalarQueryParameter
+        from google.cloud._helpers import UTC
+
+        RESOURCE = {
+            'name': 'zoned',
+            'parameterType': {'type': 'TIMESTAMP'},
+            'parameterValue': {'value': '2012-03-04 05:06:07+00:00'},
+        }
+
+        parameter = self._call_fut(RESOURCE)
+
+        self.assertIsInstance(parameter, ScalarQueryParameter)
+        self.assertEqual(parameter.name, 'zoned')
+        self.assertEqual(parameter.type_, 'TIMESTAMP')
+        self.assertEqual(
+            parameter.value,
+            datetime.datetime(2012, 3, 4, 5, 6, 7, tzinfo=UTC))
+
+    def test_w_scalar_timestamp_micros(self):
+        from google.cloud.bigquery._helpers import ScalarQueryParameter
+        from google.cloud._helpers import UTC
+
+        RESOURCE = {
+            'name': 'zoned',
+            'parameterType': {'type': 'TIMESTAMP'},
+            'parameterValue': {'value': '2012-03-04 05:06:07.250000+00:00'},
+        }
+
+        parameter = self._call_fut(RESOURCE)
+
+        self.assertIsInstance(parameter, ScalarQueryParameter)
+        self.assertEqual(parameter.name, 'zoned')
+        self.assertEqual(parameter.type_, 'TIMESTAMP')
+        self.assertEqual(
+            parameter.value,
+            datetime.datetime(2012, 3, 4, 5, 6, 7, 250000, tzinfo=UTC))
+
     def test_w_array(self):
         from google.cloud.bigquery._helpers import ArrayQueryParameter