Using skipped results to modify query offset.

gcloud/datastore/query.py

@@ -390,6 +390,7 @@ def __init__(self, query, client, limit=None, offset=None,
         self._start_cursor = start_cursor
         self._end_cursor = end_cursor
         self._page = self._more_results = None
+        self._skipped_results = None
 
     def next_page(self):
         """Fetch a single "page" of query results.
@@ -432,7 +433,8 @@ def next_page(self):
         #       results. See
         #       https://github.com/GoogleCloudPlatform/gcloud-python/issues/280
         #       for discussion.
-        entity_pbs, cursor_as_bytes, more_results_enum = query_results[:3]
+        (entity_pbs, cursor_as_bytes,
+         more_results_enum, self._skipped_results) = query_results
 
         if cursor_as_bytes == b'':
             self._start_cursor = None
@@ -466,10 +468,10 @@ def __iter__(self):
             num_results = len(self._page)
             if self._limit is not None:
                 self._limit -= num_results
-            if self._offset is not None:
+            if self._offset is not None and self._skipped_results is not None:
                 # NOTE: The offset goes down relative to the location
                 #       because we are updating the cursor each time.
-                self._offset -= num_results
+                self._offset -= self._skipped_results
             self.next_page()
 
 

gcloud/datastore/test_query.py

@@ -345,7 +345,8 @@ def _getTargetClass(self):
     def _makeOne(self, *args, **kw):
         return self._getTargetClass()(*args, **kw)
 
-    def _addQueryResults(self, connection, cursor=_END, more=False):
+    def _addQueryResults(self, connection, cursor=_END, more=False,
+                         skipped_results=None):
         from gcloud.datastore._generated import entity_pb2
         from gcloud.datastore._generated import query_pb2
         from gcloud.datastore.helpers import _new_value_pb
@@ -361,7 +362,7 @@ def _addQueryResults(self, connection, cursor=_END, more=False):
         value_pb = _new_value_pb(entity_pb, 'foo')
         value_pb.string_value = u'Foo'
         connection._results.append(
-            ([entity_pb], cursor, MORE if more else NO_MORE))
+            ([entity_pb], cursor, MORE if more else NO_MORE, skipped_results))
 
     def _makeClient(self, connection=None):
         if connection is None:
@@ -476,8 +477,8 @@ def test_next_page_w_cursors_w_bogus_more(self):
         client = self._makeClient(connection)
         query = _Query(client, self._KIND, self._PROJECT, self._NAMESPACE)
         self._addQueryResults(connection, cursor=self._END, more=True)
-        epb, cursor, _ = connection._results.pop()
-        connection._results.append((epb, cursor, 4))  # invalid enum
+        epb, cursor, _, _ = connection._results.pop()
+        connection._results.append((epb, cursor, 4, None))  # invalid enum
         iterator = self._makeOne(query, client)
         self.assertRaises(ValueError, iterator.next_page)
 
@@ -547,9 +548,12 @@ def test___iter___w_limit(self):
         connection = _Connection()
         client = self._makeClient(connection)
         query = _Query(client, self._KIND, self._PROJECT, self._NAMESPACE)
-        self._addQueryResults(connection, more=True)
+        skipped_results = 4
+        self._addQueryResults(connection, more=True,
+                              skipped_results=skipped_results)
         self._addQueryResults(connection)
-        iterator = self._makeOne(query, client, limit=2, offset=13)
+        offset = 13
+        iterator = self._makeOne(query, client, limit=2, offset=offset)
         entities = list(iterator)
 
         self.assertFalse(iterator._more_results)
@@ -561,11 +565,11 @@ def test___iter___w_limit(self):
             self.assertEqual(entities[1]['foo'], u'Foo')
         qpb1 = _pb_from_query(query)
         qpb1.limit.value = 2
-        qpb1.offset = 13
+        qpb1.offset = offset
         qpb2 = _pb_from_query(query)
         qpb2.start_cursor = self._END
         qpb2.limit.value = 1
-        qpb2.offset = 12
+        qpb2.offset = offset - skipped_results
         EXPECTED1 = {
             'project': self._PROJECT,
             'query_pb': qpb1,