use Cmr-search-after instead of paging

icepyx/core/APIformatting.py

@@ -325,7 +325,7 @@ def _get_possible_keys(self):
             }
         elif self.partype == "required":
             self._poss_keys = {
-                "search": ["page_size", "page_num"],
+                "search": ["page_size"],
                 "download": [
                     "page_size",
                     "page_num",
@@ -442,28 +442,19 @@ def build_params(self, **kwargs):
                 reqkeys = self.poss_keys[self._reqtype]
                 defaults = {
                     "page_size": 2000,
-                    "page_num": 1,
+                    "page_num": 0,
                     "request_mode": "async",
                     "include_meta": "Y",
                     "client_string": "icepyx",
                 }
                 for key in reqkeys:
                     if key in kwargs:
                         self._fmted_keys.update({key: kwargs[key]})
-                    #                 elif key in defaults:
-                    #                     if key is 'page_num':
-                    #                         pnum = math.ceil(len(is2obj.granules)/reqparams['page_size'])
-                    #                         if pnum > 0:
-                    #                             reqparams.update({key:pnum})
-                    #                         else:
-                    #                             reqparams.update({key:defaults[key]})
                     elif key in defaults:
                         self._fmted_keys.update({key: defaults[key]})
                     else:
                         pass
 
-                self._fmted_keys["page_num"] = 1
-
         else:
             if self.check_values == True and kwargs == None:
                 pass

icepyx/core/granules.py

@@ -199,16 +199,28 @@ def get_avail(self, CMRparams, reqparams):
 
         headers = {"Accept": "application/json", "Client-Id": "icepyx"}
         # note we should also check for errors whenever we ping NSIDC-API - make a function to check for errors
+
+        params = apifmt.combine_params(
+            CMRparams, {k: reqparams[k] for k in ["page_size"]}
+        )
+
+        cmr_search_after = None
+
         while True:
-            params = apifmt.combine_params(
-                CMRparams, {k: reqparams[k] for k in ("page_size", "page_num")}
-            )
+            if cmr_search_after is not None:
+                headers["CMR-Search-After"] = cmr_search_after
+
             response = requests.get(
                 granule_search_url,
                 headers=headers,
                 params=apifmt.to_string(params),
             )
 
+            try:
+                cmr_search_after = response.headers["CMR-Search-After"]
+            except KeyError:
+                cmr_search_after = None
+
             try:
                 response.raise_for_status()
             except requests.HTTPError as e:
@@ -223,16 +235,13 @@ def get_avail(self, CMRparams, reqparams):
 
             results = json.loads(response.content)
             if not results["feed"]["entry"]:
-                # Out of results, so break out of loop
+                assert len(self.avail) == int(
+                    response.headers["CMR-Hits"]
+                ), "Search failure - unexpected number of results"
                 break
 
-            # Collect results and increment page_num
+            # Collect results
             self.avail.extend(results["feed"]["entry"])
-            reqparams["page_num"] += 1
-
-        # DevNote: The above calculated page_num is wrong when mod(granule number, page_size)=0.
-        # print(reqparams['page_num'])
-        reqparams["page_num"] = int(np.ceil(len(self.avail) / reqparams["page_size"]))
 
         assert (
             len(self.avail) > 0
@@ -299,9 +308,7 @@ def place_order(
 
         base_url = "https://n5eil02u.ecs.nsidc.org/egi/request"
 
-        self.get_avail(
-            CMRparams, reqparams
-        )  # this way the reqparams['page_num'] is updated
+        self.get_avail(CMRparams, reqparams)
 
         if subset is False:
             request_params = apifmt.combine_params(
@@ -312,27 +319,30 @@ def place_order(
 
         order_fn = ".order_restart"
 
+        total_pages = int(np.ceil(len(self.avail) / reqparams["page_size"]))
         print(
             "Total number of data order requests is ",
-            request_params["page_num"],
+            total_pages,
             " for ",
             len(self.avail),
             " granules.",
         )
-        # DevNote/05/27/20/: Their page_num values are the same, but use the combined version anyway.
-        # I'm switching back to reqparams, because that value is not changed by the for loop. I shouldn't cause an issue either way, but I've had issues with mutable types in for loops elsewhere.
-        for i in range(reqparams["page_num"]):
-            #         for i in range(request_params['page_num']):
-            page_val = i + 1
 
+        if reqparams["page_num"] > 0:
+            pagenums = [reqparams["page_num"]]
+        else:
+            pagenums = range(1, total_pages + 1)
+
+        for page_num in pagenums:
             print(
                 "Data request ",
-                page_val,
+                page_num,
                 " of ",
-                reqparams["page_num"],
+                total_pages,
                 " is submitting to NSIDC",
             )
-            request_params.update({"page_num": page_val})
+            request_params = apifmt.combine_params(CMRparams, reqparams, subsetparams)
+            request_params.update({"page_num": page_num})
 
             # DevNote: earlier versions of the code used a file upload+post rather than putting the geometries
             # into the parameter dictionaries. However, this wasn't working with shapefiles, but this more general

icepyx/core/query.py

@@ -507,7 +507,7 @@ def reqparams(self):
         --------
         >>> reg_a = ipx.Query('ATL06',[-55, 68, -48, 71],['2019-02-20','2019-02-28'])
         >>> reg_a.reqparams
-        {'page_size': 2000, 'page_num': 1}
+        {'page_size': 2000}
 
         >>> reg_a = ipx.Query('ATL06',[-55, 68, -48, 71],['2019-02-20','2019-02-28']) # doctest: +SKIP
         >>> reg_a.earthdata_login(user_id,user_email) # doctest: +SKIP

icepyx/tests/is2class_query.py

@@ -20,7 +20,7 @@ def test_reqconfig_params():
     # test for search params
     reg_a.build_reqconfig_params("search")
     obs_keys = reg_a.reqparams.keys()
-    exp_keys_all = ["page_size", "page_num"]
+    exp_keys_all = ["page_size"]
     assert all(keys in obs_keys for keys in exp_keys_all)
 
     # test for download params
@@ -39,9 +39,6 @@ def test_reqconfig_params():
     assert all(keys in obs_keys for keys in exp_keys_all)
 
 
-
-
-
 # BestPractices: should do additional properties tests for each potential property type (e.g. spatial extent can have type bounding_box or polygon)
 
 

icepyx/tests/test_granules.py

@@ -628,5 +628,5 @@ def test_avail_granule_CMR_error():
     ermsg = "An error was returned from NSIDC in regards to your query: temporal start datetime is invalid: [badinput] is not a valid datetime."
     with pytest.raises(NsidcQueryError, match=re.escape(ermsg)):
         CMRparams = {"version": "003", "temporal": "badinput", "short_name": "ATL08"}
-        reqparams = {"page_size": 1, "page_num": 1}
+        reqparams = {"page_size": 1}
         Granules().get_avail(CMRparams=CMRparams, reqparams=reqparams)