Update context update function to use search-after

* updated to use `search-after` * added `useDelimiter` to Scanner in order to keep all whitespace * updated search query to ignore extraneous context products resolves #675
NASA-PDS · Oct 16, 2024 · 83d5d21 · 83d5d21
1 parent 17e0acc
commit 83d5d21
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 4 deletions.
diff --git a/src/main/java/gov/nasa/pds/validate/ValidateLauncher.java b/src/main/java/gov/nasa/pds/validate/ValidateLauncher.java
@@ -508,25 +508,41 @@ else if (Flag.MODEL.getShortName().equals(o.getOpt())) {
 
   @SuppressWarnings("unchecked")
   private void getLatestJsonContext() {
+    final String searchAfterParam = "search-after";
+    final String pageSize = "1000";
+    final String searchAfterKey = "ops:Harvest_Info.ops:harvest_date_time";
     List<ValidationProblem> pList = new ArrayList<>();
     ObjectMapper mapper = new ObjectMapper();
     String base = ToolInfo.getSearchURL();
     String endpoint = ToolInfo.getEndpoint();
     String query = ToolInfo.getQuery();
     URL url = null;
+    Scanner reader = null;
+    String searchAfter = "";
     try {
       int total = 0;
       List<Map<String,Object>> contexts = new ArrayList<Map<String,Object>>();
       do {
-        url = new URL(base + "/" + endpoint + "?" + URLEncoder.encode(query, StandardCharsets.UTF_8) + "&start=" + contexts.size());
-        Scanner reader = new Scanner(url.openStream());
+        url = new URL(base + "/" + endpoint + "?limit=1000"
+            + "&q=" + URLEncoder.encode(query, StandardCharsets.UTF_8)
+            + "&sort=" + searchAfterKey
+            + "&" + searchAfter);
+        LOG.debug("Query URL: " + url.toString());
+        reader = new Scanner(url.openStream()).useDelimiter("\\Z");
         StringBuffer buffer = new StringBuffer();
         while (reader.hasNext()) {
           buffer.append(reader.next());
         }
         Map<String, Object> response = mapper.readValue(buffer.toString(), HashMap.class);
         total = (Integer)((Map<String,Object>)response.get("summary")).get("hits");
-        contexts.addAll((List<Map<String,Object>>)response.get("data"));
+        List<Map<String, Object>> dataDocuments = (List<Map<String, Object>>) response.get("data");
+
+        contexts.addAll(dataDocuments);
+        String searchAfterValue =
+            getSearchAfterFromDocument(dataDocuments.get(dataDocuments.size() - 1), searchAfterKey);
+
+        searchAfter =
+            searchAfterParam + "=" + URLEncoder.encode(searchAfterValue, StandardCharsets.UTF_8);
       } while (contexts.size() < total);
       parseJsonObjectWriteTofile(contexts);
       ValidationProblem p1 =
@@ -551,6 +567,8 @@ private void getLatestJsonContext() {
       }
     } catch (Exception e) {
       e.printStackTrace();
+    } finally {
+      reader.close();
     }
 
     try {
@@ -563,6 +581,12 @@ private void getLatestJsonContext() {
     }
   }
 
+  private String getSearchAfterFromDocument(Map<String, Object> document, String searchAfterKey) {
+    @SuppressWarnings("unchecked")
+    Map<String, Object> properties = (Map<String, Object>) document.get("properties");
+    return ((List<String>) properties.get(searchAfterKey)).get(0);
+  }
+
   private void parseJsonObjectWriteTofile(List<Map<String,Object>> documents) {
     final List<String> empty = Arrays.asList("N/A");
     final List<String> fieldNames = Arrays.asList(
@@ -606,6 +630,7 @@ private void parseJsonObjectWriteTofile(List<Map<String,Object>> documents) {
             jsonWriter.name("name");
             jsonWriter.beginArray();
             for (Object n : names) {
+              System.out.println("name: " + (String) n);
               jsonWriter.value((String) n);
             }
             jsonWriter.endArray();

diff --git a/src/main/resources/validate.properties b/src/main/resources/validate.properties
@@ -37,4 +37,4 @@ validate.copyright=\nCopyright 2019, by the California Institute of Technology (
 validate.search_url=https://pds.nasa.gov/api
 validate.output_file_name=registered_context_products.json
 validate.endpoint=search/1/products
-validate.query=product_class eq "Product_Context"
+validate.query=(product_class eq "Product_Context" and pds:Resource.pds:type ne "Information.Science_Portal" and pds:Resource.pds:type ne "Information.Investigation")