foreign rules sync: remove old rules from DB

admin-ch · Mar 28, 2022 · d741385 · d741385
1 parent 326caa3
commit d741385
Show file tree

Hide file tree

Showing 3 changed files with 49 additions and 21 deletions.
diff --git a/...ava/ch/admin/bag/covidcertificate/backend/verifier/sync/syncer/DgcForeignRulesSyncer.java b/...ava/ch/admin/bag/covidcertificate/backend/verifier/sync/syncer/DgcForeignRulesSyncer.java
@@ -14,13 +14,16 @@
 import ch.admin.bag.covidcertificate.backend.verifier.model.ForeignRule;
 import ch.admin.bag.covidcertificate.backend.verifier.sync.utils.CmsUtil;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.sun.jdi.event.ExceptionEvent;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.security.cert.CertificateException;
 import java.time.Instant;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
 import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
 import org.bouncycastle.cms.CMSException;
 import org.bouncycastle.operator.OperatorCreationException;
 import org.slf4j.Logger;
@@ -43,19 +46,29 @@ public void sync() {
 
         List<String> countries = dgcRulesClient.getCountries();
         countries.remove("CH");
-        //download data for the country
-        countries.forEach(
-                //this gives us an array of rule IDs
-                country -> dgcRulesClient.download(country)
-                        //each rule ID has an array of rule versions
-                        .forEach((ruleId, ruleVersions) -> (ruleVersions).forEach(ruleVersion -> {
-                            //insert each rule version into the DB
-                            try {
-                                foreignRulesDataService.insertRule(decodeRule(ruleVersion));
-                            } catch (Exception e){
-                                logger.error("Failed to decode rule {}", ruleId, e);
-                            }
-                        })));
+
+        for(var country: countries){
+            var rules = dgcRulesClient.download(country).entrySet().stream()
+                    .map(
+                            rule -> {
+                                try {
+                                    return decodeRule(rule.getValue());
+                                } catch (Exception e) {
+                                    logger.error(
+                                            "Failed to decode rule {}",
+                                            rule.getKey(),
+                                            e);
+                                    return null;
+                                }
+                            })
+                    .filter(Objects::nonNull)
+                    .collect(Collectors.toList());
+            foreignRulesDataService.removeRuleSet(country);
+            rules.forEach(foreignRulesDataService::insertRule);
+            if(rules.isEmpty()){
+                logger.warn("No rules were downloaded or decoded for {}", country);
+            }
+        }
         var end = Instant.now();
         logger.info(
                 "Successfully downloaded foreign rules {} ms",

diff --git a/...h/admin/bag/covidcertificate/backend/verifier/ws/controller/ForeignRulesControllerV2.java b/...h/admin/bag/covidcertificate/backend/verifier/ws/controller/ForeignRulesControllerV2.java
@@ -13,9 +13,12 @@
 import ch.admin.bag.covidcertificate.backend.verifier.data.ForeignRulesDataService;
 import ch.admin.bag.covidcertificate.backend.verifier.data.ValueSetDataService;
 import ch.admin.bag.covidcertificate.backend.verifier.data.util.CacheUtil;
+import ch.admin.bag.covidcertificate.backend.verifier.ws.utils.EtagUtil;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import java.security.NoSuchAlgorithmException;
 import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -24,7 +27,9 @@
 import java.util.stream.Collectors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.boot.actuate.autoconfigure.metrics.MetricsProperties.Web;
 import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.GetMapping;
@@ -55,14 +60,14 @@ public ForeignRulesControllerV2(
 
     @GetMapping(value = "/foreignRules/{country}")
     public @ResponseBody ResponseEntity<Map> getForeignRules(
-            @PathVariable("country") String country) {
+            @PathVariable("country") String country, WebRequest request) {
         ObjectMapper mapper = new ObjectMapper();
         Map<String, Object> result = new HashMap<>();
         result.put("validDuration", 172800000);
 
         // Add rules to output
         var foreignRules = foreignRulesDataService.getRulesForCountry(country);
-        if(foreignRules.isEmpty()){
+        if (foreignRules.isEmpty()) {
             return ResponseEntity.notFound().build();
         }
 
@@ -101,12 +106,20 @@ public ForeignRulesControllerV2(
             }
         }
         result.put("valueSets", valueSets);
+        String etag = "";
+        try {
+            etag = EtagUtil.getSha1HashForStrings(true, mapper.writeValueAsString(result));
+        } catch (JsonProcessingException | NoSuchAlgorithmException e) {
+            logger.error("Failed to calculate ETag for rules", e);
+        }
+        if (request.checkNotModified(etag)) {
+            return ResponseEntity.status(HttpStatus.NOT_MODIFIED).build();
+        }
 
-        return ResponseEntity.ok(result);
+        return ResponseEntity.ok().headers(getVerificationRulesHeaders()).body(result);
     }
 
-    private HttpHeaders getVerificationRulesHeaders(Instant now) {
-        return CacheUtil.createExpiresHeader(
-                CacheUtil.roundToNextVerificationRulesBucketStart(now));
+    private HttpHeaders getVerificationRulesHeaders() {
+        return CacheUtil.createExpiresHeader(Instant.now().plus(48, ChronoUnit.HOURS));
     }
 }
diff --git a/...min/bag/covidcertificate/backend/verifier/ws/controller/ForeignRulesControllerV2Test.java b/...min/bag/covidcertificate/backend/verifier/ws/controller/ForeignRulesControllerV2Test.java
@@ -14,6 +14,8 @@
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+import ch.admin.bag.covidcertificate.backend.verifier.ws.utils.EtagUtil;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -63,9 +65,9 @@ public void foreignRulesTest() throws Exception {
     }
 
     @Test
-    @Disabled("calculating expected ETag from the file doesn't work for V2")
+    @Disabled
     public void notModifiedTest() throws Exception {
-        /*String expectedEtag = EtagUtil.getSha1HashForFiles(true, PATH_TO_VERIFICATION_RULES);
+        /*String expectedEtag = EtagUtil.getSha1HashForStrings()
 
         // get current etag
         MockHttpServletResponse response =