Don't block registar devices, and refactor a bit to enable Registrar unit testing

.github/workflows/testing.yml

@@ -89,7 +89,7 @@ jobs:
           fgrep 'Done with pubber run, exit code 193' pubber.out.2  # last_start auto-kill check
 
   udmi:
-    name: Sequence tests
+    name: Integration Tests
     runs-on: ubuntu-latest
     timeout-minutes: 20
     needs: redirect  # Access to UDMI-REFLECTOR is mutually exclusive

bin/test_registrar

@@ -19,30 +19,29 @@ exit_status=0
 echo Found ${devices} clean devices.
 [ "${devices}" == 4 ] || exit_status=1
 
-if [[ -f ${TEST_SITE}/site_metadata.json ]]; then
+# Test site_metadata settings for system.location.site.
+site=$(jq -r .system.location.site < ${TEST_SITE}/site_metadata.json)
 
-  # Test site_metadata settings for system.location.site.
-  site=$(jq -r .system.location.site < ${TEST_SITE}/site_metadata.json)
-
-  sm_devices=0
-  for name in ${TEST_SITE}/devices/* ; do
+sm_devices=0
+for name in ${TEST_SITE}/devices/* ; do
     if [[ -f ${name}/out/metadata_norm.json ]]; then
-      supplied_site=$(jq -r ".system.location.site" < ${name}/metadata.json)
-      # If no site value is supplied in per-device metadata, expect default.
-      if [[ "${supplied_site}" == "null" ]]; then
-        jq -e ".system.location.site == \"${site}\"" \
-            ${name}/out/metadata_norm.json > /dev/null \
-            && sm_devices=$[sm_devices+1]
-      else
-        jq -e ".system.location.site == \"${supplied_site}\"" \
-            ${name}/out/metadata_norm.json > /dev/null \
-            && sm_devices=$[sm_devices+1]
-      fi
+        supplied_site=$(jq -r ".system.location.site" < ${name}/metadata.json)
+        # If no site value is supplied in per-device metadata, expect default.
+        if [[ "${supplied_site}" == "null" ]]; then
+            jq -e ".system.location.site == \"${site}\"" \
+               ${name}/out/metadata_norm.json > /dev/null \
+                && sm_devices=$[sm_devices+1]
+        else
+            jq -e ".system.location.site == \"${supplied_site}\"" \
+               ${name}/out/metadata_norm.json > /dev/null \
+                && sm_devices=$[sm_devices+1]
+        fi
     fi
-  done
+done
 
-  echo Found ${sm_devices} devices with correct site_metadata values.
-  [ "${sm_devices}" == "${devices}" ] || exit_status=1
-fi
+echo Found ${sm_devices} devices with correct site_metadata values.
+[ "${sm_devices}" == "${devices}" ] || exit_status=1
 
+echo Done with registrar test, exit code $exit_status
 exit $exit_status
+

tests/downgrade.site/registration_summary.json

@@ -8,6 +8,6 @@
     "DWN-2" : "devices/DWN-2"
   },
   "Version" : {
-    "main" : "1.3.13-87-g8566aca1"
+    "main" : "unknown"
   }
 }

validator/bin/build

@@ -22,7 +22,7 @@ export JAVA_HOME=$JAVA_HOME_11_X64
 echo Building validataor in $PWD
 
 rm -rf build
-./gradlew shadow $check
+./gradlew shadow $check $*
 
 ls -l $jarfile
 

validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java

@@ -30,6 +30,7 @@
 import java.net.URI;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.AbstractMap.SimpleEntry;
 import java.util.ArrayList;
 import java.util.Base64;
 import java.util.Date;
@@ -77,7 +78,6 @@ public class Registrar {
   private final Map<String, JsonSchema> schemas = new HashMap<>();
   private final String generation = getGenerationString();
   private CloudIotManager cloudIotManager;
-  private File siteDir;
   private File schemaBase;
   private PubSubPusher updatePusher;
   private PubSubPusher feedPusher;
@@ -92,6 +92,8 @@ public class Registrar {
   private Map<String, Map<String, String>> lastErrorSummary;
   private boolean validateMetadata = false;
   private List<String> deviceList;
+  private boolean blockUnknown;
+  private File siteDir;
 
   /**
    * Main entry point for registrar.
@@ -131,7 +133,8 @@ static void processArgs(List<String> argList, Registrar registrar) {
           registrar.setValidateMetadata(true);
           break;
         case "--":
-          break;
+          registrar.setDeviceList(argList);
+          return;
         default:
           if (option.startsWith("-")) {
             throw new RuntimeException("Unknown cmdline option " + option);
@@ -177,6 +180,8 @@ private void setValidateMetadata(boolean validateMetadata) {
 
   private void setDeviceList(List<String> deviceList) {
     this.deviceList = deviceList;
+    Preconditions.checkNotNull(cloudIotManager, "cloudIotManager not yet defined");
+    blockUnknown = false;
   }
 
   private void setFeedTopic(String feedTopic) {
@@ -251,6 +256,7 @@ private void initializeCloudProject() {
     if (cloudIotManager.getUpdateTopic() != null) {
       updatePusher = new PubSubPusher(projectId, cloudIotManager.getUpdateTopic());
     }
+    blockUnknown = cloudIotManager.cloudIotConfig.block_unknown;
   }
 
   private String getGenerationString() {
@@ -264,11 +270,7 @@ private String getGenerationString() {
   }
 
   private void processDevices() {
-    processDevices(this.deviceList);
-  }
-
-  private void processDevices(List<String> devices) {
-    Set<String> deviceSet = calculateDevices(devices);
+    Set<String> deviceSet = calculateDevices();
     AtomicInteger updatedCount = new AtomicInteger();
     AtomicInteger processedCount = new AtomicInteger();
     try {
@@ -414,11 +416,11 @@ private void updateCloudIoT(LocalDevice localDevice) {
     }
   }
 
-  private Set<String> calculateDevices(List<String> devices) {
-    if (devices == null) {
+  private Set<String> calculateDevices() {
+    if (deviceList == null) {
       return null;
     }
-    return devices.stream().map(this::deviceNameFromPath).collect(Collectors.toSet());
+    return deviceList.stream().map(this::deviceNameFromPath).collect(Collectors.toSet());
   }
 
   private String deviceNameFromPath(String device) {
@@ -434,7 +436,7 @@ private String deviceNameFromPath(String device) {
 
   private ExceptionMap blockExtraDevices(Set<String> extraDevices) {
     ExceptionMap exceptionMap = new ExceptionMap("Block devices errors");
-    if (!cloudIotManager.cloudIotConfig.block_unknown) {
+    if (!blockUnknown) {
       return exceptionMap;
     }
     for (String extraName : extraDevices) {
@@ -534,9 +536,9 @@ private void shutdown() {
   private Set<String> fetchCloudDevices() {
     boolean requiresCloud = updateCloudIoT || (idleLimit != null);
     if (requiresCloud) {
-      Set<String> devices = cloudIotManager.fetchDeviceList();
-      System.err.printf("Fetched %d devices from cloud registry %s%n",
-          devices.size(), cloudIotManager.getRegistryPath());
+      Set<String> devices = cloudIotManager.fetchDeviceIds();
+      System.err.printf("Fetched %d devices from cloud registry %s%n", devices.size(),
+          cloudIotManager.getRegistryId());
       return devices;
     } else {
       System.err.println("Skipping remote registry fetch");
@@ -732,6 +734,10 @@ protected Map<String, JsonSchema> getSchemas() {
     return schemas;
   }
 
+  public List<Object> getMockActions() {
+    return cloudIotManager.getMockActions();
+  }
+
   class RelativeDownloader implements URIDownloader {
 
     @Override