Pubber Config Redirect (#409)

.gencode_hash.txt

@@ -12,6 +12,7 @@ f48026471ae3cd7867bce416dc21c2fb728f48d8476a8d6e95f6acaf1d8b6cf3  gencode/docs/i
 43019239edfdad71647c40d254b9b3aec68f3deb0c808ba636530c1cf9985fe0  gencode/docs/state.html
 d39d7fe37a41c74a40080af7b0a429d201ab1fdff7444428c4b98eb7b38c332b  gencode/java/udmi/schema/Asset.java
 0825a5cec83003bb0a6488c4ed7010a04ae0d3848ef36fe01bb4e6718ba7b96d  gencode/java/udmi/schema/Aux.java
+df118b8e5abebac38579027b5a17192163f0cc97e55a8e44b847b1a965263636  gencode/java/udmi/schema/Blob.java
 d5adb804697243f97cdd8589750401654f3fab075a9aeac4f2851e46695ef11d  gencode/java/udmi/schema/BlobBlobsetConfig.java
 2c03651cb2ecda072b1418222eebb5560185669f8ffdd03021ad5ad8ff7ba3b0  gencode/java/udmi/schema/BlobBlobsetState.java
 d2c5b5aae8db27b68104fc83a1f38de0a3f1b5d683f2b13599adf24e96c7d124  gencode/java/udmi/schema/BlobsetConfig.java

.github/workflows/testing.yml

@@ -52,27 +52,38 @@ jobs:
         run: |
           bin/setup_base
           bin/clone_model
-      - name: pubber run
+      - name: pubber runs
         env:
           GCP_TARGET_PROJECT: ${{ secrets.GCP_TARGET_PROJECT }}
         if: "${{ env.GCP_TARGET_PROJECT != '' }}"
         run: |
           bin/test_redirect $GCP_TARGET_PROJECT
-      - name: pubber checks
+      - name: pubber cmd checks
         env:
           GCP_TARGET_PROJECT: ${{ secrets.GCP_TARGET_PROJECT }}
         if: "${{ env.GCP_TARGET_PROJECT != '' }}"
         run: |
-          # Simple checks that a redirect happened and failed
-          set +x
-          fgrep registries/ZZ-TRI-FECTA/devices/GAT-123 pubber.out
-          fgrep 'system.config.parse success' pubber.out
-          fgrep registries/missing/devices/GAT-123 pubber.out
-          fgrep 'Not authorized to connect' pubber.out
-      - name: pubber logs
-        if: ${{ always() }}
+          echo ::::::: pubber.out.1
+          cat pubber.out.1
+          echo :::::::
+          echo Simple checks that a redirect happened and failed
+          fgrep registries/ZZ-TRI-FECTA/devices pubber.out.1
+          fgrep 'system.config.parse success' pubber.out.1
+          fgrep registries/missing/devices pubber.out.1
+          fgrep 'Not authorized to connect' pubber.out.1
+      - name: pubber config checks
+        env:
+          GCP_TARGET_PROJECT: ${{ secrets.GCP_TARGET_PROJECT }}
+        if: "${{ env.GCP_TARGET_PROJECT != '' }}"
         run: |
-          cat pubber.out || true
+          echo ::::::: pubber.out.2
+          cat pubber.out.2
+          echo :::::::
+          echo Check the redirect-by-config setup
+          fgrep registries/ZZ-TRI-FECTA/devices pubber.out.2
+          fgrep 'system.config.parse success' pubber.out.2
+          fgrep registries/reconfigure/devices pubber.out.2
+          fgrep 'Not authorized to connect' pubber.out.2
     
   udmi:
     name: Sequence tests

bin/gencode_java

@@ -31,8 +31,8 @@ echo Generating code in $OUTDIR
 $JARBIN $JOPTS --source . --target $OUTDIR
 
 # There is no way to specify enum constants in generated code, so just hack it in manually.
-echo Copying Level.java
-cp -n $ROOT_DIR/etc/Level.java $OUTDIR/udmi/schema/Level.java
+echo Copying shared constants $ROOT_DIR/etc/*.java
+cp -n $ROOT_DIR/etc/*.java $OUTDIR/udmi/schema/
 
 echo Cleaning up __ classes...
 find $OUTDIR -name \*__\*.java | xargs rm

bin/pubber

@@ -19,7 +19,7 @@ if [ ! -f $site_path/cloud_iot_config.json ]; then
 fi
 
 echo Building pubber...
-$ROOT_DIR/pubber/bin/build > /dev/null
+$ROOT_DIR/pubber/bin/build
 
 echo Running tools version `(cd $ROOT_DIR; git describe)`
 

bin/reset_config

@@ -2,25 +2,29 @@
 
 ROOT=$(realpath $(dirname $0)/..)
 
-if [[ $# != 3 ]]; then
-    echo Usage: $0 site_dir project_id device_id
+if [[ $# != 3 && $# != 4 ]]; then
+    echo Usage: $0 site_dir project_id device_id [config_file]
     false
 fi
 
 site_dir=$(realpath $1)
 project_id=$2
 device_id=$3
 shift 3
-cd ${ROOT}
 
-pwd
+config_file=generated_config.json
+if [[ -n $1 ]]; then
+    config_file=$1
+    shift
+fi
+
+cd ${ROOT}
 
-device_config=/tmp/${device_id}_config.json
-cp ${site_dir}/devices/${device_id}/out/generated_config.json ${device_config}
+dst_config=/tmp/${device_id}_config.json
+src_config=${site_dir}/devices/${device_id}/out/$config_file
 now_date=$(python3 -c 'import datetime; print(datetime.datetime.utcnow().isoformat() + "Z")')
 echo Setting config timestamp ${now_date}
-sponge < /dev/null || echo sponge command not found, might need workaround for osx or other
-jq .timestamp=\"${now_date}\" < ${device_config} | sponge ${device_config}
+jq .timestamp=\"${now_date}\" < ${src_config} > ${dst_config}
 
 echo Resetting device ${device_id} config...
-validator/bin/reflector ${site_dir} ${project_id} ${device_id} update/config:${device_config}
+validator/bin/reflector ${site_dir} ${project_id} ${device_id} update/config:${dst_config}

bin/setup_base

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-sudo apt-get install -y hxtools moreutils
+sudo apt-get install -y hxtools moreutils expect
 
 python3 -m venv venv
 

bin/test_redirect

@@ -19,26 +19,70 @@ else
 fi
 
 site_path=sites/udmi_site_model
-device_id=AHU-22 # Static device for testing. Needs to be different than other tests scripts so as not to conflict during CI.
+device_id=AHU-1 # Static device for testing. Needs to be different than other tests scripts so as not to conflict during CI.
+mkdir -p out
 
 serial_no=sequencer-$RANDOM
 echo Using pubber with serial $serial_no
 
+site_config=$site_path/cloud_iot_config.json
+cloud_region=$(jq -r .cloud_region $site_config)
+registry_id=$(jq -r .registry_id $site_config)
+
 PUBBER_OUT=pubber.out
 
-# Ideally use lock files, not grep-and-kill...
-pids=`ps ax | fgrep pubber | fgrep java | awk '{print $1}'`
-if [[ -n $pids ]]; then
-    echo Killing pubber pids $pids
-    kill $pids
-fi
+echo Killing running pubber instances...
+ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true
 
 bin/reset_config $site_path $project_id $device_id
 
-echo Writing pubber output to $PUBBER_OUT
-echo bin/pubber $site_path $project_id $device_id $serial_no
+echo Running pubber with redirectRegistry on timeout
+timeout 3m unbuffer bin/pubber $site_path $project_id $device_id $serial_no redirectRegistry=missing > $PUBBER_OUT.1 2>&1 || true
+
+echo Done with timeout, running pubber in background...
+bin/pubber $site_path $project_id $device_id $serial_no > $PUBBER_OUT.2 2>&1 &
+
+# Wait for initial connection, then reset config to redirect
+sleep 20
+
+cat <<EOF > out/endpoint.json
+{
+  "projectId": "$project_id",
+  "registryId": "reconfigure",
+  "cloudRegion": "$cloud_region"
+}
+EOF
+
+base64=$(base64 -w 0 out/endpoint.json)
 
-result=0
-timeout 5m bin/pubber $site_path $project_id $device_id $serial_no redirectRegistry=missing > $PUBBER_OUT 2>&1 || result=$?
+cat <<EOF > out/blobs.json
+{
+  "blobset": {
+    "blobs": {
+      "_iot_config": {
+        "phase": "final",
+        "base64": "$base64"
+      }
+    }
+  }
+}
+EOF
+
+out_base=$site_path/devices/$device_id/out
+# Merge JSON files together into new redirect config
+jq -s '.[0] * .[1]' $out_base/generated_config.json out/blobs.json > $out_base/redirect_config.json
+
+echo New redirection config:
+cat /tmp/${device_id}_config.json
+
+echo bin/reset_config $site_path $project_id $device_id redirect_config.json
+bin/reset_config $site_path $project_id $device_id redirect_config.json
+
+echo Let pubber react to the new configuration...
+sleep 20
+
+# Ideally use lock files, not grep-and-kill...
+echo Killing running pubber instances...
+ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true
 
-echo Pubber exit code $result
+echo Done with redirect test.

bin/test_sequencer

@@ -32,6 +32,8 @@ if [[ -n $pids ]]; then
     kill $pids
 fi
 
+bin/reset_config $site_path $project_id $device_id
+
 pubber/bin/build
 
 echo Writing pubber output to $PUBBER_OUT

bin/test_validator

@@ -15,6 +15,7 @@ shift
 
 site_path=sites/udmi_site_model
 device_id=AHU-1
+mkdir -p out
 
 serial_no=validator-$RANDOM
 echo Using pubber with serial $serial_no
@@ -23,12 +24,9 @@ PUBBER_OUT=pubber.out
 VALIDATOR_OUT=validator.out
 WAITING=10
 
-vids=`ps ax | fgrep validator | fgrep java | awk '{print $1}'`
-pids=`ps ax | fgrep pubber | fgrep java | awk '{print $1}'`
-if [[ -n "$pids$vids" ]]; then
-    echo Killing pids $pids $vids
-    kill $pids $vids
-fi
+echo Killing running pubber/validator instances...
+ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true
+ps ax | fgrep validator | fgrep java | awk '{print $1}' | xargs kill || true
 
 rm -rf $site_path/out/devices
 
@@ -40,6 +38,11 @@ pubber/bin/build
 
 bin/reset_config $site_path $project_id $device_id
 
+bin/validator $site_path $project_id &
+vpid=$!
+sleep 10
+echo Started validator pid $vpid
+
 echo Starting validator, output in $VALIDATOR_OUT
 bin/validator $site_path $project_id > $VALIDATOR_OUT 2>&1 &
 for i in `seq 1 $WAITING`; do
@@ -79,12 +82,9 @@ fi
 echo Waiting for system to run for a bit...
 timeout 60s tail -f $VALIDATOR_OUT || true
 
-vids=`ps ax | fgrep validator | fgrep java | awk '{print $1}'`
-pids=`ps ax | fgrep pubber | fgrep java | awk '{print $1}'`
-if [[ -n "$pids$vids" ]]; then
-    echo Killing pids $pids $vids
-    kill $pids $vids
-fi
+echo Killing running pubber/validator instances...
+ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true
+ps ax | fgrep validator | fgrep java | awk '{print $1}' | xargs kill || true
 
 outfiles=`find $site_path/out/devices -name \*.out | sort` || true
 if [[ -z $outfiles ]]; then

dashboard/functions/index.js

@@ -327,28 +327,35 @@ function parse_old_config(oldConfig, resetConfig) {
 
 async function modify_device_config(registryId, deviceId, subFolder, subContents, startTime) {
   const [oldConfig, version] = await get_device_config(registryId, deviceId);
+  var newConfig;
 
-  const resetConfig = subFolder == 'system' && subContents && subContents.extra_field == 'reset_config';
-  const newConfig = parse_old_config(oldConfig, resetConfig);
-  if (newConfig === null) {
-    return;
-  }
-
-  newConfig.version = UDMI_VERSION;
-  newConfig.timestamp = currentTimestamp();
-
-  console.log('Config modify version', subFolder, version, startTime);
-  if (subContents) {
-    delete subContents.version;
-    delete subContents.timestamp;
-    newConfig[subFolder] = subContents;
+  if (subFolder == 'update') {
+    console.log('Config replace version', version, startTime);
+    newConfig = subContents;
   } else {
-    if (!newConfig[subFolder]) {
-      console.log('Config target already null', subFolder, version, startTime);
+    const resetConfig = subFolder == 'system' && subContents && subContents.extra_field == 'reset_config';
+    newConfig = parse_old_config(oldConfig, resetConfig);
+    if (newConfig === null) {
       return;
     }
-    delete newConfig[subFolder];
+
+    newConfig.version = UDMI_VERSION;
+    newConfig.timestamp = currentTimestamp();
+
+    console.log('Config modify version', subFolder, version, startTime);
+    if (subContents) {
+      delete subContents.version;
+      delete subContents.timestamp;
+      newConfig[subFolder] = subContents;
+    } else {
+      if (!newConfig[subFolder]) {
+        console.log('Config target already null', subFolder, version, startTime);
+        return;
+      }
+      delete newConfig[subFolder];
+    }
   }
+
   const attributes = {
     projectId: PROJECT_ID,
     cloudRegion: registry_regions[registryId],

etc/Blob.java

@@ -0,0 +1,8 @@
+package udmi.schema;
+
+// This class is manually curated and then copied into the gencode directory. Look for the
+// proper source and don't be fooled! These are general constants for blob management.
+public abstract class Blob {
+    public static final String FINAL_PHASE = "final";
+    public static final String IOT_CONFIG_BLOB = "_iot_config";
+}