Automatic sequence doc generator

.github/workflows/testing.yml

@@ -91,7 +91,7 @@ jobs:
   udmi:
     name: Sequence tests
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 20
     needs: redirect  # Access to UDMI-REFLECTOR is mutually exclusive
     steps:
       - uses: actions/checkout@v2
@@ -109,8 +109,8 @@ jobs:
         run: |
           bin/test_sequencer $GCP_TARGET_PROJECT
           more /tmp/sequencer.out
-          echo Comparing test run results with golden file:
-          diff -u /tmp/sequencer.out etc/sequencer.out && echo No output diff detected.
+          diff -u /tmp/sequencer.out etc/sequencer.out && echo No sequencer.out diff
+          diff -u /tmp/generated.md docs/specs/sequences/generated.md && echo No generated.md diff
       - name: telemetry validator
         env:
           GCP_TARGET_PROJECT: ${{ secrets.GCP_TARGET_PROJECT }}

.gitignore

@@ -25,7 +25,7 @@ credentials.json
 /venv/
 /local/
 .pubber.pid
-pubber.out.*
+pubber.out*
 __pycache__/
 /tests/downgrade.site/devices/*/out/
 

bin/gencode

@@ -11,6 +11,7 @@ fi
 bin/gencode_java
 bin/gencode_python gencode/python schema/*.json
 bin/gencode_docs
+bin/gencode_seq
 
 if [[ -n $check ]]; then
     echo Checking gencode docs links...

bin/gencode_docs

@@ -25,8 +25,8 @@ if [[ -n $1 ]]; then
     shift
 fi
 
-OUTPUT_DIR=$ROOT_DIR/gencode/docs
-TMP_DIR=$ROOT_DIR/tmp/schema
+OUTPUT_DIR=gencode/docs
+TMP_DIR=tmp/schema
 
 schema_files=`ls schema/*.json`
 

bin/gencode_seq

@@ -0,0 +1,44 @@
+#!/bin/bash -e
+
+ROOT_DIR=$(realpath $(dirname $0)/..)
+cd $ROOT_DIR
+
+SEQUENCE_MD=docs/specs/sequences/generated.md
+
+# Create doc of generated sequence steps
+prefix=sites/udmi_site_model/out/devices/AHU-1/tests
+if [[ -d $prefix ]]; then
+    echo Updating $SEQUENCE_MD from $prefix:
+
+    # Clear out existing generated sequences
+    sed -i '/<!-- START GENERATED,/q' $SEQUENCE_MD
+
+    sequences=$(cd $prefix; find . -name sequence.md | sort)
+
+    # Generate table of contents
+    for sequence in $sequences; do
+        directory=${sequence%/sequence.md}
+        name=${directory##.*/}
+        header=$(fgrep \#\# $prefix/$sequence) || true
+        if [[ -z $header ]]; then
+            echo "  $name: Invalid or missing header."
+            continue
+        fi
+        description=": $(sed -n -n '/^1\./q;p' $prefix/$sequence | fgrep -v \#\# | xargs echo)"
+        echo "* [${name}](#${name})${description%: }" >> $SEQUENCE_MD
+    done
+
+    # Add in specific test sequences
+    for sequence in $sequences; do
+        directory=${sequence%/sequence.md}
+        name=${directory##.*/}
+        header=$(fgrep \#\# $prefix/$sequence) || true
+        if [[ -z $header ]]; then
+            continue
+        fi
+        echo "  $name"
+        cat $prefix/$sequence >> $SEQUENCE_MD
+    done
+else
+    echo $prefix not found, skipping sequence generation.
+fi

bin/test_sequencer

@@ -55,7 +55,7 @@ if [[ $i -eq $WAITING ]]; then
     false
 fi
 
-bin/sequencer -vv $site_path $project_id $device_id $serial_no
+bin/sequencer -v $site_path $project_id $device_id $serial_no
 echo Completed execution of sequencer test run.
 
 sed -i -e 's/.*sequencer RESULT/RESULT/' /tmp/sequencer.out
@@ -70,4 +70,8 @@ else
     false
 fi
 
+generated=docs/specs/sequences/generated.md
+cp $generated /tmp/  # Save for test/comparison later
+bin/gencode_seq
+
 echo Done with base test_sequencer run.

dashboard/functions/index.js

@@ -87,22 +87,22 @@ function recordMessage(attributes, message) {
 }
 
 function sendCommand(registryId, deviceId, subFolder, message) {
-  return sendCommandStr(registryId, deviceId, subFolder, JSON.stringify(message));
+  return sendCommandStr(registryId, deviceId, subFolder, JSON.stringify(message), message.nonce);
 }
 
-function sendCommandStr(registryId, deviceId, subFolder, messageStr) {
+function sendCommandStr(registryId, deviceId, subFolder, messageStr, nonce) {
   return registry_promise.then(() => {
-    return sendCommandSafe(registryId, deviceId, subFolder, messageStr);
+    return sendCommandSafe(registryId, deviceId, subFolder, messageStr, nonce);
   });
 }
 
-function sendCommandSafe(registryId, deviceId, subFolder, messageStr) {
+function sendCommandSafe(registryId, deviceId, subFolder, messageStr, nonce) {
   const cloudRegion = registry_regions[registryId];
 
   const formattedName =
         iotClient.devicePath(PROJECT_ID, cloudRegion, registryId, deviceId);
 
-  console.log('command', formattedName, subFolder);
+  console.log('command', subFolder, nonce, formattedName);
 
   const binaryData = Buffer.from(messageStr);
   const request = {
@@ -364,6 +364,7 @@ async function modify_device_config(registryId, deviceId, subFolder, subContents
       delete subContents.version;
       delete subContents.timestamp;
       newConfig[subFolder] = subContents;
+      newConfig.nonce = subContents.nonce;
     } else {
       if (!newConfig[subFolder]) {
         console.log('Config target already null', subFolder, version, startTime);
@@ -424,6 +425,7 @@ function update_device_config(message, attributes, preVersion) {
   const normalJson = extraField !== 'break_json';
   console.log('Config extra field is ' + extraField + ' ' + normalJson);
 
+  const nonce = message.nonce;
   const msgString = normalJson ? JSON.stringify(message) :
         '{ broken because extra_field == ' + message.system.extra_field;
   const binaryData = Buffer.from(msgString);
@@ -445,7 +447,7 @@ function update_device_config(message, attributes, preVersion) {
 
   return iotClient
     .modifyCloudToDeviceConfig(request)
-    .then(() => sendCommandStr(REFLECT_REGISTRY, registryId, commandFolder, msgString));
+    .then(() => sendCommandStr(REFLECT_REGISTRY, registryId, commandFolder, msgString, nonce));
 }
 
 function consolidate_config(registryId, deviceId, subFolder) {

docs/specs/sequences/config.md

@@ -93,4 +93,4 @@ When this happens, the device should:
   * Silently ignore the extra field(s) as if there was nothing wrong.
   * Generally can be implemented by setting some JSON parsers to "ignore unrecognized fields"
 
-
+

docs/specs/sequences/generated.md

@@ -0,0 +1,116 @@
+[**UDMI**](../../../) / [**Docs**](../../) / [**Specs**](../) / [**Sequences**](./) / [Generated](#)
+
+# Generated sequences
+
+These are the exact sequences being checked by the sequence tool. They are programaticaly generated
+so maybe a bit cryptic, but they accurately represent the specific steps required for each test.
+
+<!-- To regenerate the contents of this file below, use bin/test_sequencer and bin/gencode_seq -->
+
+<!-- START GENERATED, do not edit anything after this line! -->
+* [broken_config](#broken_config): Check that the device correctly handles a broken (non-json) config message.
+* [device_config_acked](#device_config_acked): Check that the device MQTT-acknowledges a sent config.
+* [extra_config](#extra_config): Check that the device correctly handles an extra out-of-schema field
+* [periodic_scan](#periodic_scan)
+* [self_enumeration](#self_enumeration)
+* [single_scan](#single_scan)
+* [system_last_update](#system_last_update): Check that last_update state is correctly set in response to a config update.
+* [system_min_loglevel](#system_min_loglevel): Check that the min log-level config is honored by the device.
+* [valid_serial_no](#valid_serial_no)
+* [writeback_states](#writeback_states)
+
+## broken_config
+
+Check that the device correctly handles a broken (non-json) config message.
+
+1. Update config:
+    * Set `system.min_loglevel` = `100`
+1. Wait for no interesting status
+1. Wait for clean config/state synced
+1. Wait for state synchronized
+1. initial stable_config matches last_config
+1. Wait for log category `system.config.receive` level `DEBUG`
+1. Wait for has interesting status
+1. Wait for log category `system.config.parse` level `ERROR`
+1. Check has not logged category `system.config.apply` level `NOTICE` (**incomplete!**)
+1. Force reset config
+1. Wait for log category `system.config.receive` level `DEBUG`
+1. Wait for no interesting status
+1. Wait for last_config updated
+1. Wait for log category `system.config.apply` level `NOTICE`
+1. Wait for log category `system.config.parse` level `DEBUG`
+
+## device_config_acked
+
+Check that the device MQTT-acknowledges a sent config.
+
+1. Wait for config acked
+
+## extra_config
+
+Check that the device correctly handles an extra out-of-schema field
+
+1. Update config:
+    * Set `system.min_loglevel` = `100`
+1. Wait for last_config not null
+1. Wait for system operational
+1. Wait for no interesting status
+1. Wait for log category `system.config.receive` level `DEBUG`
+1. Wait for last_config updated
+1. Wait for system operational
+1. Wait for no interesting status
+1. Wait for log category `system.config.parse` level `DEBUG`
+1. Wait for log category `system.config.apply` level `NOTICE`
+1. Wait for log category `system.config.receive` level `DEBUG`
+1. Wait for last_config updated again
+1. Wait for system operational
+1. Wait for no interesting status
+1. Wait for log category `system.config.parse` level `DEBUG`
+1. Wait for log category `system.config.apply` level `NOTICE`
+
+## periodic_scan
+
+1. Update config:
+    * Add `discovery` = { "families": {  } }
+1. Wait for all scans not active
+1. Wait for scan iterations
+
+## self_enumeration
+
+1. Wait for enumeration not active
+1. Update config to discovery generation:
+    * Add `discovery` = { "enumeration": { "generation": _generation start time_ } }
+1. Wait for enumeration generation
+1. Wait for enumeration still not active
+
+## single_scan
+
+1. Update config:
+    * Add `discovery` = { "families": {  } }
+1. Wait for all scans not active
+1. Wait for scheduled scan start
+1. Wait for scan activation
+1. Wait for scan completed
+
+## system_last_update
+
+Check that last_update state is correctly set in response to a config update.
+
+1. Wait for state last_config matches config timestamp
+
+## system_min_loglevel
+
+Check that the min log-level config is honored by the device.
+
+1. Check has not logged category `system.config.apply` level `NOTICE` (**incomplete!**)
+1. Update config:
+    * Set `system.min_loglevel` = `400`
+1. Update config:
+    * Set `system.min_loglevel` = `200`
+1. Wait for log category `system.config.apply` level `NOTICE`
+
+## valid_serial_no
+
+
+## writeback_states
+

docs/specs/sequences/readme.md

@@ -2,14 +2,14 @@
 
 # Sequences
 
-Sequences are defined device behavior which are formulated from a series of events or communication messages
+Sequences are defined device behavior which are formulated from a series of events or communication messages.
+They can be tested using the [`sequencer` tool](../../tools/sequencer.md). The
+[generated steps](generated.md) for each test detail exactly what the tool actually checks.
 
-## Defined Sequences
+## Functional Sequences
 
-- [Config and State](config.md)
-- [Discovery](discovery.md)
-- [Writeback](writeback.md)
+More explanatory detail for some of the key functional specifications:
 
-## Validation
-
-Sequences can be validated using the [`sequencer` sequence validator tool](../../tools/sequencer.md)
+- [Config and State](config.md): Basic handling of _config_ and _state_ messages.
+- [Discovery](discovery.md): Flow for on-prem network and point discovery.
+- [Writeback](writeback.md): Ability to control on-prem resouces from the cloud.

docs/tools/README.md

@@ -4,11 +4,12 @@
 
 - [keygen](keygen.md) - a script to generate an RSA or ES key for single devices
 - [pagent](pagent.md) - a tool for automated cloud provisioning of devices (GCP)
-- [pubber](pubber.md) - a sample implementation of a client-side 'device' that implements the UDMI schema.
+- [pubber](pubber.md) - a sample implementation of a client-side 'device' that implements the UDMI schema
 - [registrar](registrar.md) - a utility to register and updates devices in Cloud IoT Core (GCP)
 - [sequencer](sequencer.md) - a utility to validate device [sequences](../specs/sequences/) (GCP)
 - [validator](validator.md) - a utility for validating messages (GCP)
+- [gcloud](gcloud.md) - various tips and tricks for working with gcloud on GCP
 
 ## Setup
 
-[Setup instructions (GCP)](setup.md) are provided to set up the local environment for using tools
+[Setup instructions (GCP)](setup.md) are provided to set up the local environment for using tools.

docs/tools/gcloud.md

@@ -0,0 +1,16 @@
+[**UDMI**](../../) / [**Docs**](../) / [**Tools**](./) / [Registrar](#)
+
+Various notes and tips and tricks for working with GCP. These are more or less hints as to what can be done, but don't expect
+them to work out-of-the-box without a deeper understanding of what's going on!
+
+# Viewing subscription
+
+bin/pull_message:gcloud --format=json --project=$project_id pubsub subscriptions pull $subscription --auto-ack
+
+# View cloud function logs
+
+gcloud --project=bos-peringknife-ci functions logs read udmi_config --sort-by=time_utc --limit=1000
+
+# Update devices' GCP IoT Core configuration (sent down to the device)
+
+bin/reset_config