Fixes for gateway tests (#831)

bin/pull_messages

@@ -12,37 +12,36 @@ read < /tmp/namespace_tmp gcp project_id region cluster namespace
 
 subscription=$namespace~debug
 tmp_file=/tmp/captured.json
+pull_limit=100
 
 while true; do
-    gcloud --format=json --project=$project_id pubsub subscriptions pull $subscription --auto-ack > $tmp_file || true
-
-    subType=$(jq -r '.[0].message.attributes.subType' $tmp_file 2> /dev/null)
-    subFolder=$(jq -r '.[0].message.attributes.subFolder' $tmp_file 2> /dev/null)
-    deviceId=$(jq -r '.[0].message.attributes.deviceId' $tmp_file 2> /dev/null)
-    registryId=$(jq -r '.[0].message.attributes.deviceRegistryId' $tmp_file 2> /dev/null)
-    timestamp=$(jq -r '.[0].message.publishTime' $tmp_file 2> /dev/null)
-    raw_data=$(jq -r .[0].message.data $tmp_file)
-    # There's two different base64 formats, so replace - with + to handle both.
-    data=$(echo $raw_data | tr - + | base64 --decode)
-
-    if [[ $raw_data == null ]]; then
-        continue
-    fi
-
-    if [[ -z $data ]]; then
-        echo Bad/empty message data: $raw_data
-    fi
-
-    if [[ $subType == null ]]; then
-        subType=event
-    fi
-
-    if [[ $subFolder != null ]]; then
+    gcloud --format=json --project=$project_id pubsub subscriptions pull $subscription --limit $pull_limit --auto-ack > $tmp_file || true
+
+    for index in $(seq 0 $((pull_limit-1))); do
+        subType=$(jq -r .[$index].message.attributes.subType $tmp_file 2> /dev/null)
+        subFolder=$(jq -r .[$index].message.attributes.subFolder $tmp_file 2> /dev/null)
+        deviceId=$(jq -r .[$index].message.attributes.deviceId $tmp_file 2> /dev/null)
+        registryId=$(jq -r .[$index].message.attributes.deviceRegistryId $tmp_file 2> /dev/null)
+        timestamp=$(jq -r .[$index].message.publishTime $tmp_file 2> /dev/null)
+        raw_data=$(jq -r .[$index].message.data $tmp_file)
+        # There's two different base64 formats, so replace - with + to handle both.
+        data=$(echo $raw_data | tr - + | base64 --decode)
+
+        if [[ $raw_data == null ]]; then
+            break
+        fi
+
+        if [[ -z $data ]]; then
+            echo Bad/empty message data: $raw_data
+        fi
+
+        if [[ $subType == null ]]; then
+            subType=event
+        fi
+
         out_file=$ROOT/out/registries/$registryId/devices/$deviceId/${timestamp}_${subFolder}_${subType}.json
         echo $out_file
         mkdir -p $(dirname $out_file)
         echo $data | jq . > $out_file
-    else
-        echo No matching messages found.
-    fi
+    done
 done

bin/test_itemized

@@ -6,12 +6,12 @@ cd $UDMI_ROOT
 source etc/shell_common.sh
 
 if [[ $# != 1 && $# != 2 ]]; then
-    echo Usage: $0 PROJECT_ID [target test]
+    echo Usage: $0 PROJECT_ID [target_index]
     false
 fi
 PROJECT_ID=$1
 shift 1
-TARGET_TEST=$1
+TARGET_INDEX=$1
 
 SITE_PATH=sites/udmi_site_model
 DEVICE_ID=AHU-1
@@ -48,9 +48,7 @@ while read -u 7 action test_name pubber_opts; do
     SEQUENCER_OUT=out/sequencer.out
     SCHEMA_OUT=out/schema.out
 
-    if [[ $action == TEST && -n $TARGET_TEST && $TARGET_TEST != $test_name ]]; then
-        continue
-    fi
+    [[ -n $TARGET_INDEX && $action == TEST && $TARGET_INDEX != $test_marker ]] && continue
 
     if [[ $action == WITH ]]; then
         DEVICE_ID=$test_name
@@ -128,10 +126,13 @@ while read -u 7 action test_name pubber_opts; do
 
 done 7< $INPUT_FILE
 
+[[ ! -f $RESULTS_OUT ]] && fail Not output file, likely no tests executed.
+
 sed -E -i $RESULTS_OUT \
-    -e 's/(Pipeline type event error: While processing message ).*/\1REDACTED/'
+    -e 's/(Pipeline type event error: While processing message ).*/\1REDACTED/' \
+    -e 's/ 202[-0-9T:]+Z / TIMESTAMP /g'
 
-if [[ -z $TARGET_TEST && -z $MATRIX_SHARD_COUNT ]]; then
+if [[ -z $TARGET_INDEX && -z $MATRIX_SHARD_COUNT ]]; then
     bin/test_itemcheck
 else
     echo Checking for correctness of completed tests in $GOLDEN_FILE...

bin/test_sequcheck

@@ -17,7 +17,7 @@ if [[ -z $suffix ]]; then
     echo '####################' Finished sequencer cache test: $failures
 fi
 
-cat out/sequencer${suffix}.out | sort -k 3,4 | sponge out/sequencer{$suffix}.out
+cat out/sequencer${suffix}.out | sort -k 3,4 | sponge out/sequencer${suffix}.out
 
 echo Comparing diff out/sequencer${suffix}.out etc/sequencer${suffix}.out
 diff -u out/sequencer${suffix}.out $UDMI_ROOT/etc/sequencer${suffix}.out || failures+="sequencer "

docs/specs/sequences/generated.md

@@ -31,6 +31,7 @@ Some caveats:
 -->
 
 <!-- START GENERATED, do not edit anything after this line! -->
+* [bad_target_family](#bad_target_family-preview): Error handling for badly formed target address family
 * [broken_config](#broken_config-beta): Check that the device correctly handles a broken (non-json) config message.
 * [config_logging](#config_logging-beta): Check that the device publishes minimum required log entries when receiving config
 * [device_config_acked](#device_config_acked-beta): Check that the device MQTT-acknowledges a sent config.
@@ -46,7 +47,6 @@ Some caveats:
 * [family_ipv4_addr](#family_ipv4_addr-preview)
 * [family_ipv6_addr](#family_ipv6_addr-preview)
 * [feature_enumeration](#feature_enumeration-preview): Check enumeration of device features
-* [gateway_attach_handling](#gateway_attach_handling-preview): Check adequate logging for gateway detach, errors, and reattach
 * [gateway_proxy_events](#gateway_proxy_events-beta): Check that a gateway proxies pointset events for indicated devices
 * [pointset_publish](#pointset_publish-beta): Check that a device publishes pointset events
 * [pointset_publish_interval](#pointset_publish_interval-beta): Check handling of sample_rate_sec and sample_limit_sec
@@ -57,6 +57,12 @@ Some caveats:
 * [system_last_update](#system_last_update-stable): Check that last_update state is correctly set in response to a config update.
 * [valid_serial_no](#valid_serial_no-beta)
 
+## bad_target_family (PREVIEW)
+
+Error handling for badly formed target address family
+
+1. Test skipped: Not a proxied device
+
 ## broken_config (BETA)
 
 Check that the device correctly handles a broken (non-json) config message.
@@ -287,12 +293,6 @@ Check enumeration of device features
 1. Check that all enumerated features are official buckets
 1. Check that no point enumeration
 
-## gateway_attach_handling (PREVIEW)
-
-Check adequate logging for gateway detach, errors, and reattach
-
-1. Test skipped: Not a gateway
-
 ## gateway_proxy_events (BETA)
 
 Check that a gateway proxies pointset events for indicated devices
@@ -324,35 +324,37 @@ Check handling of sample_rate_sec and sample_limit_sec
 
 Check that pointset state does not report an unconfigured point
 
-1. Wait for pointset state reports same points as defined in config
-1. Wait for pointset event contains correct points with present_value
+1. Wait for pointset state matches config
+1. Wait for pointset event contains correct points
 1. Update config before pointset status does not contain removed point:
     * Remove `pointset.points[random_point]`
 1. Wait for pointset status does not contain removed point
-1. Wait for pointset state reports same points as defined in config
-1. Wait for pointset event contains correct points with present_value
+1. Wait for pointset state matches config
+1. Wait for pointset event contains correct points
 1. Update config before pointset status contains removed point:
     * Add `pointset.points[random_point]` = point configuration
 1. Wait for pointset status contains removed point
-1. Wait for pointset state reports same points as defined in config
-1. Wait for pointset event contains correct points with present_value
+1. Wait for pointset state matches config
+1. Wait for pointset event contains correct points
 
 ## pointset_request_extraneous (BETA)
 
 Check error when pointset configuration contains extraneous point
 
-1. Wait for pointset state reports same points as defined in config
-1. Wait for pointset event contains correct points with present_value
+1. Update config Before pointset state matches config:
+    * Add `pointset.sample_rate_sec` = `10`
+1. Wait for pointset state matches config
+1. Wait for pointset event contains correct points
 1. Update config before pointset status contains extraneous point error:
     * Add `pointset.points[extraneous_point]` = point configuration
 1. Wait for pointset status contains extraneous point error
-1. Wait for pointset state reports same points as defined in config
-1. Wait for pointset event contains correct points with present_value
+1. Wait for pointset state matches config
+1. Wait for pointset event contains correct points
 1. Update config before pointset status removes extraneous point error:
     * Remove `pointset.points[extraneous_point]`
 1. Wait for pointset status removes extraneous point error
-1. Wait for pointset state reports same points as defined in config
-1. Wait for pointset event contains correct points with present_value
+1. Wait for pointset state matches config
+1. Wait for pointset event contains correct points
 
 ## state_make_model (BETA)
 

etc/sequencer.out

@@ -12,8 +12,8 @@ RESULT pass enumeration multi_enumeration ALPHA 5/5 Sequence complete
 RESULT pass enumeration.families family_enumeration ALPHA 5/5 Sequence complete
 RESULT pass enumeration.features feature_enumeration PREVIEW 5/5 Sequence complete
 RESULT pass enumeration.pointset pointset_enumeration ALPHA 5/5 Sequence complete
-RESULT skip gateway bad_target_family ALPHA 0/0 Not a proxied device
-RESULT skip gateway gateway_attach_handling PREVIEW 0/0 Not a gateway
+RESULT skip gateway bad_target_family PREVIEW 0/0 Not a proxied device
+RESULT skip gateway gateway_attach_handling ALPHA 0/0 Not a gateway
 RESULT skip gateway gateway_proxy_events BETA 0/0 Not a gateway
 RESULT pass pointset pointset_publish BETA 5/5 Sequence complete
 RESULT pass pointset pointset_publish_interval BETA 5/5 Sequence complete

etc/sequencer_nostate.out

@@ -6,7 +6,7 @@ RESULT skip endpoint.config endpoint_failure_and_restart PREVIEW 0/0 State testi
 RESULT skip endpoint.config endpoint_redirect_and_restart PREVIEW 0/0 State testing disabled
 RESULT skip enumeration empty_enumeration PREVIEW 0/0 State testing disabled
 RESULT skip enumeration.features feature_enumeration PREVIEW 0/0 State testing disabled
-RESULT skip gateway gateway_attach_handling PREVIEW 0/0 Not a gateway
+RESULT skip gateway bad_target_family PREVIEW 0/0 Not a proxied device
 RESULT skip gateway gateway_proxy_events BETA 0/0 Not a gateway
 RESULT pass pointset pointset_publish BETA 5/5 Sequence complete
 RESULT pass pointset pointset_publish_interval BETA 5/5 Sequence complete

etc/test_itemized.out

@@ -1,6 +1,6 @@
 01 RESULT fail system device_config_acked BETA 0/5 Timeout waiting for config acked
-02 RESULT fail pointset pointset_request_extraneous BETA 0/5 Timeout waiting for pointset event contains correct points with present_value
-03 RESULT fail pointset pointset_remove_point BETA 0/5 Timeout waiting for pointset state reports same points as defined in config
+02 RESULT fail pointset pointset_request_extraneous BETA 0/5 Timeout waiting for config TIMESTAMP event TIMESTAMP differences: llama
+03 RESULT fail pointset pointset_remove_point BETA 0/5 Timeout waiting for config TIMESTAMP state TIMESTAMP differences: filter_alarm_pressure_status
 04 RESULT fail pointset pointset_remove_point BETA 0/5 Timeout waiting for no applicable system status
 05 RESULT fail system too_much_state ALPHA 0/5 Failed check that No more than 6 state updates in 60s
 06 RESULT fail enumeration.features feature_enumeration PREVIEW 0/5 Failed check that feature enumeration matches metadata; missing { enumeration }, extra { unknown }
@@ -16,9 +16,9 @@
 14 RESULT pass system broken_config BETA 5/5 Sequence complete
 15 CPBLTY skip system broken_config.logging ALPHA 0/1 Never executed
 15 RESULT fail system broken_config BETA 0/5 Timeout waiting for no applicable system status
-16 CPBLTY fail system broken_config.logging ALPHA 0/1 Timeout after 30s waiting for log category `system.config.apply` level `NOTICE` to be logged
+16 CPBLTY fail system broken_config.logging ALPHA 0/1 While log category `system.config.apply` level `NOTICE` to be logged
 16 RESULT pass system broken_config BETA 4/5 Sequence complete
-17 CPBLTY fail system broken_config.logging ALPHA 0/1 Timeout after 30s waiting for log category `system.config.receive` level `DEBUG` to be logged
+17 CPBLTY fail system broken_config.logging ALPHA 0/1 While log category `system.config.receive` level `DEBUG` to be logged
 17 RESULT fail system broken_config BETA 0/5 expected:<system.config.[parse]> but was:<system.config.[apply]>
 18 RESULT pass system system_last_update STABLE 5/5 Sequence complete
 19 RESULT pass system system_last_update STABLE 5/5 Sequence complete
@@ -32,4 +32,4 @@
 36 RESULT pass gateway gateway_proxy_events BETA 5/5 Sequence complete
 37 RESULT fail gateway gateway_proxy_events BETA 0/5 Received state update with no-state device
 41 RESULT skip system device_config_acked BETA 0/0 No config check for proxy device
-42 RESULT pass gateway bad_target_family ALPHA 5/5 Sequence complete
+42 RESULT pass gateway bad_target_family PREVIEW 5/5 Sequence complete

validator/sequences/bad_target_family/sequence.md

@@ -1,5 +1,5 @@
 
-## bad_target_family (ALPHA)
+## bad_target_family (PREVIEW)
 
 Error handling for badly formed target address family