diff --git a/.gencode_hash.txt b/.gencode_hash.txt index 279f5c6b00..98bf659276 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -12,6 +12,7 @@ f48026471ae3cd7867bce416dc21c2fb728f48d8476a8d6e95f6acaf1d8b6cf3 gencode/docs/i 43019239edfdad71647c40d254b9b3aec68f3deb0c808ba636530c1cf9985fe0 gencode/docs/state.html d39d7fe37a41c74a40080af7b0a429d201ab1fdff7444428c4b98eb7b38c332b gencode/java/udmi/schema/Asset.java 0825a5cec83003bb0a6488c4ed7010a04ae0d3848ef36fe01bb4e6718ba7b96d gencode/java/udmi/schema/Aux.java +df118b8e5abebac38579027b5a17192163f0cc97e55a8e44b847b1a965263636 gencode/java/udmi/schema/Blob.java d5adb804697243f97cdd8589750401654f3fab075a9aeac4f2851e46695ef11d gencode/java/udmi/schema/BlobBlobsetConfig.java 2c03651cb2ecda072b1418222eebb5560185669f8ffdd03021ad5ad8ff7ba3b0 gencode/java/udmi/schema/BlobBlobsetState.java d2c5b5aae8db27b68104fc83a1f38de0a3f1b5d683f2b13599adf24e96c7d124 gencode/java/udmi/schema/BlobsetConfig.java diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index f0f0add296..813d57a9f2 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -52,27 +52,38 @@ jobs: run: | bin/setup_base bin/clone_model - - name: pubber run + - name: pubber runs env: GCP_TARGET_PROJECT: ${{ secrets.GCP_TARGET_PROJECT }} if: "${{ env.GCP_TARGET_PROJECT != '' }}" run: | bin/test_redirect $GCP_TARGET_PROJECT - - name: pubber checks + - name: pubber cmd checks env: GCP_TARGET_PROJECT: ${{ secrets.GCP_TARGET_PROJECT }} if: "${{ env.GCP_TARGET_PROJECT != '' }}" run: | - # Simple checks that a redirect happened and failed - set +x - fgrep registries/ZZ-TRI-FECTA/devices/GAT-123 pubber.out - fgrep 'system.config.parse success' pubber.out - fgrep registries/missing/devices/GAT-123 pubber.out - fgrep 'Not authorized to connect' pubber.out - - name: pubber logs - if: ${{ always() }} + echo ::::::: pubber.out.1 + cat pubber.out.1 + echo ::::::: + echo Simple checks that a redirect happened and failed + fgrep registries/ZZ-TRI-FECTA/devices pubber.out.1 + fgrep 'system.config.parse success' pubber.out.1 + fgrep registries/missing/devices pubber.out.1 + fgrep 'Not authorized to connect' pubber.out.1 + - name: pubber config checks + env: + GCP_TARGET_PROJECT: ${{ secrets.GCP_TARGET_PROJECT }} + if: "${{ env.GCP_TARGET_PROJECT != '' }}" run: | - cat pubber.out || true + echo ::::::: pubber.out.2 + cat pubber.out.2 + echo ::::::: + echo Check the redirect-by-config setup + fgrep registries/ZZ-TRI-FECTA/devices pubber.out.2 + fgrep 'system.config.parse success' pubber.out.2 + fgrep registries/reconfigure/devices pubber.out.2 + fgrep 'Not authorized to connect' pubber.out.2 udmi: name: Sequence tests diff --git a/bin/gencode_java b/bin/gencode_java index 1205ef6879..a5a8b9b958 100755 --- a/bin/gencode_java +++ b/bin/gencode_java @@ -31,8 +31,8 @@ echo Generating code in $OUTDIR $JARBIN $JOPTS --source . --target $OUTDIR # There is no way to specify enum constants in generated code, so just hack it in manually. -echo Copying Level.java -cp -n $ROOT_DIR/etc/Level.java $OUTDIR/udmi/schema/Level.java +echo Copying shared constants $ROOT_DIR/etc/*.java +cp -n $ROOT_DIR/etc/*.java $OUTDIR/udmi/schema/ echo Cleaning up __ classes... find $OUTDIR -name \*__\*.java | xargs rm diff --git a/bin/pubber b/bin/pubber index 55cc8173ba..35d7b03c82 100755 --- a/bin/pubber +++ b/bin/pubber @@ -19,7 +19,7 @@ if [ ! -f $site_path/cloud_iot_config.json ]; then fi echo Building pubber... -$ROOT_DIR/pubber/bin/build > /dev/null +$ROOT_DIR/pubber/bin/build echo Running tools version `(cd $ROOT_DIR; git describe)` diff --git a/bin/reset_config b/bin/reset_config index 4917dd7508..b084c514f9 100755 --- a/bin/reset_config +++ b/bin/reset_config @@ -2,8 +2,8 @@ ROOT=$(realpath $(dirname $0)/..) -if [[ $# != 3 ]]; then - echo Usage: $0 site_dir project_id device_id +if [[ $# != 3 && $# != 4 ]]; then + echo Usage: $0 site_dir project_id device_id [config_file] false fi @@ -11,16 +11,20 @@ site_dir=$(realpath $1) project_id=$2 device_id=$3 shift 3 -cd ${ROOT} -pwd +config_file=generated_config.json +if [[ -n $1 ]]; then + config_file=$1 + shift +fi + +cd ${ROOT} -device_config=/tmp/${device_id}_config.json -cp ${site_dir}/devices/${device_id}/out/generated_config.json ${device_config} +dst_config=/tmp/${device_id}_config.json +src_config=${site_dir}/devices/${device_id}/out/$config_file now_date=$(python3 -c 'import datetime; print(datetime.datetime.utcnow().isoformat() + "Z")') echo Setting config timestamp ${now_date} -sponge < /dev/null || echo sponge command not found, might need workaround for osx or other -jq .timestamp=\"${now_date}\" < ${device_config} | sponge ${device_config} +jq .timestamp=\"${now_date}\" < ${src_config} > ${dst_config} echo Resetting device ${device_id} config... -validator/bin/reflector ${site_dir} ${project_id} ${device_id} update/config:${device_config} +validator/bin/reflector ${site_dir} ${project_id} ${device_id} update/config:${dst_config} diff --git a/bin/setup_base b/bin/setup_base index ce8da695fe..fa9721123f 100755 --- a/bin/setup_base +++ b/bin/setup_base @@ -1,6 +1,6 @@ #!/bin/bash -sudo apt-get install -y hxtools moreutils +sudo apt-get install -y hxtools moreutils expect python3 -m venv venv diff --git a/bin/test_redirect b/bin/test_redirect index 59b5b079bb..e6e91e50ae 100755 --- a/bin/test_redirect +++ b/bin/test_redirect @@ -19,26 +19,70 @@ else fi site_path=sites/udmi_site_model -device_id=AHU-22 # Static device for testing. Needs to be different than other tests scripts so as not to conflict during CI. +device_id=AHU-1 # Static device for testing. Needs to be different than other tests scripts so as not to conflict during CI. +mkdir -p out serial_no=sequencer-$RANDOM echo Using pubber with serial $serial_no +site_config=$site_path/cloud_iot_config.json +cloud_region=$(jq -r .cloud_region $site_config) +registry_id=$(jq -r .registry_id $site_config) + PUBBER_OUT=pubber.out -# Ideally use lock files, not grep-and-kill... -pids=`ps ax | fgrep pubber | fgrep java | awk '{print $1}'` -if [[ -n $pids ]]; then - echo Killing pubber pids $pids - kill $pids -fi +echo Killing running pubber instances... +ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true bin/reset_config $site_path $project_id $device_id -echo Writing pubber output to $PUBBER_OUT -echo bin/pubber $site_path $project_id $device_id $serial_no +echo Running pubber with redirectRegistry on timeout +timeout 3m unbuffer bin/pubber $site_path $project_id $device_id $serial_no redirectRegistry=missing > $PUBBER_OUT.1 2>&1 || true + +echo Done with timeout, running pubber in background... +bin/pubber $site_path $project_id $device_id $serial_no > $PUBBER_OUT.2 2>&1 & + +# Wait for initial connection, then reset config to redirect +sleep 20 + +cat < out/endpoint.json +{ + "projectId": "$project_id", + "registryId": "reconfigure", + "cloudRegion": "$cloud_region" +} +EOF + +base64=$(base64 -w 0 out/endpoint.json) -result=0 -timeout 5m bin/pubber $site_path $project_id $device_id $serial_no redirectRegistry=missing > $PUBBER_OUT 2>&1 || result=$? +cat < out/blobs.json +{ + "blobset": { + "blobs": { + "_iot_config": { + "phase": "final", + "base64": "$base64" + } + } + } +} +EOF + +out_base=$site_path/devices/$device_id/out +# Merge JSON files together into new redirect config +jq -s '.[0] * .[1]' $out_base/generated_config.json out/blobs.json > $out_base/redirect_config.json + +echo New redirection config: +cat /tmp/${device_id}_config.json + +echo bin/reset_config $site_path $project_id $device_id redirect_config.json +bin/reset_config $site_path $project_id $device_id redirect_config.json + +echo Let pubber react to the new configuration... +sleep 20 + +# Ideally use lock files, not grep-and-kill... +echo Killing running pubber instances... +ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true -echo Pubber exit code $result +echo Done with redirect test. diff --git a/bin/test_sequencer b/bin/test_sequencer index cd6df9eddf..b355765201 100755 --- a/bin/test_sequencer +++ b/bin/test_sequencer @@ -32,6 +32,8 @@ if [[ -n $pids ]]; then kill $pids fi +bin/reset_config $site_path $project_id $device_id + pubber/bin/build echo Writing pubber output to $PUBBER_OUT diff --git a/bin/test_validator b/bin/test_validator index eb5c1715a0..738cce7ae2 100755 --- a/bin/test_validator +++ b/bin/test_validator @@ -15,6 +15,7 @@ shift site_path=sites/udmi_site_model device_id=AHU-1 +mkdir -p out serial_no=validator-$RANDOM echo Using pubber with serial $serial_no @@ -23,12 +24,9 @@ PUBBER_OUT=pubber.out VALIDATOR_OUT=validator.out WAITING=10 -vids=`ps ax | fgrep validator | fgrep java | awk '{print $1}'` -pids=`ps ax | fgrep pubber | fgrep java | awk '{print $1}'` -if [[ -n "$pids$vids" ]]; then - echo Killing pids $pids $vids - kill $pids $vids -fi +echo Killing running pubber/validator instances... +ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true +ps ax | fgrep validator | fgrep java | awk '{print $1}' | xargs kill || true rm -rf $site_path/out/devices @@ -40,6 +38,11 @@ pubber/bin/build bin/reset_config $site_path $project_id $device_id +bin/validator $site_path $project_id & +vpid=$! +sleep 10 +echo Started validator pid $vpid + echo Starting validator, output in $VALIDATOR_OUT bin/validator $site_path $project_id > $VALIDATOR_OUT 2>&1 & for i in `seq 1 $WAITING`; do @@ -79,12 +82,9 @@ fi echo Waiting for system to run for a bit... timeout 60s tail -f $VALIDATOR_OUT || true -vids=`ps ax | fgrep validator | fgrep java | awk '{print $1}'` -pids=`ps ax | fgrep pubber | fgrep java | awk '{print $1}'` -if [[ -n "$pids$vids" ]]; then - echo Killing pids $pids $vids - kill $pids $vids -fi +echo Killing running pubber/validator instances... +ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true +ps ax | fgrep validator | fgrep java | awk '{print $1}' | xargs kill || true outfiles=`find $site_path/out/devices -name \*.out | sort` || true if [[ -z $outfiles ]]; then diff --git a/dashboard/functions/index.js b/dashboard/functions/index.js index 463bed3360..f961000eee 100644 --- a/dashboard/functions/index.js +++ b/dashboard/functions/index.js @@ -327,28 +327,35 @@ function parse_old_config(oldConfig, resetConfig) { async function modify_device_config(registryId, deviceId, subFolder, subContents, startTime) { const [oldConfig, version] = await get_device_config(registryId, deviceId); + var newConfig; - const resetConfig = subFolder == 'system' && subContents && subContents.extra_field == 'reset_config'; - const newConfig = parse_old_config(oldConfig, resetConfig); - if (newConfig === null) { - return; - } - - newConfig.version = UDMI_VERSION; - newConfig.timestamp = currentTimestamp(); - - console.log('Config modify version', subFolder, version, startTime); - if (subContents) { - delete subContents.version; - delete subContents.timestamp; - newConfig[subFolder] = subContents; + if (subFolder == 'update') { + console.log('Config replace version', version, startTime); + newConfig = subContents; } else { - if (!newConfig[subFolder]) { - console.log('Config target already null', subFolder, version, startTime); + const resetConfig = subFolder == 'system' && subContents && subContents.extra_field == 'reset_config'; + newConfig = parse_old_config(oldConfig, resetConfig); + if (newConfig === null) { return; } - delete newConfig[subFolder]; + + newConfig.version = UDMI_VERSION; + newConfig.timestamp = currentTimestamp(); + + console.log('Config modify version', subFolder, version, startTime); + if (subContents) { + delete subContents.version; + delete subContents.timestamp; + newConfig[subFolder] = subContents; + } else { + if (!newConfig[subFolder]) { + console.log('Config target already null', subFolder, version, startTime); + return; + } + delete newConfig[subFolder]; + } } + const attributes = { projectId: PROJECT_ID, cloudRegion: registry_regions[registryId], diff --git a/etc/Blob.java b/etc/Blob.java new file mode 100644 index 0000000000..933264fdac --- /dev/null +++ b/etc/Blob.java @@ -0,0 +1,8 @@ +package udmi.schema; + +// This class is manually curated and then copied into the gencode directory. Look for the +// proper source and don't be fooled! These are general constants for blob management. +public abstract class Blob { + public static final String FINAL_PHASE = "final"; + public static final String IOT_CONFIG_BLOB = "_iot_config"; +} diff --git a/gencode/java/udmi/schema/Blob.java b/gencode/java/udmi/schema/Blob.java new file mode 100644 index 0000000000..933264fdac --- /dev/null +++ b/gencode/java/udmi/schema/Blob.java @@ -0,0 +1,8 @@ +package udmi.schema; + +// This class is manually curated and then copied into the gencode directory. Look for the +// proper source and don't be fooled! These are general constants for blob management. +public abstract class Blob { + public static final String FINAL_PHASE = "final"; + public static final String IOT_CONFIG_BLOB = "_iot_config"; +} diff --git a/pubber/.idea/runConfigurations/Pubber.xml b/pubber/.idea/runConfigurations/Pubber.xml index e60f811adc..71f95c4348 100644 --- a/pubber/.idea/runConfigurations/Pubber.xml +++ b/pubber/.idea/runConfigurations/Pubber.xml @@ -2,7 +2,7 @@