From 640f0354ed575c28fef8110f7747efa58403dd2e Mon Sep 17 00:00:00 2001
From: Noureddine <noureddine@saidi.uk>
Date: Wed, 21 Sep 2022 20:42:43 +0100
Subject: [PATCH] Endpoint redirection documentation (#447)

---
 _includes/head-custom.html                    |   2 +-
 bin/gencode                                   |   7 +-
 bin/{check_links => gencode_docs_checklinks}  |   0
 bin/gencode_docs_examples                     | 142 ++++++++++++++++
 .../sequences/endpoint_reconfiguration.md     | 151 ++++++++++++++++++
 docs/specs/sequences/readme.md                |   1 +
 .../endpoint_reconfiguration.json             |  13 ++
 .../config.tests/endpoint_reconfiguration.out |   0
 .../state.tests/endpoint_reconfiguration.json |  23 +++
 .../state.tests/endpoint_reconfiguration.out  |   0
 .../endpoint_reconfiguration_failed.json      |  29 ++++
 .../endpoint_reconfiguration_failed.out       |   0
 12 files changed, 366 insertions(+), 2 deletions(-)
 rename bin/{check_links => gencode_docs_checklinks} (100%)
 create mode 100755 bin/gencode_docs_examples
 create mode 100644 docs/specs/sequences/endpoint_reconfiguration.md
 create mode 100644 tests/config.tests/endpoint_reconfiguration.json
 create mode 100644 tests/config.tests/endpoint_reconfiguration.out
 create mode 100644 tests/state.tests/endpoint_reconfiguration.json
 create mode 100644 tests/state.tests/endpoint_reconfiguration.out
 create mode 100644 tests/state.tests/endpoint_reconfiguration_failed.json
 create mode 100644 tests/state.tests/endpoint_reconfiguration_failed.out

diff --git a/_includes/head-custom.html b/_includes/head-custom.html
index 567b51d031..45eb548a63 100644
--- a/_includes/head-custom.html
+++ b/_includes/head-custom.html
@@ -2,7 +2,7 @@
 <script>
   document.addEventListener("DOMContentLoaded", function() {
       document.querySelectorAll("pre>code.language-mermaid").forEach($element => {
-        $element.parentElement.outerHTML = `<div class="mermaid">${$element.textContent}</div>`
+        $element.parentElement.outerHTML = `<div class="mermaid">${$element.innerHTML}</div>`
       });
      mermaid.initialize();
 });
diff --git a/bin/gencode b/bin/gencode
index dbac00c7d8..5d1f399e03 100755
--- a/bin/gencode
+++ b/bin/gencode
@@ -8,6 +8,11 @@ if [[ $1 == check ]]; then
     shift
 fi
 
+if [[ -n $check ]]; then
+    # This check must be run before gencode_docs is run
+    bin/gencode_docs_examples check
+fi
+
 bin/gencode_java
 bin/gencode_python gencode/python schema/*.json
 bin/gencode_docs
@@ -15,7 +20,7 @@ bin/gencode_seq
 
 if [[ -n $check ]]; then
     echo Checking gencode docs links...
-    bin/gencode_docs check_links
+    bin/gencode_docs_checklinks
 
     echo "Checking gencode hash (dynamic/static)..."
     files=`find gencode/ -type f | sort`
diff --git a/bin/check_links b/bin/gencode_docs_checklinks
similarity index 100%
rename from bin/check_links
rename to bin/gencode_docs_checklinks
diff --git a/bin/gencode_docs_examples b/bin/gencode_docs_examples
new file mode 100755
index 0000000000..6dd4589867
--- /dev/null
+++ b/bin/gencode_docs_examples
@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+""" Inline update documents with JSON payloads from tests directory"""
+
+import glob
+import re
+import sys
+import argparse
+
+from datetime import datetime
+
+
+REGEX_NORMALIZE = r'(?s)(<!--example:\w+\/\w+.json-->\n)(```json.*?```)'
+REGEX_INCLUDE = r'<!--example:(\w+)\/(\w+).json-->\n'
+
+
+def parse_command_line_args():
+  parser = argparse.ArgumentParser()
+
+  parser.add_argument('check', nargs='?', default=False,
+            help='Check in-line examples match examples in tests directory')
+
+  return parser.parse_args()
+
+
+def validate_code_blocks(file_contents):
+  """ Run some very primitive checks to validate templated placeholder code
+  blocks within file to prevent unwanted deletion of parts of documents. Works
+  by checking the code blocks for lines which start unlike a JSON line (caused
+  by mismatched ``` which result in the regex match extending into the of the
+  document) or the letter F(ile not found)
+
+  Arguments
+    file_contents contents of file
+  Returns
+    true/false if file is valid
+  """
+  matches = re.findall(REGEX_NORMALIZE, file_contents)
+  for match in matches:
+    code_block_removed = re.sub(r'(?s)```json(.*)```', r'\g<1>', match[1])
+    if re.search(r'(?m)^[^{}"\sF\/]', code_block_removed):
+      return False
+  return True
+
+
+def read_example_file(match):
+  """ Used as re.sub callback to read example from a file
+  Arguments
+    match   match object
+  Returns
+    json from example file appended to original expression
+  """
+  expression = match.group(0)
+  schema = match.group(1)
+  file = match.group(2)
+
+  include_path = f'tests/{schema}.tests/{file}.json'
+
+  try:
+    with open(include_path, 'r', encoding='utf-8') as f:
+      return f'{expression}```json\n{f.read().rstrip()}\n```'
+  except FileNotFoundError:
+    # Append time for not found errors so there's always a diff
+    return f'{expression}```json\nFILE NOT FOUND ({datetime.now()})\n```'
+
+
+def include_examples(file_contents):
+  """ Replaces examples within a string
+  Arguments
+    file_contents   string to replace in (contents of documentation file)
+  Returns
+    string contents of file
+  """
+  normalized_file = re.sub(REGEX_NORMALIZE, r'\g<1>', file_contents)
+  return re.sub(REGEX_INCLUDE, read_example_file, normalized_file)
+
+
+def diff_examples(original, updated):
+  """ Compares documentation before and after external sources are updated
+  and returns a list of JSON code blocks which are different between the two
+  Arguments:
+    original  original documentation file contents
+    updated   updated (in memory) documentation file contents
+  Returns:
+    list  list of expressions (<!--example:metadata/tutorial_hvac_min.json-->)
+          which differ
+  """
+  diffs = []
+
+  matches_original = re.findall(REGEX_NORMALIZE, original)
+  matches_updated = re.findall(REGEX_NORMALIZE, updated)
+
+  for match_original, match_updated in zip(matches_original, matches_updated):
+    if match_original[1] != match_updated[1]:
+      diffs.append(match_original[0].rstrip())
+
+  return diffs
+
+
+def main():
+  file_paths = glob.glob('docs/**/*.md', recursive=True)
+
+  args = parse_command_line_args()
+  example_diffs = {}
+  error = False
+
+  for file_path in file_paths:
+
+    with open(file_path, 'r', encoding='utf-8') as f:
+      file_contents = f.read()
+
+    if not validate_code_blocks(file_contents):
+      error = True
+      print(f'Error processing {file_path}:')
+      print('\tMismatched code block termination (```) ',
+            'or invalid JSON in codeblock')
+      continue
+
+    updated_file_contents = include_examples(file_contents)
+
+    if updated_file_contents != file_contents:
+      if args.check:
+        example_diffs[file_path] = diff_examples(file_contents,
+            updated_file_contents)
+      else:
+        with open(file_path, 'w', encoding='utf-8') as f:
+          f.write(updated_file_contents)
+  if args.check:
+    if example_diffs:
+      error = True
+      print('Examples do not match source in following files:')
+      for file, diffs in example_diffs.items():
+        print(f'{file}')
+        for diff in diffs:
+          print(f'** {diff}')
+    else:
+      print('Documentation in line examples match source examples')
+  if error:
+    sys.exit(1)
+
+
+if __name__ == '__main__':
+  main()
diff --git a/docs/specs/sequences/endpoint_reconfiguration.md b/docs/specs/sequences/endpoint_reconfiguration.md
new file mode 100644
index 0000000000..117e32dd83
--- /dev/null
+++ b/docs/specs/sequences/endpoint_reconfiguration.md
@@ -0,0 +1,151 @@
+[**UDMI**](../../../) / [**Docs**](../../) / [**Specs**](../) / [**Sequences**](./) / [Endpoint Reconfiguration](#)
+
+# Endpoint Reconfiguration
+
+Endpoint reconfiguration is the reconfiguration of the UDMI endpoint through the
+UDMI blob delivery mechanisms via UDMI config messages.
+
+The [endpoint configuration blob](https://github.com/faucetsdn/udmi/blob/master/tests/configuration_endpoint.tests/simple.json) is a JSON object defined by
+[configuration_endpoint.json](https://faucetsdn.github.io/udmi/gencode/docs/configuration_endpoint.html), which is base64 encoded in the config message.
+
+
+## Tests
+
+### Valid Endpoint (Successful Reconfiguration)
+
+**Notes**
+- `<NEW_ENDPOINT>` is a **base64** encoded endpoint object
+- `blobset.blobs._iot_endpoint_config` is present in a device's state message if, and only if,   the last received config message has a `blobset.blobs._iot_endpoint_config` block.
+
+```mermaid
+%%{wrap}%%
+sequenceDiagram
+    autonumber
+    participant D as Device
+    participant E as Original Endpoint
+    participant E' as New Endpoint
+    E->>D:CONFIG MESSAGE<br/>blobset.blobs._iot_endpoint_config.base64 = <NEW_ENDPOINT><br/>blobset.blobs._iot_endpoint_config.phase = "final"
+    D->>E:STATE MESSAGE<br/>blobset.blobs._iot_endpoint_config.phase = "apply"
+    loop Total duration < 30 seconds
+    D-->>E':CONNECTION ATTEMPT
+    end
+    E'->>D:CONFIG MESSAGE<br/>blobset.blobs._iot_endpoint_config.base64 = <NEW_ENDPOINT><br/>blobset.blobs._iot_endpoint_config.phase = "final"
+    D->>E':STATE MESSAGE<br/>blobset.blobs._iot_endpoint_config.phase = "final"
+    note over E': system.last_update in state matches timestamp of config from new endpoint
+    E'->>D:CONFIG MESSAGE<br/>blobset.blobs._iot_endpoint_config = null
+    D->>E':STATE MESSAGE<br/>blobset.blobs._iot_endpoint_config = null
+```
+
+### Invalid Endpoint (Unsuccessful Reconfiguration)
+
+```mermaid
+%%{wrap}%%
+sequenceDiagram
+    autonumber
+    participant D as Device
+    participant E as Original Endpoint
+    participant E' as New Endpoint
+    E->>D:CONFIG MESSAGE<br/>blobset.blobs._iot_endpoint_config.blob = <NEW_ENDPOINT><br>blobset.blobs._iot_endpoint_config.blob.phase = "final"
+    D->>E:STATE MESSAGE<br/>blobset.blobs._iot_endpoint_config.phase = "apply"
+    loop Total duration < 30 seconds
+    D-->>E':CONNECTION ATTEMPT
+    note over D: Failure, e.g. endpoint cannot be reached, incorrect credentials...
+    end
+    D-->>E:CONNECTION ATTEMPT
+    E->>D:CONFIG MESSAGE
+    D->>E:STATE MESSAGE<br/>blobset.blobs._iot_endpoint_config.phase = "final"<br/>blobset.blobs._iot_endpoint_config.status.level = 500 (ERROR)<br/>blobset.blobs._iot_endpoint_config.status.category = "blobset.blob.apply"
+
+```
+
+## Message Examples
+
+Config message to initiate Reconfiguration (sequence #1 in diagrams above)
+<!--example:config/endpoint_reconfiguration.json-->
+```json
+{
+  "version": 1,
+  "blobset": {
+    "blobs": {
+      "_iot_endpoint_config": {
+        "phase": "final",
+        "content_type": "application/json",
+        "base64": "ewogICJwcm90b2NvbCI6ICJtcXR0IiwKICAiY2xpZW50X2lkIjogInByb2plY3RzL2Jvcy1zbm9yay1kZXYvbG9jYXRpb25zL3VzLWNlbnRyYWwxL3JlZ2lzdHJpZXMvWlotVFJJLUZFQ1RBL2RldmljZXMvQUhVLTEiLAogICJob3N0bmFtZSI6ICJtcXR0Lmdvb2dsZWFwaXMuY29tIgp9"
+      }
+    }
+  },
+  "timestamp": "2022-07-13T12:00:00.000Z"
+}
+```
+
+The base64 encoded value decodes to:
+<!--example:configuration_endpoint/simple.json-->
+```json
+{
+  "protocol": "mqtt",
+  "client_id": "projects/bos-snork-dev/locations/us-central1/registries/ZZ-TRI-FECTA/devices/AHU-1",
+  "hostname": "mqtt.googleapis.com"
+}
+```
+
+Example successful state message sent to the new endpoint from device following
+a successful reconfiguration
+<!--example:state/endpoint_reconfiguration.json-->
+```json
+{
+  "version": 1,
+  "timestamp": "2022-07-13T12:00:10.000Z",
+  "system": {
+    "hardware": {
+      "make": "ACME",
+      "model": "Bird Trap"
+    },
+    "software": {
+      "firmware": "1.2"
+    },
+    "serial_no": "000000",
+    "last_config": "2022-07-13T12:00:00.000Z",
+    "operational": true
+  },
+  "blobset": {
+    "blobs": {
+      "_iot_endpoint_config": {
+        "phase": "final"
+      }
+    }
+  }
+}
+```
+
+This is an example of the state message sent to the original endpoint after a failure
+<!--example:state/endpoint_reconfiguration_failed.json-->
+```json
+{
+  "version": 1,
+  "timestamp": "2022-07-13T12:00:11.000Z",
+  "system": {
+    "hardware": {
+      "make": "ACME",
+      "model": "Bird Trap"
+    },
+    "software": {
+      "firmware": "1.2"
+    },
+    "serial_no": "000000",
+    "last_config": "2022-07-13T12:00:00.000Z",
+    "operational": true
+  },
+  "blobset": {
+    "blobs": {
+      "_iot_endpoint_config": {
+        "phase": "final",
+        "status": {
+          "message": "Could not connect to new endpoint",
+          "category": "blobset.blob.apply",
+          "timestamp": "2022-07-13T12:00:11.000Z",
+          "level": 500
+        }
+      }
+    }
+  }
+}
+```
diff --git a/docs/specs/sequences/readme.md b/docs/specs/sequences/readme.md
index 0449cd8a05..3d23a5feb2 100644
--- a/docs/specs/sequences/readme.md
+++ b/docs/specs/sequences/readme.md
@@ -12,4 +12,5 @@ More explanatory detail for some of the key functional specifications:
 
 - [Config and State](config.md): Basic handling of _config_ and _state_ messages.
 - [Discovery](discovery.md): Flow for on-prem network and point discovery.
+- [Endpoint Reconfiguration](endpoint_reconfiguration.md): Ability to reconfigure endpoint (e.g. MQTT) from the cloud
 - [Writeback](writeback.md): Ability to control on-prem resouces from the cloud.
diff --git a/tests/config.tests/endpoint_reconfiguration.json b/tests/config.tests/endpoint_reconfiguration.json
new file mode 100644
index 0000000000..9cb934fc44
--- /dev/null
+++ b/tests/config.tests/endpoint_reconfiguration.json
@@ -0,0 +1,13 @@
+{
+  "version": 1,
+  "blobset": {
+    "blobs": {
+      "_iot_endpoint_config": {
+        "phase": "final",
+        "content_type": "application/json",
+        "base64": "ewogICJwcm90b2NvbCI6ICJtcXR0IiwKICAiY2xpZW50X2lkIjogInByb2plY3RzL2Jvcy1zbm9yay1kZXYvbG9jYXRpb25zL3VzLWNlbnRyYWwxL3JlZ2lzdHJpZXMvWlotVFJJLUZFQ1RBL2RldmljZXMvQUhVLTEiLAogICJob3N0bmFtZSI6ICJtcXR0Lmdvb2dsZWFwaXMuY29tIgp9"
+      }
+    }
+  },
+  "timestamp": "2022-07-13T12:00:00.000Z"
+}
diff --git a/tests/config.tests/endpoint_reconfiguration.out b/tests/config.tests/endpoint_reconfiguration.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests/state.tests/endpoint_reconfiguration.json b/tests/state.tests/endpoint_reconfiguration.json
new file mode 100644
index 0000000000..263e60d54e
--- /dev/null
+++ b/tests/state.tests/endpoint_reconfiguration.json
@@ -0,0 +1,23 @@
+{
+  "version": 1,
+  "timestamp": "2022-07-13T12:00:10.000Z",
+  "system": {
+    "hardware": {
+      "make": "ACME",
+      "model": "Bird Trap"
+    },
+    "software": {
+      "firmware": "1.2"
+    },
+    "serial_no": "000000",
+    "last_config": "2022-07-13T12:00:00.000Z",
+    "operational": true
+  },
+  "blobset": {
+    "blobs": {
+      "_iot_endpoint_config": {
+        "phase": "final"
+      }
+    }
+  }
+}
diff --git a/tests/state.tests/endpoint_reconfiguration.out b/tests/state.tests/endpoint_reconfiguration.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests/state.tests/endpoint_reconfiguration_failed.json b/tests/state.tests/endpoint_reconfiguration_failed.json
new file mode 100644
index 0000000000..7a7d37e126
--- /dev/null
+++ b/tests/state.tests/endpoint_reconfiguration_failed.json
@@ -0,0 +1,29 @@
+{
+  "version": 1,
+  "timestamp": "2022-07-13T12:00:11.000Z",
+  "system": {
+    "hardware": {
+      "make": "ACME",
+      "model": "Bird Trap"
+    },
+    "software": {
+      "firmware": "1.2"
+    },
+    "serial_no": "000000",
+    "last_config": "2022-07-13T12:00:00.000Z",
+    "operational": true
+  },
+  "blobset": {
+    "blobs": {
+      "_iot_endpoint_config": {
+        "phase": "final",
+        "status": {
+          "message": "Could not connect to new endpoint",
+          "category": "blobset.blob.apply",
+          "timestamp": "2022-07-13T12:00:11.000Z",
+          "level": 500
+        }
+      }
+    }
+  }
+}
diff --git a/tests/state.tests/endpoint_reconfiguration_failed.out b/tests/state.tests/endpoint_reconfiguration_failed.out
new file mode 100644
index 0000000000..e69de29bb2