From b9a2b6e7143fe08afae13b9204809f81281d82f7 Mon Sep 17 00:00:00 2001 From: Noureddine Date: Fri, 29 Apr 2022 12:03:19 +0100 Subject: [PATCH 1/4] fix front page links and change check_links --- README.md | 20 ++++++++++---------- bin/check_links | 38 +++++++++++++++++++++++++++++++++++--- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 9b24303ad8..c69aefd3ac 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ management and operation of physical IoT systems. This data is typically exchang with a cloud entity that can maintain a "digital twin" or "shadow device" in the cloud. * [Core UDMI documentation](http://faucetsdn.github.io/udmi/docs/) for tools and specifications -* [Message schema definition](https://github.com/faucetsdn/udmi/tree/master/schema) with ([_🧬Interactive Viewer_](http://faucetsdn.github.io/udmi/gencode/docs/) +* [Message schema definition](https://github.com/faucetsdn/udmi/tree/master/schema) with ([_🧬Interactive Viewer_](http://faucetsdn.github.io/udmi/gencode/docs/)) * [udmi-discuss@googlegroups.com](https://groups.google.com/forum/#!forum/udmi-discuss) email discussion list * Bi-weekly _UDMI Discuss_ video meeting open to all (join the mailing list to get an invite) @@ -19,14 +19,14 @@ By design, this schema is intended to be: * **M**anagement: Focus on device _management_, rather than command & control. * **I**nterface: Define an interface specification, rather than a client-library or RPC mechanism. -See the associated [UDMI Tech Stack](http://faucetsdn.github.io/udmi/docs/specs/tech_stack.md) for details about transport mechanism +See the associated [UDMI Tech Stack](http://faucetsdn.github.io/udmi/docs/specs/tech_stack) for details about transport mechanism outside of the core schema definition. Nominally meant for use with [Google's Cloud IoT Core](https://cloud.google.com/iot/docs/), it can be applied to any set of data or hosting setup. ## Recommended Workflow -The [recommended workflow](http://faucetsdn.github.io/udmi/docs/guides/workflow.md) for UDMI covers using the _registrar_ and +The [recommended workflow](http://faucetsdn.github.io/udmi/docs/guides) for UDMI covers using the _registrar_ and _validator_ tools to configure and test a cloud project. Additionally, the _pubber_ tool is instrumental in setting up and testing the system independent of actual device setup. @@ -40,7 +40,7 @@ manual operation (aren't automated), and increase the security exposure of the s UDMI is intended to support a few primary use-cases: * _Telemetry Ingestion_: Ingest device data points in a standardized format. -* [_Gateway Proxy_](http://faucetsdn.github.io/udmi/docs/specs/gateway.md): Proxy data/connection for non-UDMI devices, +* [_Gateway Proxy_](http://faucetsdn.github.io/udmi/docs/specs/gateway): Proxy data/connection for non-UDMI devices, allowing adaptation to legacy systems. * _On-Prem Actuation_: Ability to effect on-prem device behavior. * _Device Testability_: e.g. Trigger a fake alarm to test reporting mechanisms. @@ -83,10 +83,10 @@ very large structures or high-bandwidth streams. UDMI provides a means to multiplex multiple functional subsystems through the same shared communication channel. There are a number of subsystems that make up the core UDMI spec: -* Core [_system_](http://faucetsdn.github.io/udmi/docs/messages/system.md) messages about the base device itself. -* Device [_pointset_](http://faucetsdn.github.io/udmi/docs/messages/pointset.md) for device telemetry organized by points. -* Optional [_gateway_](http://faucetsdn.github.io/udmi/docs/specs/gateway.md) functionality for proxying device/MQTT connections. -* Local [_discover_](http://faucetsdn.github.io/udmi/docs/specs/discovery.md) for discovering device and network capabilities. +* Core [_system_](http://faucetsdn.github.io/udmi/docs/messages/system) messages about the base device itself. +* Device [_pointset_](http://faucetsdn.github.io/udmi/docs/messages/pointset) for device telemetry organized by points. +* Optional [_gateway_](http://faucetsdn.github.io/udmi/docs/specs/gateway) functionality for proxying device/MQTT connections. +* Local [_discover_](http://faucetsdn.github.io/udmi/docs/specs/discovery) for discovering device and network capabilities. ## Schema Structure @@ -120,6 +120,6 @@ An interactive view of the schema is available on [https://faucetsdn.github.io/u ### Metadata Registration and Validation Using UDMI on a project entails not only the base device and server implementations, but also -properly registering and validating device configuration. The [registrar](https://faucetsdn.github.io/udmi/docs/tools/registrar.md) -tool and [validator](https://faucetsdn.github.io/udmi/docs/tools/validator.md) tool provide a means to configure and check site +properly registering and validating device configuration. The [registrar](https://faucetsdn.github.io/udmi/docs/tools/registrar) +tool and [validator](https://faucetsdn.github.io/udmi/docs/tools/validator) tool provide a means to configure and check site installations, respectively. diff --git a/bin/check_links b/bin/check_links index c01049ba36..835fe62a4d 100755 --- a/bin/check_links +++ b/bin/check_links @@ -8,6 +8,8 @@ import glob import re import os +HOSTED_GITHUB_PAGES = r"https?:\/\/faucetsdn.github.io\/udmi\/" +HOSTED_GITHUB = r'https?:\/\/github.com\/faucetsdn\/udmi\/(?:blob|tree)\/master\/' def split_link_fragment(link): """ Splits a link into the path and the fragment. @@ -26,6 +28,23 @@ def split_link_fragment(link): return matches.groups() + +def blank_regex_substitutions(string, *args): + """ + Applies multiple blank regex substitutions on a string and returns the result + + Arguments + string to substitute in + *args regex for substitution + + Returns + string with all regex substitutions applied + """ + for regex in args: + string = re.sub(regex, '', string) + return string + + def check_links(file_path): """ Checks if inline markdown links within the given file_path @@ -49,8 +68,9 @@ def check_links(file_path): failing_links.append((file_path, 'header', file_lines[0])) if file_lines[1]: failing_links.append((file_path, 'missing blank', '')) - links = re.findall(r'\[.*\]\(([^\)]*)\)', file_data) + links = re.findall(r'\[(?:[^\]]*)\]\(([^\)]*)\)', file_data) for link in links: + if not re.match('https?://', link): link_path, link_anchor = split_link_fragment(link) dir_name = os.path.dirname(os.path.realpath(file_path)) @@ -60,12 +80,24 @@ def check_links(file_path): if not os.path.exists(resolved_path): failing_links.append((file_path, 'link', link)) - + else: + # Rewrite hosted links (e.g. github pages) to local + rewritten_link = blank_regex_substitutions(link, HOSTED_GITHUB, HOSTED_GITHUB_PAGES) + + if not re.match('https?://', rewritten_link): + resolved_path = os.path.realpath(rewritten_link) + # Append .md to any files without extensions (assumed from github pages links) + root, ext = os.path.splitext(resolved_path) + if not ext and not os.path.isdir(root): + resolved_path = f'{resolved_path}.md' + + if not os.path.exists(resolved_path): + failing_links.append((file_path, 'link', link)) + return failing_links if __name__ == "__main__": - check_paths = ['*.md', 'docs/**/*.md', 'schema/*.json'] for check_path in check_paths: file_paths = glob.glob(check_path, recursive=True) From fa4089aa3b59485df447a9bac2ba0c67d450528f Mon Sep 17 00:00:00 2001 From: Noureddine Date: Fri, 29 Apr 2022 12:06:48 +0100 Subject: [PATCH 2/4] fix broken link --- docs/specs/sequences/writeback.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/specs/sequences/writeback.md b/docs/specs/sequences/writeback.md index e496dad87c..91c814079b 100644 --- a/docs/specs/sequences/writeback.md +++ b/docs/specs/sequences/writeback.md @@ -2,7 +2,7 @@ # Writeback -This file documents UDMI's specification for cloud to device control i.e. writeback. At a high level, the cloud controls a device by writing to the [pointset block](../messages/pAointfdsSset.md) in the device [config](../../messages/config.md). After receiving the config, the device updates its state to reflect the status of the writeback attempt e.g. success, failure, etc. +This file documents UDMI's specification for cloud to device control i.e. writeback. At a high level, the cloud controls a device by writing to the [pointset block](../../messages/pointset.md) in the device [config](../../messages/config.md). After receiving the config, the device updates its state to reflect the status of the writeback attempt e.g. success, failure, etc. ![Basic writeback sequence diagram](images/writeback-basic.png) From e317fb3a563b28e7144bf36b1380097ae54c8ad7 Mon Sep 17 00:00:00 2001 From: Noureddine Date: Tue, 3 May 2022 09:40:14 +0100 Subject: [PATCH 3/4] incorporate comments --- bin/check_links | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/bin/check_links b/bin/check_links index 835fe62a4d..ce51fc8a7f 100755 --- a/bin/check_links +++ b/bin/check_links @@ -8,8 +8,8 @@ import glob import re import os -HOSTED_GITHUB_PAGES = r"https?:\/\/faucetsdn.github.io\/udmi\/" -HOSTED_GITHUB = r'https?:\/\/github.com\/faucetsdn\/udmi\/(?:blob|tree)\/master\/' +HOSTED_GITHUB_PAGES = r'^https?:\/\/faucetsdn\.github\.io\/udmi\/' +HOSTED_GITHUB = r'^https?:\/\/(?:www\.)?github\.com\/faucetsdn\/udmi\/(?:blob|tree)\/master\/' def split_link_fragment(link): """ Splits a link into the path and the fragment. @@ -47,8 +47,9 @@ def blank_regex_substitutions(string, *args): def check_links(file_path): """ - Checks if inline markdown links within the given file_path - resolve + Checks if inline markdown links within the given file_path resolve to a + valid file or directory, or to a hosted (Github or Github Pages) copy of + the repository Arguments: file_path: file to check links within @@ -64,14 +65,14 @@ def check_links(file_path): if file_path.endswith('.md'): file_lines = file_data.split('\n', 2) # TODO: Make this more comprehensive and check actual path. - if not re.match('^\[\*\*UDMI.*\]\(\#\)$', file_lines[0]): + if not re.search('^\[\*\*UDMI.*\]\(\#\)$', file_lines[0]): failing_links.append((file_path, 'header', file_lines[0])) if file_lines[1]: failing_links.append((file_path, 'missing blank', '')) links = re.findall(r'\[(?:[^\]]*)\]\(([^\)]*)\)', file_data) for link in links: - if not re.match('https?://', link): + if not re.search('^https?://', link): link_path, link_anchor = split_link_fragment(link) dir_name = os.path.dirname(os.path.realpath(file_path)) @@ -81,10 +82,11 @@ def check_links(file_path): if not os.path.exists(resolved_path): failing_links.append((file_path, 'link', link)) else: - # Rewrite hosted links (e.g. github pages) to local + # Rewrite hosted links (e.g. github pages) to local by subtracting the web host rewritten_link = blank_regex_substitutions(link, HOSTED_GITHUB, HOSTED_GITHUB_PAGES) - if not re.match('https?://', rewritten_link): + if not re.search('^https?://', rewritten_link): + # The modified link now directs to a local path resolved_path = os.path.realpath(rewritten_link) # Append .md to any files without extensions (assumed from github pages links) root, ext = os.path.splitext(resolved_path) From ae9e80c5500c289e8ef4cf232c3773de4d858252 Mon Sep 17 00:00:00 2001 From: Noureddine Date: Tue, 3 May 2022 09:40:42 +0100 Subject: [PATCH 4/4] trailing space --- bin/check_links | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/check_links b/bin/check_links index ce51fc8a7f..3709205ef0 100755 --- a/bin/check_links +++ b/bin/check_links @@ -76,7 +76,7 @@ def check_links(file_path): link_path, link_anchor = split_link_fragment(link) dir_name = os.path.dirname(os.path.realpath(file_path)) - # Links are relative to the file they were found in + # Links are relative to the file they were found in resolved_path = os.path.realpath(os.path.join(dir_name, link_path)) if not os.path.exists(resolved_path):