adds ability to render mermaid in markdown and script to export SVG

GSA · Dec 13, 2023 · 73435d4 · 73435d4 · github-actions · Dec 13, 2023
1 parent 34f2f00
commit 73435d4
Show file tree

Hide file tree

Showing 32 changed files with 1,280 additions and 22 deletions.
diff --git a/.gitignore b/.gitignore
@@ -7,6 +7,8 @@ pytest-coverage.txt
 __pycache__/
 venv/
 
+# node 
+node_modules/
 # any previous versions of schemas
 **/dataset_**.json
 **/catalog_**.json
@@ -15,4 +17,4 @@ venv/
 tmp/
 
 # vscode debugger
-.vscode/
+.vscode/
diff --git a/docs/arcgis.svg b/docs/arcgis.svg
diff --git a/docs/dcat.svg b/docs/dcat.svg
diff --git a/docs/diagrams/mermaid/dest/arcgis-1.svg b/docs/diagrams/mermaid/dest/arcgis-1.svg
diff --git a/docs/diagrams/mermaid/dest/arcgis.md b/docs/diagrams/mermaid/dest/arcgis.md
@@ -0,0 +1 @@
+![diagram](./arcgis-1.svg)
diff --git a/docs/diagrams/mermaid/dest/dcat-1.svg b/docs/diagrams/mermaid/dest/dcat-1.svg
diff --git a/docs/diagrams/mermaid/dest/dcat.md b/docs/diagrams/mermaid/dest/dcat.md
@@ -0,0 +1 @@
+![diagram](./dcat-1.svg)
diff --git a/docs/diagrams/mermaid/dest/new_harvesting-1.svg b/docs/diagrams/mermaid/dest/new_harvesting-1.svg
diff --git a/docs/diagrams/mermaid/dest/new_harvesting.md b/docs/diagrams/mermaid/dest/new_harvesting.md
@@ -0,0 +1 @@
+![diagram](./new_harvesting-1.svg)
diff --git a/docs/diagrams/mermaid/dest/old_harvesting-1.svg b/docs/diagrams/mermaid/dest/old_harvesting-1.svg
diff --git a/docs/diagrams/mermaid/dest/old_harvesting.md b/docs/diagrams/mermaid/dest/old_harvesting.md
@@ -0,0 +1 @@
+![diagram](./old_harvesting-1.svg)
diff --git a/docs/diagrams/mermaid/dest/single_xml-1.svg b/docs/diagrams/mermaid/dest/single_xml-1.svg
diff --git a/docs/diagrams/mermaid/dest/single_xml.md b/docs/diagrams/mermaid/dest/single_xml.md
@@ -0,0 +1 @@
+![diagram](./single_xml-1.svg)
diff --git a/docs/diagrams/mermaid/dest/waf_xml-1.svg b/docs/diagrams/mermaid/dest/waf_xml-1.svg
diff --git a/docs/diagrams/mermaid/dest/waf_xml.md b/docs/diagrams/mermaid/dest/waf_xml.md
@@ -0,0 +1 @@
+![diagram](./waf_xml-1.svg)
diff --git a/docs/diagrams/mermaid/makeDoc.mjs b/docs/diagrams/mermaid/makeDoc.mjs
@@ -0,0 +1,19 @@
+import { run } from "@mermaid-js/mermaid-cli"
+import { readdir } from 'node:fs/promises';
+import { resolve } from 'node:path';
+
+(async (req, res) => {
+    try {
+        let fileSrc = resolve('./docs/diagrams/mermaid/src')
+        let fileDest = resolve('./docs/diagrams/mermaid/dest')
+        const files = await readdir(fileSrc);
+        for (const file of files) {
+            console.log(`Found file: ${file}`);
+            await run(
+            `${fileSrc}/${file}`, `${fileDest}/${file}`, {puppeteerConfig: {"headless": "old"}},
+            )
+        }
+    } catch (err) {
+        console.error(err)
+    }
+})();
diff --git a/docs/arcgis.md → docs/diagrams/mermaid/src/arcgis.md b/docs/arcgis.md → docs/diagrams/mermaid/src/arcgis.md
diff --git a/docs/dcat.mmd → docs/diagrams/mermaid/src/dcat.md b/docs/dcat.mmd → docs/diagrams/mermaid/src/dcat.md
@@ -1,3 +1,4 @@
+```mermaid
 flowchart LR
 
   %% Algorithm
@@ -221,3 +222,4 @@ flowchart LR
   has_title{Does the dataset have a title?}
   does_parent_exist{Does Parent exist?}
   is_title_valid{Is the title valid?}
+```
diff --git a/docs/new_harvesting.mmd → docs/diagrams/mermaid/src/new_harvesting.md b/docs/new_harvesting.mmd → docs/diagrams/mermaid/src/new_harvesting.md
@@ -1,7 +1,8 @@
+```mermaid
 flowchart TD
   %% Operations
   sc([SOURCE CREATION])
-  extract([Extract Catalog Source])
+  extract([Extract Faceted Catalog Source])
   compare([Compare Source Catalog to Data.gov Catalog])
   load([Load into Data.gov Catalog])
   validate([Validate Dataset])
@@ -30,3 +31,4 @@ flowchart TD
   load --> completed
   compare --> nochanges
   nochanges --> completed
+```
diff --git a/docs/old_harvesting.mmd → docs/diagrams/mermaid/src/old_harvesting.md b/docs/old_harvesting.mmd → docs/diagrams/mermaid/src/old_harvesting.md
@@ -1,3 +1,4 @@
+```mermaid
 flowchart LR
   %% Operations
   sc([SOURCE CREATION])
@@ -15,3 +16,4 @@ flowchart LR
   scheduled --> gs
   gs --> fs
   fs --> is
+```
diff --git a/docs/single_xml.mmd → docs/diagrams/mermaid/src/single_xml.md b/docs/single_xml.mmd → docs/diagrams/mermaid/src/single_xml.md
@@ -1,3 +1,4 @@
+```mermaid
 flowchart LR
 
   %% Algorithm
@@ -140,3 +141,4 @@ flowchart LR
   is_source_private{Is the Harvest Source Private?}
   is_source_part_of_topic{Is the Harvest Source part of a Topic?}
   is_status_new{Is the Status new?}
+```
diff --git a/docs/waf_xml.mmd → docs/diagrams/mermaid/src/waf_xml.md b/docs/waf_xml.mmd → docs/diagrams/mermaid/src/waf_xml.md
@@ -1,3 +1,4 @@
+```mermaid
 flowchart LR
 
   %% Algorithm
@@ -192,3 +193,4 @@ flowchart LR
   is_source_private{Is the Harvest Source Private?}
   is_source_part_of_topic{Is the Harvest Source part of a Topic?}
   is_status_new{Is the Status new?}
+```
diff --git a/docs/diagrams/out/erd/harvester2.0 ERD.png → ...diagrams/uml/out/erd/harvester2.0 ERD.png b/docs/diagrams/out/erd/harvester2.0 ERD.png → ...diagrams/uml/out/erd/harvester2.0 ERD.png
diff --git a/docs/diagrams/src/README.md → docs/diagrams/uml/src/README.md b/docs/diagrams/src/README.md → docs/diagrams/uml/src/README.md
diff --git a/docs/diagrams/src/erd.plantuml → docs/diagrams/uml/src/erd.plantuml b/docs/diagrams/src/erd.plantuml → docs/diagrams/uml/src/erd.plantuml
diff --git a/docs/harvesting.md b/docs/harvesting.md
diff --git a/docs/new_harvesting.svg b/docs/new_harvesting.svg
File	Stmts	Miss	Cover	Missing
harvester
__init__.py	3	0	100%
harvester/db/models
__init__.py	5	0	100%
models.py	53	0	100%
harvester/extract
__init__.py	19	2	2	89%
dcatus.py	11	2	2	82%
harvester/utils
__init__.py	0	0	100%
json.py	22	6	6	73%
pg.py	35	4	4	89%
s3.py	24	6	6	75%
harvester/validate
__init__.py	0	0	100%
dcat_us.py	24	0	100%
TOTAL	196	20	90%