From 6752cfe7fae6ee98edd48f58e5250c1e1e526625 Mon Sep 17 00:00:00 2001
From: Will Bamberg <wbamberg@mozilla.com>
Date: Mon, 6 May 2019 15:13:04 -0700
Subject: [PATCH 1/7] Add jsdom and BCD dependencies, upgrade npm

---
 package.json | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/package.json b/package.json
index 9d16ba06..4a647af1 100644
--- a/package.json
+++ b/package.json
@@ -14,18 +14,24 @@
     },
     "scripts": {
         "lint-md": "remark content",
+        "package": "node scripts/package/package.js",
         "spell-md": "mdspell -a -n --en-us 'content/**/*.md'"
     },
     "dependencies": {
+        "jsdom": "^12.2.0",
         "markdown-spellcheck": "1.3.1",
-        "npm": "^6.4.1",
+        "mdn-browser-compat-data": "0.x",
+        "npm": "^6.9.0",
         "remark-cli": "6.0.1",
         "remark-preset-lint-recommended": "3.0.2"
     },
     "remarkConfig": {
         "plugins": [
             "remark-preset-lint-recommended",
-            ["remark-lint-list-item-indent", "space"]
+            [
+                "remark-lint-list-item-indent",
+                "space"
+            ]
         ]
     }
 }

From a1710dd7de0a0d19f12208a6a1072765ebbdf682 Mon Sep 17 00:00:00 2001
From: Will Bamberg <wbamberg@mozilla.com>
Date: Tue, 7 May 2019 20:34:54 -0700
Subject: [PATCH 2/7] Add simple packager

---
 content/html/elements/video/prose.md    | 18 ++----
 package.json                            |  3 +
 scripts/package/package-attributes.js   | 75 ++++++++++++++++++++++
 scripts/package/package-bcd.js          | 12 ++++
 scripts/package/package-contributors.js | 15 +++++
 scripts/package/package-examples.js     | 48 ++++++++++++++
 scripts/package/package-prose.js        | 84 +++++++++++++++++++++++++
 scripts/package/package.js              | 55 ++++++++++++++++
 8 files changed, 297 insertions(+), 13 deletions(-)
 create mode 100644 scripts/package/package-attributes.js
 create mode 100644 scripts/package/package-bcd.js
 create mode 100644 scripts/package/package-contributors.js
 create mode 100644 scripts/package/package-examples.js
 create mode 100644 scripts/package/package-prose.js
 create mode 100644 scripts/package/package.js

diff --git a/content/html/elements/video/prose.md b/content/html/elements/video/prose.md
index 4aa89d8b..7159c644 100644
--- a/content/html/elements/video/prose.md
+++ b/content/html/elements/video/prose.md
@@ -1,9 +1,7 @@
-<!-- <short-description> -->
+<!-- short-description -->
 The **HTML Video element** (**`<video>`**) embeds a media player which
 supports video playback into the document.
-<!-- </short-description> -->
-
-<!-- <overview> -->
+<!-- overview -->
 You can use `<video>` for audio content as well, but the [`<audio>`](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/audio)
 element may provide a more appropriate user experience.
 
@@ -53,9 +51,7 @@ A good general source of information on using HTML `<video>` is the
 [Video and audio
 content](https://developer.mozilla.org/en-US/docs/Learn/HTML/Multimedia_and_embedding/Video_and_audio_content)
 beginner's tutorial.
-<!-- </overview> -->
-
-<!-- <usage-notes> -->
+<!-- usage-notes -->
 Usage notes
 -----------
 
@@ -103,9 +99,7 @@ AddType video/webm .webm
 
 Your web host may provide an easy interface to MIME type configuration
 changes for new technologies until a global update naturally occurs.
-<!-- </usage-notes> -->
-
-<!-- <accessibility-concerns> -->
+<!-- accessibility-concerns -->
 Videos should provide both captions and transcripts that accurately
 describe its content (see [Adding captions and subtitles to HTML5
 video](https://developer.mozilla.org/en-US/docs/Web/Apps/Fundamentals/Audio_and_video_delivery/Adding_captions_and_subtitles_to_HTML5_video)
@@ -156,9 +150,7 @@ setting](https://developer.mozilla.org/en-US/docs/Web/API/WebVTT_API#Cue_setting
   2.0](https://www.w3.org/TR/UNDERSTANDING-WCAG20/media-equiv-av-only-alt.html)
 - [Understanding Success Criterion 1.2.2 | W3C Understanding WCAG
   2.0](https://www.w3.org/TR/UNDERSTANDING-WCAG20/media-equiv-captions.html)
-<!-- </accessibility-concerns> -->
-
-<!-- <see-also> -->
+<!-- see-also -->
 See also
 --------
 
diff --git a/package.json b/package.json
index 4a647af1..823b717d 100644
--- a/package.json
+++ b/package.json
@@ -18,7 +18,10 @@
         "spell-md": "mdspell -a -n --en-us 'content/**/*.md'"
     },
     "dependencies": {
+        "gray-matter": "4.0.2",
         "jsdom": "^12.2.0",
+        "js-yaml": "3.13.1",
+        "marked": "0.6.2",
         "markdown-spellcheck": "1.3.1",
         "mdn-browser-compat-data": "0.x",
         "npm": "^6.9.0",
diff --git a/scripts/package/package-attributes.js b/scripts/package/package-attributes.js
new file mode 100644
index 00000000..a5866827
--- /dev/null
+++ b/scripts/package/package-attributes.js
@@ -0,0 +1,75 @@
+const fs = require('fs');
+const path = require('path');
+
+const yaml = require('js-yaml');
+const matter = require('gray-matter');
+const marked = require('marked');
+const jsdom = require('jsdom');
+const bcd = require('mdn-browser-compat-data');
+
+const { JSDOM } = jsdom;
+
+function extractFromSiblings(node, terminatorTag, contentType) {
+    let content = '';
+    let sib = node.nextSibling;
+    while (sib && sib.nodeName != terminatorTag) {
+        if (sib.outerHTML) {
+            if (contentType === 'html') {
+                content += sib.outerHTML;
+            } else if (contentType === 'text') {
+                content += sib.textContent;
+            }
+        }
+        sib = sib.nextSibling;
+    }
+    return content;
+}
+
+function packageValues(heading, dom) {
+    const values = [];
+    const valueHeadings = dom.querySelectorAll('h3');
+    for (let valueHeading of valueHeadings) {
+        let value = {
+            value: valueHeading.textContent,
+            description: extractFromSiblings(valueHeading, 'H3', 'html')
+        }
+        values.push(value);
+    }
+    return values;
+}
+
+function packageAttribute(attributePath) {
+    const attributeMD = fs.readFileSync(attributePath, 'utf8');
+    const {data, content} = matter(attributeMD);
+    const dom = JSDOM.fragment(marked(content));
+    const attribute = {};
+
+    // extract the name property
+    const name = dom.querySelector('h1');
+    attribute.name = name.textContent;
+
+    // extract the description property
+    attribute.description = extractFromSiblings(name, 'H2', 'html');
+
+    // extract the type property
+    const h2Headings = dom.querySelectorAll('h2');
+    const typeHeading = (h2Headings.length === 2)? h2Headings[1]: h2Headings[0];
+    attribute.type = extractFromSiblings(typeHeading, 'H2', 'text');
+
+    // extract the values property
+    if (h2Headings.length === 2) {
+        valuesHeading = h2Headings[0];
+        attribute.values = packageValues(valuesHeading, dom);
+    }
+
+    return attribute;
+}
+
+function package(root) {
+    const attributePaths = fs.readdirSync(root).map(relative => path.join(root, relative));
+    return attributePaths.map(packageAttribute);
+}
+
+module.exports = {
+    package
+}
diff --git a/scripts/package/package-bcd.js b/scripts/package/package-bcd.js
new file mode 100644
index 00000000..54e10bdd
--- /dev/null
+++ b/scripts/package/package-bcd.js
@@ -0,0 +1,12 @@
+const bcd = require('mdn-browser-compat-data');
+
+function package(query) {
+    let data = query.split('.').reduce(function(prev, curr) {
+        return prev ? prev[curr] : undefined
+    }, bcd);
+    return data;
+}
+
+module.exports = {
+    package
+}
diff --git a/scripts/package/package-contributors.js b/scripts/package/package-contributors.js
new file mode 100644
index 00000000..0bfe1d73
--- /dev/null
+++ b/scripts/package/package-contributors.js
@@ -0,0 +1,15 @@
+const fs = require('fs');
+const path = require('path');
+
+const marked = require('marked');
+
+function package(contributorsPath) {
+    const contributorsMD = fs.readFileSync(contributorsPath, 'utf8');
+    console.log(contributorsMD)
+        console.log(marked(contributorsMD))
+    return marked(contributorsMD);
+}
+
+module.exports = {
+  package
+}
diff --git a/scripts/package/package-examples.js b/scripts/package/package-examples.js
new file mode 100644
index 00000000..c92b356c
--- /dev/null
+++ b/scripts/package/package-examples.js
@@ -0,0 +1,48 @@
+const fs = require('fs');
+const path = require('path');
+
+const matter = require('gray-matter');
+const marked = require('marked');
+
+function packageDescription(examplePath) {
+    const descriptionMD = fs.readFileSync(path.join(examplePath, 'description.md'), 'utf8');
+    const {data, content} = matter(descriptionMD);
+    data.content = marked(content);
+    return data;
+}
+
+function packageSources(examplePath) {
+    const exampleSource = {};
+
+    const jsPath = path.join(examplePath, 'example.js');
+    if (fs.existsSync(jsPath)) {
+        exampleSource.js = fs.readFileSync(jsPath, 'utf8');
+    }
+
+    const cssPath = path.join(examplePath, 'example.css');
+    if (fs.existsSync(cssPath)) {
+        exampleSource.css = fs.readFileSync(cssPath, 'utf8');
+    }
+
+    const htmlPath = path.join(examplePath, 'example.html');
+    if (fs.existsSync(htmlPath)) {
+        exampleSource.html = fs.readFileSync(htmlPath, 'utf8');
+    }
+
+    return exampleSource;
+}
+
+function packageExample(examplePath) {
+    return {
+        description: packageDescription(examplePath),
+        sources: packageSources(examplePath)
+    }
+}
+
+function package(paths) {
+    return paths.map(packageExample);
+}
+
+module.exports = {
+    package
+}
diff --git a/scripts/package/package-prose.js b/scripts/package/package-prose.js
new file mode 100644
index 00000000..9d71a40c
--- /dev/null
+++ b/scripts/package/package-prose.js
@@ -0,0 +1,84 @@
+const fs = require('fs');
+const path = require('path');
+
+const yaml = require('js-yaml');
+const matter = require('gray-matter');
+const marked = require('marked');
+const jsdom = require('jsdom');
+const bcd = require('mdn-browser-compat-data');
+
+const { JSDOM } = jsdom;
+const commentNode = 8;
+const namedSections = [
+  'short-description',
+  'overview',
+  'attributes-text',
+  'usage-notes',
+  'accessibility-concerns',
+  'see-also'
+];
+
+function extractFromSiblings(node, terminatorTag, contentType) {
+    let content = '';
+    let sib = node.nextSibling;
+    while (sib && sib.nodeName != terminatorTag) {
+        if (sib.outerHTML) {
+            if (contentType === 'html') {
+                content += sib.outerHTML;
+            } else if (contentType === 'text') {
+                content += sib.textContent;
+            }
+        }
+        sib = sib.nextSibling;
+    }
+    return content;
+}
+
+function packageValues(heading, dom) {
+    const values = [];
+    const valueHeadings = dom.querySelectorAll('h3');
+    for (let valueHeading of valueHeadings) {
+        let value = {
+            value: valueHeading.textContent,
+            description: extractFromSiblings(valueHeading, 'H3', 'html')
+        }
+        values.push(value);
+    }
+    return values;
+}
+
+function getSection(node, sections) {
+    const sectionName = node.textContent.trim();
+    const sectionContent = extractFromSiblings(node, '#comment', 'html');
+    const extraSections = [];
+
+    if (namedSections.includes(sectionName)) {
+        sections[sectionName] = sectionContent;
+    } else {
+        const additionalSection = {
+            name: sectionName,
+            content: sectionContent
+        };
+      sections['additional-sections'].push(additionalSection);
+    }
+}
+
+function package(prosePath) {
+    const proseMD = fs.readFileSync(prosePath, 'utf8');
+    const dom = JSDOM.fragment(marked(proseMD));
+    const sections = {
+        'additional-sections': []
+    };
+    let node = dom.firstChild;
+    while (node) {
+        if (node.nodeType === commentNode) {
+            getSection(node, sections);
+        }
+        node = node.nextSibling;
+    }
+    return sections;
+}
+
+module.exports = {
+    package
+}
diff --git a/scripts/package/package.js b/scripts/package/package.js
new file mode 100644
index 00000000..10a21547
--- /dev/null
+++ b/scripts/package/package.js
@@ -0,0 +1,55 @@
+const fs = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+
+const bcd = require('./package-bcd');
+const examples = require('./package-examples');
+const attributes = require('./package-attributes');
+const prose = require('./package-prose');
+const contributors = require('./package-contributors');
+
+const htmlElements = '/content/html/elements';
+
+const writeToFile = (propertyName, json) => {
+  const data = {
+    html: {
+      elements: {
+        [propertyName]: json,
+      }
+    }
+  };
+
+  const dest = path.join(process.cwd(),'packaged', `${propertyName}.json`);
+  const destDir = path.dirname(dest);
+  if (!fs.existsSync(destDir)) {
+      fs.mkdirSync(destDir, { recursive: true });
+  }
+  fs.writeFileSync(dest, `${JSON.stringify(data, null, 2)}`);
+};
+
+function package(elementName) {
+    const elementPath = path.join(process.cwd(), htmlElements, elementName);
+
+    // open meta.yaml
+    const meta = yaml.safeLoad(fs.readFileSync(path.join(elementPath, 'meta.yaml'), 'utf8'));
+
+    // initialise some paths for more resources
+    const examplesPaths = meta.examples.map(relativePath => path.join(elementPath, relativePath));
+    const attributesPath = path.join(elementPath, meta.attributes['element-specific']);
+    const prosePath = path.join(elementPath, 'prose.md');
+    const contributorsPath = path.join(elementPath, 'contributors.md');
+  
+    // make the package
+    const element = {};
+    element.title = meta.title;
+    element.interactive_example_url = meta['interactive-example'];
+    element.browser_compatibility = bcd.package(meta['browser-compatibility']);
+    element.examples = examples.package(examplesPaths);
+    element.attributes = attributes.package(attributesPath);
+    element.prose = prose.package(prosePath);
+    element.contributors = contributors.package(contributorsPath);
+
+    writeToFile(elementName, element);
+}
+
+package(process.argv[2]);

From 8f8622803f52654933c32b422c5ff10b4dc93291 Mon Sep 17 00:00:00 2001
From: Will Bamberg <wbamberg@mozilla.com>
Date: Wed, 8 May 2019 15:42:02 -0700
Subject: [PATCH 3/7] Remove logging

---
 scripts/package/package-contributors.js | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scripts/package/package-contributors.js b/scripts/package/package-contributors.js
index 0bfe1d73..c8bafd39 100644
--- a/scripts/package/package-contributors.js
+++ b/scripts/package/package-contributors.js
@@ -5,8 +5,6 @@ const marked = require('marked');
 
 function package(contributorsPath) {
     const contributorsMD = fs.readFileSync(contributorsPath, 'utf8');
-    console.log(contributorsMD)
-        console.log(marked(contributorsMD))
     return marked(contributorsMD);
 }
 

From 8e7bc0bb6486d4e3939234672767fb8c210d9453 Mon Sep 17 00:00:00 2001
From: Will Bamberg <wbamberg@mozilla.com>
Date: Wed, 8 May 2019 15:47:00 -0700
Subject: [PATCH 4/7] Add a check that the given element exists

---
 scripts/package/package.js | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scripts/package/package.js b/scripts/package/package.js
index 10a21547..2a101b8d 100644
--- a/scripts/package/package.js
+++ b/scripts/package/package.js
@@ -30,6 +30,11 @@ const writeToFile = (propertyName, json) => {
 function package(elementName) {
     const elementPath = path.join(process.cwd(), htmlElements, elementName);
 
+    if (!fs.exists(elementPath)) {
+        console.error(`Could not find an element called "${elementName}"`);
+        return;
+    }
+
     // open meta.yaml
     const meta = yaml.safeLoad(fs.readFileSync(path.join(elementPath, 'meta.yaml'), 'utf8'));
 

From a4802d6733e6953b2f401f3471614c4f0fc3dc4d Mon Sep 17 00:00:00 2001
From: Will Bamberg <wbamberg@mozilla.com>
Date: Fri, 10 May 2019 14:23:24 -0700
Subject: [PATCH 5/7] Use existsSync rather than exists

---
 scripts/package/package.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/package/package.js b/scripts/package/package.js
index 2a101b8d..8ea62c88 100644
--- a/scripts/package/package.js
+++ b/scripts/package/package.js
@@ -30,7 +30,7 @@ const writeToFile = (propertyName, json) => {
 function package(elementName) {
     const elementPath = path.join(process.cwd(), htmlElements, elementName);
 
-    if (!fs.exists(elementPath)) {
+    if (!fs.existsSync(elementPath)) {
         console.error(`Could not find an element called "${elementName}"`);
         return;
     }

From 418f3ef440df579fd83871cdac53e237bae75fcd Mon Sep 17 00:00:00 2001
From: Will Bamberg <wbamberg@mozilla.com>
Date: Fri, 10 May 2019 14:34:18 -0700
Subject: [PATCH 6/7] Rename scripts

---
 package.json                                           |  2 +-
 .../{package/package.js => build-json/build-json.js}   | 10 +++++-----
 .../compose-attributes.js}                             |  0
 .../compose-examples.js}                               |  0
 .../package-bcd.js => build-json/resolve-bcd.js}       |  0
 .../resolve-contributors.js}                           |  0
 .../package-prose.js => build-json/slice-prose.js}     |  0
 7 files changed, 6 insertions(+), 6 deletions(-)
 rename scripts/{package/package.js => build-json/build-json.js} (88%)
 rename scripts/{package/package-attributes.js => build-json/compose-attributes.js} (100%)
 rename scripts/{package/package-examples.js => build-json/compose-examples.js} (100%)
 rename scripts/{package/package-bcd.js => build-json/resolve-bcd.js} (100%)
 rename scripts/{package/package-contributors.js => build-json/resolve-contributors.js} (100%)
 rename scripts/{package/package-prose.js => build-json/slice-prose.js} (100%)

diff --git a/package.json b/package.json
index 823b717d..a975ddb3 100644
--- a/package.json
+++ b/package.json
@@ -14,7 +14,7 @@
     },
     "scripts": {
         "lint-md": "remark content",
-        "package": "node scripts/package/package.js",
+        "build-json": "node scripts/build-json/build-json.js",
         "spell-md": "mdspell -a -n --en-us 'content/**/*.md'"
     },
     "dependencies": {
diff --git a/scripts/package/package.js b/scripts/build-json/build-json.js
similarity index 88%
rename from scripts/package/package.js
rename to scripts/build-json/build-json.js
index 8ea62c88..4a1dd3fc 100644
--- a/scripts/package/package.js
+++ b/scripts/build-json/build-json.js
@@ -2,11 +2,11 @@ const fs = require('fs');
 const path = require('path');
 const yaml = require('js-yaml');
 
-const bcd = require('./package-bcd');
-const examples = require('./package-examples');
-const attributes = require('./package-attributes');
-const prose = require('./package-prose');
-const contributors = require('./package-contributors');
+const bcd = require('./resolve-bcd');
+const examples = require('./compose-examples');
+const attributes = require('./compose-attributes');
+const prose = require('./slice-prose');
+const contributors = require('./resolve-contributors');
 
 const htmlElements = '/content/html/elements';
 
diff --git a/scripts/package/package-attributes.js b/scripts/build-json/compose-attributes.js
similarity index 100%
rename from scripts/package/package-attributes.js
rename to scripts/build-json/compose-attributes.js
diff --git a/scripts/package/package-examples.js b/scripts/build-json/compose-examples.js
similarity index 100%
rename from scripts/package/package-examples.js
rename to scripts/build-json/compose-examples.js
diff --git a/scripts/package/package-bcd.js b/scripts/build-json/resolve-bcd.js
similarity index 100%
rename from scripts/package/package-bcd.js
rename to scripts/build-json/resolve-bcd.js
diff --git a/scripts/package/package-contributors.js b/scripts/build-json/resolve-contributors.js
similarity index 100%
rename from scripts/package/package-contributors.js
rename to scripts/build-json/resolve-contributors.js
diff --git a/scripts/package/package-prose.js b/scripts/build-json/slice-prose.js
similarity index 100%
rename from scripts/package/package-prose.js
rename to scripts/build-json/slice-prose.js

From 46ad957e068e943286d049edf4f3299f25059ed3 Mon Sep 17 00:00:00 2001
From: Will Bamberg <wbamberg@mozilla.com>
Date: Mon, 13 May 2019 14:42:03 -0700
Subject: [PATCH 7/7] Create output in nested directory

---
 scripts/build-json/build-json.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/build-json/build-json.js b/scripts/build-json/build-json.js
index 4a1dd3fc..1068ca32 100644
--- a/scripts/build-json/build-json.js
+++ b/scripts/build-json/build-json.js
@@ -19,7 +19,7 @@ const writeToFile = (propertyName, json) => {
     }
   };
 
-  const dest = path.join(process.cwd(),'packaged', `${propertyName}.json`);
+  const dest = path.join(process.cwd(),'packaged/html/elements', `${propertyName}.json`);
   const destDir = path.dirname(dest);
   if (!fs.existsSync(destDir)) {
       fs.mkdirSync(destDir, { recursive: true });