organize serializers

src/components/Widgets/Markdown/MarkdownControl/RawEditor/index.js

@@ -1,6 +1,6 @@
 import React, { PropTypes } from 'react';
 import { Editor as Slate, Plain } from 'slate';
-import { markdownToRemark, remarkToMarkdown } from '../../unified';
+import { markdownToRemark, remarkToMarkdown } from '../../serializers';
 import Toolbar from '../Toolbar/Toolbar';
 import { Sticky } from '../../../../UI/Sticky/Sticky';
 import styles from './index.css';

src/components/Widgets/Markdown/MarkdownControl/VisualEditor/index.js

@@ -1,7 +1,7 @@
 import React, { Component, PropTypes } from 'react';
 import { get, isEmpty } from 'lodash';
 import { Editor as Slate, Raw, Block, Text } from 'slate';
-import { slateToRemark, remarkToSlate, htmlToSlate } from '../../unified';
+import { slateToRemark, remarkToSlate, htmlToSlate } from '../../serializers';
 import registry from '../../../../../lib/registry';
 import Toolbar from '../Toolbar/Toolbar';
 import { Sticky } from '../../../../UI/Sticky/Sticky';

src/components/Widgets/Markdown/MarkdownControl/index.js

@@ -1,6 +1,6 @@
 import React, { PropTypes } from 'react';
 import registry from '../../../../lib/registry';
-import { markdownToRemark, remarkToMarkdown } from '../unified';
+import { markdownToRemark, remarkToMarkdown } from '../serializers'
 import RawEditor from './RawEditor';
 import VisualEditor from './VisualEditor';
 import { StickyContainer } from '../../../UI/Sticky/Sticky';

src/components/Widgets/Markdown/MarkdownPreview/index.js

@@ -1,5 +1,5 @@
 import React, { PropTypes } from 'react';
-import { remarkToHtml } from '../unified';
+import { remarkToHtml } from '../serializers';
 import previewStyle from '../../defaultPreviewStyle';
 
 const MarkdownPreview = ({ value, getAsset }) => {

src/components/Widgets/Markdown/serializers/index.js

@@ -0,0 +1,203 @@
+import { get, isEmpty, reduce } from 'lodash';
+import unified from 'unified';
+import u from 'unist-builder';
+import markdownToRemarkPlugin from 'remark-parse';
+import remarkToMarkdownPlugin from 'remark-stringify';
+import remarkToRehype from 'remark-rehype';
+import rehypeToHtml from 'rehype-stringify';
+import htmlToRehype from 'rehype-parse';
+import rehypeToRemark from 'rehype-remark';
+import rehypeMinifyWhitespace from 'rehype-minify-whitespace';
+import remarkToRehypeShortcodes from './remark-rehype-shortcodes';
+import rehypeRemoveEmpty from './rehype-remove-empty';
+import rehypePaperEmoji from './rehype-paper-emoji';
+import remarkNestedList from './remark-nested-list';
+import remarkToSlatePlugin from './remark-slate';
+import remarkImagesToText from './remark-images-to-text';
+import remarkShortcodes from './remark-shortcodes';
+import registry from '../../../../lib/registry';
+
+export const remarkToHtml = (mdast, getAsset) => {
+  const result = unified()
+    .use(remarkToRehypeShortcodes, { plugins: registry.getEditorComponents(), getAsset })
+    .use(remarkToRehype, { allowDangerousHTML: true })
+    .runSync(mdast);
+
+  const output = unified()
+    .use(rehypeToHtml, { allowDangerousHTML: true, allowDangerousCharacters: true })
+    .stringify(result);
+  return output
+}
+
+export const htmlToSlate = html => {
+  const hast = unified()
+    .use(htmlToRehype, { fragment: true })
+    .parse(html);
+
+  const result = unified()
+    .use(rehypeRemoveEmpty)
+    .use(rehypeMinifyWhitespace)
+    .use(rehypePaperEmoji)
+    .use(rehypeToRemark)
+    .use(remarkNestedList)
+    .use(remarkToSlatePlugin)
+    .runSync(hast);
+
+  return result;
+};
+
+export const markdownToRemark = markdown => {
+  const parsed = unified()
+    .use(markdownToRemarkPlugin, { fences: true, pedantic: true, footnotes: true, commonmark: true })
+    .parse(markdown);
+
+  const result = unified()
+    .use(remarkImagesToText)
+    .use(remarkShortcodes, { plugins: registry.getEditorComponents() })
+    .runSync(parsed);
+
+  return result;
+};
+
+export const remarkToMarkdown = obj => {
+  /**
+   * Rewrite the remark-stringify text visitor to simply return the text value,
+   * without encoding or escaping any characters. This means we're completely
+   * trusting the markdown that we receive.
+   */
+  function remarkAllowAllText() {
+    const Compiler = this.Compiler;
+    const visitors = Compiler.prototype.visitors;
+    visitors.text = node => node.value;
+  };
+
+  const mdast = obj || u('root', [u('paragraph', [u('text', '')])]);
+  const result = unified()
+    .use(remarkToMarkdownPlugin, { listItemIndent: '1', fences: true, pedantic: true, commonmark: true })
+    .use(remarkAllowAllText)
+    .stringify(mdast);
+  return result;
+};
+
+export const remarkToSlate = mdast => {
+  const result = unified()
+    .use(remarkToSlatePlugin)
+    .runSync(mdast);
+  return result;
+};
+
+export const slateToRemark = (raw, shortcodePlugins) => {
+  const typeMap = {
+    'paragraph': 'paragraph',
+    'heading-one': 'heading',
+    'heading-two': 'heading',
+    'heading-three': 'heading',
+    'heading-four': 'heading',
+    'heading-five': 'heading',
+    'heading-six': 'heading',
+    'quote': 'blockquote',
+    'code': 'code',
+    'numbered-list': 'list',
+    'bulleted-list': 'list',
+    'list-item': 'listItem',
+    'table': 'table',
+    'table-row': 'tableRow',
+    'table-cell': 'tableCell',
+    'thematic-break': 'thematicBreak',
+    'link': 'link',
+    'image': 'image',
+  };
+  const markMap = {
+    bold: 'strong',
+    italic: 'emphasis',
+    strikethrough: 'delete',
+    code: 'inlineCode',
+  };
+  const transform = node => {
+    const children = isEmpty(node.nodes) ? node.nodes : node.nodes.reduce((acc, childNode) => {
+      if (childNode.kind !== 'text') {
+        acc.push(transform(childNode));
+        return acc;
+      }
+      if (childNode.ranges) {
+        childNode.ranges.forEach(range => {
+          const { marks = [], text } = range;
+          const markTypes = marks.map(mark => markMap[mark.type]);
+          if (markTypes.includes('inlineCode')) {
+            acc.push(u('inlineCode', text));
+          } else {
+            const textNode = u('html', text);
+            const nestedText = !markTypes.length ? textNode : markTypes.reduce((acc, markType) => {
+              const nested = u(markType, [acc]);
+              return nested;
+            }, textNode);
+            acc.push(nestedText);
+          }
+        });
+      } else {
+
+        acc.push(u('html', childNode.text));
+      }
+      return acc;
+    }, []);
+
+    if (node.type === 'root') {
+      return u('root', children);
+    }
+
+    if (node.type === 'shortcode') {
+      const { data } = node;
+      const plugin = shortcodePlugins.get(data.shortcode);
+      const text = plugin.toBlock(data.shortcodeData);
+      const textNode = u('html', text);
+      return u('paragraph', { data }, [ textNode ]);
+    }
+
+    if (node.type.startsWith('heading')) {
+      const depths = { one: 1, two: 2, three: 3, four: 4, five: 5, six: 6 };
+      const depth = node.type.split('-')[1];
+      const props = { depth: depths[depth] };
+      return u(typeMap[node.type], props, children);
+    }
+
+    if (['paragraph', 'quote', 'list-item', 'table', 'table-row', 'table-cell'].includes(node.type)) {
+      return u(typeMap[node.type], children);
+    }
+
+    if (node.type === 'code') {
+      const value = get(node.nodes, [0, 'text']);
+      const props = { lang: get(node.data, 'lang') };
+      return u(typeMap[node.type], props, value);
+    }
+
+    if (['numbered-list', 'bulleted-list'].includes(node.type)) {
+      const ordered = node.type === 'numbered-list';
+      const props = { ordered, start: get(node.data, 'start') || 1 };
+      return u(typeMap[node.type], props, children);
+    }
+
+    if (node.type === 'thematic-break') {
+      return u(typeMap[node.type]);
+    }
+
+    if (node.type === 'link') {
+      const data = get(node, 'data', {});
+      const { url, title } = data;
+      return u(typeMap[node.type], data, children);
+    }
+
+    if (node.type === 'image') {
+      const data = get(node, 'data', {});
+      const { url, title, alt } = data;
+      return u(typeMap[node.type], data);
+    }
+  }
+  raw.type = 'root';
+  const mdast = transform(raw);
+
+  const result = unified()
+    .use(remarkShortcodes, { plugins: registry.getEditorComponents() })
+    .runSync(mdast);
+
+  return result;
+};

src/components/Widgets/Markdown/serializers/rehype-paper-emoji.js

@@ -0,0 +1,15 @@
+/**
+ * Dropbox Paper outputs emoji characters as images, and stores the actual
+ * emoji character in a `data-emoji-ch` attribute on the image. This plugin
+ * replaces the images with the emoji characters.
+ */
+export default function rehypePaperEmoji() {
+  const transform = node => {
+    if (node.tagName === 'img' && node.properties.dataEmojiCh) {
+      return { type: 'text', value: node.properties.dataEmojiCh };
+    }
+    node.children = node.children ? node.children.map(transform) : node.children;
+    return node;
+  };
+  return transform;
+}

src/components/Widgets/Markdown/serializers/rehype-remove-empty.js

@@ -0,0 +1,32 @@
+import { find, capitalize } from 'lodash';
+
+/**
+ * Remove empty nodes, including the top level parents of deeply nested empty nodes.
+ */
+export default function rehypeRemoveEmpty() {
+  const isVoidElement = node => ['img', 'hr', 'br'].includes(node.tagName);
+  const isNonEmptyLeaf = node => ['text', 'raw'].includes(node.type) && node.value;
+  const isShortcode = node => node.properties && node.properties[`data${capitalize(shortcodeAttributePrefix)}`];
+  const isNonEmptyNode =  node => {
+    return isVoidElement(node)
+      || isNonEmptyLeaf(node)
+      || isShortcode(node)
+      || find(node.children, isNonEmptyNode);
+  };
+
+  const transform = node => {
+    if (isVoidElement(node) || isNonEmptyLeaf(node) || isShortcode(node)) {
+      return node;
+    }
+    if (node.children) {
+      node.children = node.children.reduce((acc, childNode) => {
+        if (isVoidElement(childNode) || isNonEmptyLeaf(childNode) || isShortcode(node)) {
+          return acc.concat(childNode);
+        }
+        return find(childNode.children, isNonEmptyNode) ? acc.concat(transform(childNode)) : acc;
+      }, []);
+    }
+    return node;
+  };
+  return transform;
+}

src/components/Widgets/Markdown/serializers/remark-images-to-text.js

@@ -0,0 +1,18 @@
+/**
+ * Images must be parsed as shortcodes for asset proxying. This plugin converts
+ * MDAST image nodes back to text to allow shortcode pattern matching.
+ */
+export default function remarkImagesToText() {
+  return transform;
+
+  function transform(node) {
+    const children = node.children ? node.children.map(transform) : node.children;
+    if (node.type === 'image') {
+      const alt = node.alt || '';
+      const url = node.url || '';
+      const title = node.title ? ` "${node.title}"` : '';
+      return { type: 'text', value: `![${alt}](${url}${title})` };
+    }
+    return { ...node, children };
+  }
+}

src/components/Widgets/Markdown/serializers/remark-nested-list.js

@@ -0,0 +1,33 @@
+/**
+ * If the first child of a list item is a list, include it in the previous list
+ * item. Otherwise it translates to markdown as having two bullets. When
+ * rehype-remark processes a list and finds children that are not list items, it
+ * wraps them in list items, which leads to the condition this plugin addresses.
+ * Dropbox Paper currently outputs this kind of HTML, which is invalid. We have
+ * a support issue open for it, and this plugin can potentially be removed when
+ * that's resolved.
+ */
+
+export default function remarkNestedList() {
+  const transform = node => {
+    if (node.type === 'list' && node.children && node.children.length > 1) {
+      node.children = node.children.reduce((acc, childNode, index) => {
+        if (index && childNode.children && childNode.children[0].type === 'list') {
+          acc[acc.length - 1].children.push(transform(childNode.children.shift()))
+          if (childNode.children.length) {
+            acc.push(transform(childNode));
+          }
+        } else {
+          acc.push(transform(childNode));
+        }
+        return acc;
+      }, []);
+      return node;
+    }
+    if (node.children) {
+      node.children = node.children.map(transform);
+    }
+    return node;
+  };
+  return transform;
+}

src/components/Widgets/Markdown/serializers/remark-rehype-shortcodes.js

@@ -0,0 +1,50 @@
+import { map, has } from 'lodash';
+import { renderToString } from 'react-dom/server';
+import u from 'unist-builder';
+
+/**
+ * This plugin doesn't actually transform Remark (MDAST) nodes to Rehype
+ * (HAST) nodes, but rather, it prepares an MDAST shortcode node for HAST
+ * conversion by replacing the shortcode text with stringified HTML for
+ * previewing the shortcode output.
+ */
+export default function remarkToRehypeShortcodes({ plugins, getAsset }) {
+  return transform;
+
+  function transform(root) {
+    const transformedChildren = map(root.children, processShortcodes);
+    return { ...root, children: transformedChildren };
+  }
+
+  /**
+   * Mapping function to transform nodes that contain shortcodes.
+   */
+  function processShortcodes(node) {
+    /**
+     * If the node doesn't contain shortcode data, return the original node.
+     */
+    if (!has(node, ['data', 'shortcode'])) return node;
+
+    /**
+     * Get shortcode data from the node, and retrieve the matching plugin by
+     * key.
+     */
+    const { shortcode, shortcodeData } = node.data;
+    const plugin = plugins.get(shortcode);
+
+    /**
+     * Run the shortcode plugin's `toPreview` method, which will return either
+     * an HTML string or a React component. If a React component is returned,
+     * render it to an HTML string.
+     */
+    const value = plugin.toPreview(shortcodeData, getAsset);
+    const valueHtml = typeof value === 'string' ? value : renderToString(value);
+
+    /**
+     * Return a new 'html' type node containing the shortcode preview markup.
+     */
+    const textNode = u('html', valueHtml);
+    const children = [ textNode ];
+    return { ...node, children };
+  }
+}