From 1f62c2491500a886d8a8de3b912d3f43e8723901 Mon Sep 17 00:00:00 2001 From: akabeko Date: Mon, 19 Apr 2021 19:37:58 +0900 Subject: [PATCH] feat: implemented math syntax with MathJax --- docs/vfm.md | 72 +++++++------- package.json | 3 +- src/cli.ts | 6 ++ src/index.ts | 45 +++++++-- src/plugins/math.ts | 190 +++++++++++++++++++++++++++++++++++-- src/plugins/metadata.ts | 31 +++--- src/revive-parse.ts | 22 ++--- src/revive-rehype.ts | 15 ++- tests/defs.test.ts | 19 ++-- tests/index.test.ts | 2 +- tests/math.test.ts | 204 ++++++++++++++-------------------------- tests/metadata.test.ts | 27 +++++- tests/utils.ts | 2 + types/remark.d.ts | 1 + yarn.lock | 19 ++-- 15 files changed, 422 insertions(+), 236 deletions(-) diff --git a/docs/vfm.md b/docs/vfm.md index a4bca9f..6da56d1 100644 --- a/docs/vfm.md +++ b/docs/vfm.md @@ -299,47 +299,44 @@ section.author { ## Math equation -PRE-RELEASE +Outputs HTML processed by [MathJax](https://www.mathjax.org/). + +It is disabled by default. It is activated by satisfying one of the following. + +- VFM options: `math: true` +- CLI options: `--math` +- Frontmatter: `math: true` (Priority over others) **VFM** +The VFM syntax for MathJax inline is `$...$` and the display is `$$...$$`. + ```markdown -$$\sum$$ +inline: $x = y$ + +display: $$1 + 1 = 2$$ ``` **HTML** -```html -

- - - - - -

-``` - -**CSS** +It also outputs the ` + + +

inline: \(x = y\)

+

display: $$1 + 1 = 2$$

+ + ``` ## Frontmatter -PRE-RELEASE - Frontmatter is a way of defining metadata in Markdown (file) units. ```yaml @@ -347,18 +344,20 @@ Frontmatter is a way of defining metadata in Markdown (file) units. title: 'Introduction to VFM' author: 'Author' class: 'my-class' +math: true --- ``` #### Reserved words -| Property | Type | Description | -| -------- | ------ | ------------------------------------------------------------------------------------------- | -| title | String | Document title. If missing, very first heading `#` of the content will be treated as title. | -| author | String | Document author. | -| class | String | Custom classes applied to `` | -| theme | String | Vivliostyle theme package or bare CSS file. | +| Property | Type | Description | +| -------- | ------- | ------------------------------------------------------------------------------------------- | +| title | String | Document title. If missing, very first heading `#` of the content will be treated as title. | +| author | String | Document author. | +| class | String | Custom classes applied to `` | +| math | Boolean | Enable math syntax. | +| theme | String | Vivliostyle theme package or bare CSS file. | The priority of `title` is as follows. @@ -366,6 +365,11 @@ The priority of `title` is as follows. 2. First heading `#` of the content 3. `title` option of VFM +The priority of `math` is as follows. + +1. `math` property of the frontmatter +2. `math` option of VFM + **class** ```yaml diff --git a/package.json b/package.json index e64fbba..5e8b6ba 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "release": "release-it", "release:pre": "release-it --preRelease --npm.tag=latest", "test": "jest", - "test:debug": "jest tests/xxx.test.ts --silent=false --verbose false" + "test:debug": "jest tests/math.test.ts --silent=false --verbose false" }, "dependencies": { "debug": "^4.3.1", @@ -59,6 +59,7 @@ "hast-util-is-element": "^1.1.0", "hastscript": "^6.0.0", "js-yaml": "^4.0.0", + "mdast-util-find-and-replace": "^1.1.1", "mdast-util-to-hast": "^10.1.1", "mdast-util-to-string": "^2.0.0", "meow": "^9.0.0", diff --git a/src/cli.ts b/src/cli.ts index 1f7c30c..9182c49 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -18,6 +18,7 @@ const cli = meow( --language Document language (ignored in partial mode) --hard-line-breaks Add
at the position of hard line breaks, without needing spaces --disable-format-html Disable automatic HTML format + --math Enable math syntax Examples $ vfm input.md @@ -45,6 +46,9 @@ const cli = meow( disableFormatHtml: { type: 'boolean', }, + math: { + type: 'boolean', + }, }, }, ); @@ -59,6 +63,7 @@ function compile(input: string) { language: cli.flags.language, hardLineBreaks: cli.flags.hardLineBreaks, disableFormatHtml: cli.flags.disableFormatHtml, + math: cli.flags.math, }), ); } @@ -71,6 +76,7 @@ function main( language: { type: 'string' }; hardLineBreaks: { type: 'boolean' }; disableFormatHtml: { type: 'boolean' }; + math: { type: 'boolean' }; }>, ) { try { diff --git a/src/index.ts b/src/index.ts index c66ac4a..9bf2794 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,11 +2,12 @@ import doc from 'rehype-document'; import rehypeFormat from 'rehype-format'; import rehypeStringify from 'rehype-stringify'; import unified, { Processor } from 'unified'; -import { hast as clearHtmlLang } from './plugins/clear-html-lang'; -import { hast as metadata } from './plugins/metadata'; +import { hast as hastClearHtmlLang } from './plugins/clear-html-lang'; +import { hast as hastMath } from './plugins/math'; +import { hast as hastMetadata, MetadataVFile } from './plugins/metadata'; import { replace as handleReplace, ReplaceRule } from './plugins/replace'; import { reviveParse as markdown } from './revive-parse'; -import html from './revive-rehype'; +import { reviveRehype as html } from './revive-rehype'; import { debug } from './utils'; /** @@ -27,6 +28,8 @@ export interface StringifyMarkdownOptions { hardLineBreaks?: boolean; /** Disable automatic HTML format. */ disableFormatHtml?: boolean; + /** Enable math syntax. */ + math?: boolean; } export interface Hooks { @@ -34,7 +37,25 @@ export interface Hooks { } /** - * Create Unified processor for MDAST and HAST. + * Update the settings by comparing the options with the frontmatter metadata. + * @param options Options. + * @param md Markdown string. + * @returns Options updated by checking. + */ +const checkOptions = (options: StringifyMarkdownOptions, md: string) => { + // Reduce processing as much as possible because it only reads metadata. + const processor = VFM({ partial: true, disableFormatHtml: true }); + const metadata = (processor.processSync(md) as MetadataVFile).data; + const opts = options; + + opts.title = metadata.title === undefined ? opts.title : metadata.title; + opts.math = metadata.math === undefined ? opts.math : metadata.math; + + return opts; +}; + +/** + * Create Unified processor for Markdown AST and Hypertext AST. * @param options Options. * @returns Unified processor. */ @@ -46,9 +67,10 @@ export function VFM({ replace = undefined, hardLineBreaks = false, disableFormatHtml = false, + math = false, }: StringifyMarkdownOptions = {}): Processor { const processor = unified() - .use(markdown({ hardLineBreaks })) + .use(markdown(hardLineBreaks, math)) .data('settings', { position: false }) .use(html); @@ -59,13 +81,18 @@ export function VFM({ if (!partial) { processor.use(doc, { language, css: style, title }); if (!language) { - processor.use(clearHtmlLang); + processor.use(hastClearHtmlLang); } } - processor.use(metadata); + processor.use(hastMetadata); processor.use(rehypeStringify); + // Must be run after `rehype-document` to write to `` + if (math) { + processor.use(hastMath); + } + // Explicitly specify true if want unformatted HTML during development or debug if (!disableFormatHtml) { processor.use(rehypeFormat); @@ -75,7 +102,7 @@ export function VFM({ } /** - * Convert Markdown to a stringify (HTML). + * Convert markdown to a stringify (HTML). * @param markdownString Markdown string. * @param options Options. * @returns HTML string. @@ -84,7 +111,7 @@ export function stringify( markdownString: string, options: StringifyMarkdownOptions = {}, ): string { - const processor = VFM(options); + const processor = VFM(checkOptions(options, markdownString)); const vfile = processor.processSync(markdownString); debug(vfile.data); return String(vfile); diff --git a/src/plugins/math.ts b/src/plugins/math.ts index b364110..32f4466 100644 --- a/src/plugins/math.ts +++ b/src/plugins/math.ts @@ -1,13 +1,185 @@ -import is from 'hast-util-is-element'; -import katex from 'rehype-katex'; +import { Element } from 'hast'; +import findReplace from 'mdast-util-find-and-replace'; +import { Handler } from 'mdast-util-to-hast'; +import { Plugin, Transformer } from 'unified'; import { Node } from 'unist'; -import remove from 'unist-util-remove'; +import u from 'unist-builder'; +import visit from 'unist-util-visit'; -const removeMathML = () => (tree: Node) => { - remove(tree, (node: any) => { - const isKatexMathML = node.properties?.className?.includes('katex-mathml'); - return is(node, 'span') && isKatexMathML; - }); +/** Inline math format, e.g. `$...$`. */ +const regexpInline = /\$([^$].*?[^$])\$(?!\$)/g; + +/** Display math format, e.g. `$$...$$`. */ +const regexpDisplay = /\$\$([^$].*?[^$])\$\$(?!\$)/g; + +/** Type of inline math in Markdown AST. */ +const typeInline = 'inlineMath'; + +/** Type of display math in Markdown AST. */ +const typeDisplay = 'displayMath'; + +/** URL of MathJax v2 supported by Vivliostyle. */ +const mathUrl = + 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.9/MathJax.js?config=TeX-MML-AM_CHTML'; + +const createTokenizer = () => { + const tokenizerInlineMath: Tokenizer = function (eat, value, silent) { + if (!value.startsWith('$') || value.startsWith('$$')) { + return; + } + + const match = new RegExp(regexpInline).exec(value); + if (!match) { + return; + } + + if (silent) { + return true; + } + + const [eaten, valueText] = match; + const now = eat.now(); + now.column += 1; + now.offset += 1; + + return eat(eaten)({ + type: typeInline, + children: [], + data: { hName: typeInline, value: valueText }, + }); + }; + + tokenizerInlineMath.notInLink = true; + tokenizerInlineMath.locator = function (value: string, fromIndex: number) { + return value.indexOf('$', fromIndex); + }; + + const tokenizerDisplayMath: Tokenizer = function (eat, value, silent) { + if (!value.startsWith('$$') || value.startsWith('$$$')) { + return; + } + + const match = new RegExp(regexpDisplay).exec(value); + if (!match) { + return; + } + + if (silent) { + return true; + } + + const [eaten, valueText] = match; + const now = eat.now(); + now.column += 1; + now.offset += 1; + + return eat(eaten)({ + type: typeDisplay, + children: [], + data: { hName: typeDisplay, value: valueText }, + }); + }; + + tokenizerDisplayMath.notInLink = true; + tokenizerDisplayMath.locator = function (value: string, fromIndex: number) { + return value.indexOf('$$', fromIndex); + }; + + return { tokenizerInlineMath, tokenizerDisplayMath }; +}; + +/** + * Process Markdown AST. + * @returns Transformer or undefined (less than remark 13). + */ +export const mdast: Plugin = function (): Transformer | undefined { + // For less than remark 13 with exclusive other markdown syntax + if ( + this.Parser && + this.Parser.prototype.inlineTokenizers && + this.Parser.prototype.inlineMethods + ) { + const { inlineTokenizers, inlineMethods } = this.Parser.prototype; + const tokenizers = createTokenizer(); + inlineTokenizers[typeInline] = tokenizers.tokenizerInlineMath; + inlineTokenizers[typeDisplay] = tokenizers.tokenizerDisplayMath; + inlineMethods.splice(inlineMethods.indexOf('text'), 0, typeInline); + inlineMethods.splice(inlineMethods.indexOf('text'), 0, typeDisplay); + return; + } + + return (tree: Node) => { + findReplace(tree, regexpInline, (_: string, valueText: string) => { + return { + type: typeInline, + data: { + hName: typeInline, + value: valueText, + }, + children: [], + }; + }); + + findReplace(tree, regexpDisplay, (_: string, valueText: string) => { + return { + type: typeDisplay, + data: { + hName: typeDisplay, + value: valueText, + }, + children: [], + }; + }); + }; }; -export const hast = { plugins: [katex, removeMathML] }; +/** + * Handle inline math to Hypertext AST. + * @param h Hypertext AST formatter. + * @param node Node. + * @returns Hypertext AST. + */ +export const handlerInlineMath: Handler = (h, node: Node) => { + if (!node.data) node.data = {}; + + return u('text', `\\(${node.data.value as string}\\)`); +}; + +/** + * Handle display math to Hypertext AST. + * @param h Hypertext AST formatter. + * @param node Node. + * @returns Hypertext AST. + */ +export const handlerDisplayMath: Handler = (h, node: Node) => { + if (!node.data) node.data = {}; + + return u('text', `$$${node.data.value as string}$$`); +}; + +/** + * Process math related Hypertext AST. + * Set the ` + + +

\\(x=y\\)

+ + +`; + expect(received).toBe(expected); +}); - it( - 'math with newline', - buildProcessorTestingCode( - `$x\ny$`, - stripIndent` - root[1] - └─0 paragraph[1] - └─0 inlineMath "x\\ny" - data: {"hName":"span","hProperties":{"className":["math","math-inline"]},"hChildren":[{"type":"text","value":"x\\ny"}]} - `, - `

`, - ), - ); +it('disable', () => { + const markdown = `--- +math: false +--- +$x=y$ +`; + const received = stringify(markdown, { math: true, disableFormatHtml: true }); + const expected = ` + + + + + + +

$x=y$

+ + +`; + expect(received).toBe(expected); }); diff --git a/tests/metadata.test.ts b/tests/metadata.test.ts index a2d0c6f..afd4003 100644 --- a/tests/metadata.test.ts +++ b/tests/metadata.test.ts @@ -1,4 +1,4 @@ -import { stringify } from '../src/index'; +import { stringify, VFM } from '../src/index'; it('all', () => { const received = stringify( @@ -15,9 +15,9 @@ class: 'my-class' + Title - Title
@@ -35,8 +35,8 @@ it('title from heading, missing "title" property of Frontmatter', () => { - Page Title +
@@ -73,6 +73,27 @@ class: 'my-class' expect(received).toBe(expected); }); +it('title with only frontmatter', () => { + const md = `--- +title: 'Title' +--- +`; + const received = String(VFM().processSync(md)); + // Since there is no option update that takes into account the stringify metadata, + // `` is added to the end by your own processing instead of `rehype-document`. + const expected = `<!doctype html> +<html> + <head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <title>Title + + + +`; + expect(received).toBe(expected); +}); + it('overwrite optional title by frontmatter', () => { const received = stringify( `--- diff --git a/tests/utils.ts b/tests/utils.ts index 89c4e05..4b7e349 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -20,6 +20,7 @@ export const buildProcessorTestingCode = ( replace = undefined, hardLineBreaks = false, disableFormatHtml = true, + math = false, }: StringifyMarkdownOptions = {}, ) => (): any => { const vfm = VFM({ @@ -30,6 +31,7 @@ export const buildProcessorTestingCode = ( replace, hardLineBreaks, disableFormatHtml, + math, }).freeze(); expect(unistInspect.noColor(vfm.parse(input))).toBe(expectedMdast.trim()); expect(String(vfm.processSync(input))).toBe(expectedHtml); diff --git a/types/remark.d.ts b/types/remark.d.ts index 9cc56a9..edb8690 100644 --- a/types/remark.d.ts +++ b/types/remark.d.ts @@ -10,6 +10,7 @@ declare module 'hast-util-is-element'; declare module 'hastscript'; declare module 'mdast-util-to-hast/lib/all'; declare module 'mdast-util-to-string'; +declare module 'mdast-util-find-and-replace'; declare module 'rehype-katex'; declare module 'remark-slug'; declare module 'rehype-slug'; diff --git a/yarn.lock b/yarn.lock index 4470ce7..23a191a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4813,6 +4813,15 @@ dependencies: "unist-util-visit" "^2.0.0" +"mdast-util-find-and-replace@^1.1.1": + "integrity" "sha512-9cKl33Y21lyckGzpSmEQnIDjEfeeWelN5s1kUW1LwdB0Fkuq2u+4GdqcGEygYxJE8GVqCl0741bYXHgamfWAZA==" + "resolved" "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-1.1.1.tgz" + "version" "1.1.1" + dependencies: + "escape-string-regexp" "^4.0.0" + "unist-util-is" "^4.0.0" + "unist-util-visit-parents" "^3.0.0" + "mdast-util-to-hast@^10.0.0", "mdast-util-to-hast@^10.1.1": "integrity" "sha512-+hvJrYiUgK2aY0Q1h1LaHQ4h0P7VVumWdAcUuG9k49lYglyU9GtTrA4O8hMh5gRnyT22wC15takM2qrrlpvNxQ==" "resolved" "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-10.1.1.tgz" @@ -7338,15 +7347,7 @@ dependencies: "unist-util-is" "^3.0.0" -"unist-util-visit-parents@^3.0.0": - "integrity" "sha512-yJEfuZtzFpQmg1OSCyS9M5NJRrln/9FbYosH3iW0MG402QbdbaB8ZESwUv9RO6nRfLAKvWcMxCwdLWOov36x/g==" - "resolved" "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-3.0.2.tgz" - "version" "3.0.2" - dependencies: - "@types/unist" "^2.0.0" - "unist-util-is" "^4.0.0" - -"unist-util-visit-parents@^3.1.1": +"unist-util-visit-parents@^3.0.0", "unist-util-visit-parents@^3.1.1": "integrity" "sha512-1KROIZWo6bcMrZEwiH2UrXDyalAa0uqzWCxCJj6lPOvTve2WkfgCytoDTPaMnodXh1WrXOq0haVYHj99ynJlsg==" "resolved" "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-3.1.1.tgz" "version" "3.1.1"