From 0a88c4f1c3582516f2d65ae18fa8bafc45e2fd1a Mon Sep 17 00:00:00 2001 From: John Reilly Date: Wed, 21 Apr 2021 07:52:11 +0000 Subject: [PATCH 1/7] start work --- .devcontainer/devcontainer.json | 2 +- .../docusaurus-plugin-content-blog/src/blogFrontMatter.ts | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 99227b6ce8aa..5295b8560034 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "Docusaurus Dev Container", - "image": "mcr.microsoft.com/vscode/devcontainers/typescript-node:0-10-buster", + "image": "mcr.microsoft.com/vscode/devcontainers/typescript-node:12-buster", "settings": { "terminal.integrated.shell.linux": "/bin/bash" }, diff --git a/packages/docusaurus-plugin-content-blog/src/blogFrontMatter.ts b/packages/docusaurus-plugin-content-blog/src/blogFrontMatter.ts index 660a7874cb8e..e1255f4582de 100644 --- a/packages/docusaurus-plugin-content-blog/src/blogFrontMatter.ts +++ b/packages/docusaurus-plugin-content-blog/src/blogFrontMatter.ts @@ -6,6 +6,7 @@ */ import {Joi} from '@docusaurus/utils-validation'; +import chalk from 'chalk'; import {Tag} from './types'; // TODO complete this frontmatter + add unit tests @@ -39,5 +40,10 @@ const BlogFrontMatterSchema = Joi.object({ export function assertBlogPostFrontMatter( frontMatter: Record, ): asserts frontMatter is BlogPostFrontMatter { - Joi.attempt(frontMatter, BlogFrontMatterSchema); + try { + Joi.attempt(frontMatter, BlogFrontMatterSchema, {convert: true}); + } catch (e) { + console.error(chalk.red(`bad frontmatter: ${JSON.stringify(frontMatter)}`)); + throw e; + } } From fbcbd96b008da5c99b51c295e6057908dfd094ff Mon Sep 17 00:00:00 2001 From: John Reilly Date: Wed, 21 Apr 2021 07:53:25 +0000 Subject: [PATCH 2/7] use orta.vscode-jest --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 5295b8560034..c7c6f3e97e90 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -4,7 +4,7 @@ "settings": { "terminal.integrated.shell.linux": "/bin/bash" }, - "extensions": ["dbaeumer.vscode-eslint"], + "extensions": ["dbaeumer.vscode-eslint", "orta.vscode-jest"], "forwardPorts": [3000], "postCreateCommand": "yarn install" } From f18ff390da9ae83ed0f4923f5d0aaafbbe8e8130 Mon Sep 17 00:00:00 2001 From: John Reilly Date: Wed, 21 Apr 2021 09:59:18 +0000 Subject: [PATCH 3/7] node 14 --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c7c6f3e97e90..80716af9c10f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "Docusaurus Dev Container", - "image": "mcr.microsoft.com/vscode/devcontainers/typescript-node:12-buster", + "image": "mcr.microsoft.com/vscode/devcontainers/typescript-node:14-buster", "settings": { "terminal.integrated.shell.linux": "/bin/bash" }, From 1f64d4a8c8e88eb4054e4d6a3c3fe927e619c876 Mon Sep 17 00:00:00 2001 From: slorber Date: Wed, 21 Apr 2021 15:15:19 +0200 Subject: [PATCH 4/7] add some better infra to validate markdown frontmatter --- .../src/__tests__/blogFrontMatter.test.ts | 67 +++++++++++++++++++ .../src/blogFrontMatter.ts | 24 +++---- .../src/blogUtils.ts | 6 +- .../src/__tests__/validationUtils.test.ts | 47 +++++++++++++ .../src/validationUtils.ts | 39 +++++++++++ 5 files changed, 167 insertions(+), 16 deletions(-) create mode 100644 packages/docusaurus-plugin-content-blog/src/__tests__/blogFrontMatter.test.ts create mode 100644 packages/docusaurus-utils-validation/src/__tests__/validationUtils.test.ts diff --git a/packages/docusaurus-plugin-content-blog/src/__tests__/blogFrontMatter.test.ts b/packages/docusaurus-plugin-content-blog/src/__tests__/blogFrontMatter.test.ts new file mode 100644 index 000000000000..d343a47aeb23 --- /dev/null +++ b/packages/docusaurus-plugin-content-blog/src/__tests__/blogFrontMatter.test.ts @@ -0,0 +1,67 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import { + BlogPostFrontMatter, + validateBlogPostFrontMatter, +} from '../blogFrontMatter'; + +describe('validateBlogPostFrontMatter', () => { + test('accept empty object', () => { + const frontMatter = {}; + expect(validateBlogPostFrontMatter(frontMatter)).toEqual(frontMatter); + }); + + test('accept valid values', () => { + const frontMatter: BlogPostFrontMatter = { + id: 'blog', + title: 'title', + description: 'description', + date: 'date', + slug: 'slug', + draft: true, + tags: ['hello', {label: 'tagLabel', permalink: '/tagPermalink'}], + }; + expect(validateBlogPostFrontMatter(frontMatter)).toEqual(frontMatter); + }); + + // See https://github.com/facebook/docusaurus/issues/4591#issuecomment-822372398 + test('accept empty title', () => { + const frontMatter: BlogPostFrontMatter = {title: ''}; + expect(validateBlogPostFrontMatter(frontMatter)).toEqual(frontMatter); + }); + + // See https://github.com/facebook/docusaurus/issues/4591#issuecomment-822372398 + test('accept empty description', () => { + const frontMatter: BlogPostFrontMatter = {description: ''}; + expect(validateBlogPostFrontMatter(frontMatter)).toEqual(frontMatter); + }); + + // See https://github.com/facebook/docusaurus/issues/4642 + test('convert tags as numbers', () => { + const frontMatter: BlogPostFrontMatter = { + tags: [ + // @ts-expect-error: number for test + 42, + { + // @ts-expect-error: number for test + label: 84, + permalink: '/tagPermalink', + }, + ], + }; + expect(validateBlogPostFrontMatter(frontMatter)).toEqual({ + tags: [ + '42', + { + label: '84', + permalink: '/tagPermalink', + }, + ], + }); + }); +}); diff --git a/packages/docusaurus-plugin-content-blog/src/blogFrontMatter.ts b/packages/docusaurus-plugin-content-blog/src/blogFrontMatter.ts index e1255f4582de..dc14b7da6174 100644 --- a/packages/docusaurus-plugin-content-blog/src/blogFrontMatter.ts +++ b/packages/docusaurus-plugin-content-blog/src/blogFrontMatter.ts @@ -5,12 +5,14 @@ * LICENSE file in the root directory of this source tree. */ -import {Joi} from '@docusaurus/utils-validation'; -import chalk from 'chalk'; +import { + JoiFrontMatter as Joi, // Custom instance for frontmatter + validateFrontMatter, +} from '@docusaurus/utils-validation'; import {Tag} from './types'; // TODO complete this frontmatter + add unit tests -type BlogPostFrontMatter = { +export type BlogPostFrontMatter = { id?: string; title?: string; description?: string; @@ -30,20 +32,16 @@ const BlogTagSchema = Joi.alternatives().try( const BlogFrontMatterSchema = Joi.object({ id: Joi.string(), - title: Joi.string(), - description: Joi.string(), + title: Joi.string().allow(''), + description: Joi.string().allow(''), tags: Joi.array().items(BlogTagSchema), slug: Joi.string(), draft: Joi.boolean(), + date: Joi.string().allow(''), // TODO validate the date better! }).unknown(); -export function assertBlogPostFrontMatter( +export function validateBlogPostFrontMatter( frontMatter: Record, -): asserts frontMatter is BlogPostFrontMatter { - try { - Joi.attempt(frontMatter, BlogFrontMatterSchema, {convert: true}); - } catch (e) { - console.error(chalk.red(`bad frontmatter: ${JSON.stringify(frontMatter)}`)); - throw e; - } +): BlogPostFrontMatter { + return validateFrontMatter(frontMatter, BlogFrontMatterSchema); } diff --git a/packages/docusaurus-plugin-content-blog/src/blogUtils.ts b/packages/docusaurus-plugin-content-blog/src/blogUtils.ts index eace24dd319c..759fb74973fc 100644 --- a/packages/docusaurus-plugin-content-blog/src/blogUtils.ts +++ b/packages/docusaurus-plugin-content-blog/src/blogUtils.ts @@ -29,7 +29,7 @@ import { replaceMarkdownLinks, } from '@docusaurus/utils'; import {LoadContext} from '@docusaurus/types'; -import {assertBlogPostFrontMatter} from './blogFrontMatter'; +import {validateBlogPostFrontMatter} from './blogFrontMatter'; export function truncate(fileString: string, truncateMarker: RegExp): string { return fileString.split(truncateMarker, 1).shift()!; @@ -142,12 +142,12 @@ export async function generateBlogPosts( const source = path.join(blogDirPath, blogSourceFile); const { - frontMatter, + frontMatter: unsafeFrontMatter, content, contentTitle, excerpt, } = await parseMarkdownFile(source); - assertBlogPostFrontMatter(frontMatter); + const frontMatter = validateBlogPostFrontMatter(unsafeFrontMatter); const aliasedSource = aliasedSitePath(source, siteDir); diff --git a/packages/docusaurus-utils-validation/src/__tests__/validationUtils.test.ts b/packages/docusaurus-utils-validation/src/__tests__/validationUtils.test.ts new file mode 100644 index 000000000000..438553aa8243 --- /dev/null +++ b/packages/docusaurus-utils-validation/src/__tests__/validationUtils.test.ts @@ -0,0 +1,47 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import Joi from '../Joi'; +import {JoiFrontMatter, validateFrontMatter} from '../validationUtils'; + +describe('validateFrontMatter', () => { + test('should accept good values', () => { + const schema = Joi.object<{test: string}>({ + test: Joi.string(), + }); + const frontMatter = { + test: 'hello', + }; + expect(validateFrontMatter(frontMatter, schema)).toEqual(frontMatter); + }); + + test('should reject bad values', () => { + const consoleError = jest.spyOn(console, 'error').mockImplementation(); + const schema = Joi.object<{test: string}>({ + test: Joi.string(), + }); + const frontMatter = { + test: true, + }; + expect(() => + validateFrontMatter(frontMatter, schema), + ).toThrowErrorMatchingInlineSnapshot(`"\\"test\\" must be a string"`); + expect(consoleError).toHaveBeenCalledWith( + expect.stringContaining('FrontMatter contains invalid values: '), + ); + }); + + test('should convert values', () => { + const schema = Joi.object<{test: string}>({ + test: JoiFrontMatter.string(), + }); + const frontMatter = { + test: 42, + }; + expect(validateFrontMatter(frontMatter, schema)).toEqual({test: '42'}); + }); +}); diff --git a/packages/docusaurus-utils-validation/src/validationUtils.ts b/packages/docusaurus-utils-validation/src/validationUtils.ts index 1ec61a338d51..b1f8ec7d58b2 100644 --- a/packages/docusaurus-utils-validation/src/validationUtils.ts +++ b/packages/docusaurus-utils-validation/src/validationUtils.ts @@ -83,3 +83,42 @@ export function normalizeThemeConfig( } return value; } + +// Enhance the default Joi.string() type so that it can convert number to strings +// If user use frontmatter "tag: 2021", we shouldn't need to ask the user to write "tag: '2021'" +// see https://github.com/facebook/docusaurus/issues/4642 +// see https://github.com/sideway/joi/issues/1442#issuecomment-823997884 +const JoiFrontMatterString: Joi.Extension = { + type: 'string', + base: Joi.string(), + prepare: (value) => { + if (typeof value === 'number') { + return {value: value.toString()}; + } + return {value}; + }, +}; +export const JoiFrontMatter: typeof Joi = Joi.extend(JoiFrontMatterString); + +export function validateFrontMatter( + frontMatter: Record, + schema: Joi.ObjectSchema, +): T { + try { + return JoiFrontMatter.attempt(frontMatter, schema, { + convert: true, + allowUnknown: true, + }); + } catch (e) { + console.error( + chalk.red( + `FrontMatter contains invalid values: ${JSON.stringify( + frontMatter, + null, + 2, + )}`, + ), + ); + throw e; + } +} From 0c87accea05ef93f67504b5b5dec7929a44ef24c Mon Sep 17 00:00:00 2001 From: slorber Date: Wed, 21 Apr 2021 15:26:43 +0200 Subject: [PATCH 5/7] better docs frontmatter validation --- .../src/blogFrontMatter.ts | 4 ++ .../src/__tests__/docFrontMatter.test.ts | 37 +++++++++++++++++++ .../src/docFrontMatter.ts | 18 ++++----- 3 files changed, 50 insertions(+), 9 deletions(-) create mode 100644 packages/docusaurus-plugin-content-docs/src/__tests__/docFrontMatter.test.ts diff --git a/packages/docusaurus-plugin-content-blog/src/blogFrontMatter.ts b/packages/docusaurus-plugin-content-blog/src/blogFrontMatter.ts index dc14b7da6174..ed051aacba73 100644 --- a/packages/docusaurus-plugin-content-blog/src/blogFrontMatter.ts +++ b/packages/docusaurus-plugin-content-blog/src/blogFrontMatter.ts @@ -22,6 +22,10 @@ export type BlogPostFrontMatter = { date?: string; }; +// NOTE: we don't add any default value on purpose here +// We don't want default values to magically appear in doc metadatas and props +// While the user did not provide those values explicitly +// We use default values in code instead const BlogTagSchema = Joi.alternatives().try( Joi.string().required(), Joi.object({ diff --git a/packages/docusaurus-plugin-content-docs/src/__tests__/docFrontMatter.test.ts b/packages/docusaurus-plugin-content-docs/src/__tests__/docFrontMatter.test.ts new file mode 100644 index 000000000000..54b78691e2dc --- /dev/null +++ b/packages/docusaurus-plugin-content-docs/src/__tests__/docFrontMatter.test.ts @@ -0,0 +1,37 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import {DocFrontMatter, validateDocFrontMatter} from '../docFrontMatter'; + +describe('validateDocFrontMatter', () => { + test('accept empty object', () => { + const frontMatter: DocFrontMatter = {}; + expect(validateDocFrontMatter(frontMatter)).toEqual(frontMatter); + }); + + test('accept valid values', () => { + const frontMatter: DocFrontMatter = { + id: 'blog', + title: 'title', + description: 'description', + slug: 'slug', + }; + expect(validateDocFrontMatter(frontMatter)).toEqual(frontMatter); + }); + + // See https://github.com/facebook/docusaurus/issues/4591#issuecomment-822372398 + test('accept empty title', () => { + const frontMatter: DocFrontMatter = {title: ''}; + expect(validateDocFrontMatter(frontMatter)).toEqual(frontMatter); + }); + + // See https://github.com/facebook/docusaurus/issues/4591#issuecomment-822372398 + test('accept empty description', () => { + const frontMatter: DocFrontMatter = {description: ''}; + expect(validateDocFrontMatter(frontMatter)).toEqual(frontMatter); + }); +}); diff --git a/packages/docusaurus-plugin-content-docs/src/docFrontMatter.ts b/packages/docusaurus-plugin-content-docs/src/docFrontMatter.ts index 4fc9282ddaa8..08a57e5d1c32 100644 --- a/packages/docusaurus-plugin-content-docs/src/docFrontMatter.ts +++ b/packages/docusaurus-plugin-content-docs/src/docFrontMatter.ts @@ -5,17 +5,20 @@ * LICENSE file in the root directory of this source tree. */ -import {Joi} from '@docusaurus/utils-validation'; +import { + JoiFrontMatter as Joi, // Custom instance for frontmatter + validateFrontMatter, +} from '@docusaurus/utils-validation'; // TODO complete this frontmatter + add unit tests -type DocFrontMatter = { +export type DocFrontMatter = { id?: string; title?: string; description?: string; slug?: string; sidebar_label?: string; sidebar_position?: number; - custom_edit_url?: string; + custom_edit_url?: string | null; parse_number_prefixes?: boolean; }; @@ -25,8 +28,8 @@ type DocFrontMatter = { // We use default values in code instead const DocFrontMatterSchema = Joi.object({ id: Joi.string(), - title: Joi.string(), - description: Joi.string(), + title: Joi.string().allow(''), // see https://github.com/facebook/docusaurus/issues/4591#issuecomment-822372398 + description: Joi.string().allow(''), // see https://github.com/facebook/docusaurus/issues/4591#issuecomment-822372398 slug: Joi.string(), sidebar_label: Joi.string(), sidebar_position: Joi.number(), @@ -37,8 +40,5 @@ const DocFrontMatterSchema = Joi.object({ export function validateDocFrontMatter( frontMatter: Record, ): DocFrontMatter { - return Joi.attempt(frontMatter, DocFrontMatterSchema, { - convert: true, - allowUnknown: true, - }); + return validateFrontMatter(frontMatter, DocFrontMatterSchema); } From 5b0c416f76c335aa3b5e52ac5f7a3cfb3749289a Mon Sep 17 00:00:00 2001 From: slorber Date: Wed, 21 Apr 2021 15:44:14 +0200 Subject: [PATCH 6/7] fix Yaml / Joi validation issues --- .../src/__tests__/validationUtils.test.ts | 17 ++++++++++++++++- .../src/validationUtils.ts | 4 +++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/packages/docusaurus-utils-validation/src/__tests__/validationUtils.test.ts b/packages/docusaurus-utils-validation/src/__tests__/validationUtils.test.ts index 438553aa8243..cb4a0235ef2c 100644 --- a/packages/docusaurus-utils-validation/src/__tests__/validationUtils.test.ts +++ b/packages/docusaurus-utils-validation/src/__tests__/validationUtils.test.ts @@ -35,7 +35,8 @@ describe('validateFrontMatter', () => { ); }); - test('should convert values', () => { + // Fix Yaml trying to auto-convert strings to numbers automatically + test('should convert number values to string when string schema', () => { const schema = Joi.object<{test: string}>({ test: JoiFrontMatter.string(), }); @@ -44,4 +45,18 @@ describe('validateFrontMatter', () => { }; expect(validateFrontMatter(frontMatter, schema)).toEqual({test: '42'}); }); + + // Helps to fix Yaml trying to auto-convert strings to dates automatically + test('should convert date values when string schema', () => { + const schema = Joi.object<{test: string}>({ + test: JoiFrontMatter.string(), + }); + const date = new Date(); + const frontMatter = { + test: date, + }; + expect(validateFrontMatter(frontMatter, schema)).toEqual({ + test: date.toString(), + }); + }); }); diff --git a/packages/docusaurus-utils-validation/src/validationUtils.ts b/packages/docusaurus-utils-validation/src/validationUtils.ts index b1f8ec7d58b2..72a0f75a8907 100644 --- a/packages/docusaurus-utils-validation/src/validationUtils.ts +++ b/packages/docusaurus-utils-validation/src/validationUtils.ts @@ -86,13 +86,15 @@ export function normalizeThemeConfig( // Enhance the default Joi.string() type so that it can convert number to strings // If user use frontmatter "tag: 2021", we shouldn't need to ask the user to write "tag: '2021'" +// Also yaml tries to convert patterns like "2019-01-01" to dates automatically // see https://github.com/facebook/docusaurus/issues/4642 // see https://github.com/sideway/joi/issues/1442#issuecomment-823997884 const JoiFrontMatterString: Joi.Extension = { type: 'string', base: Joi.string(), + // Fix Yaml that tries to auto-convert many things to string out of the box prepare: (value) => { - if (typeof value === 'number') { + if (typeof value === 'number' || value instanceof Date) { return {value: value.toString()}; } return {value}; From fbfa3457b63282f428f17959a243cf34c9e502e3 Mon Sep 17 00:00:00 2001 From: slorber Date: Wed, 21 Apr 2021 15:46:02 +0200 Subject: [PATCH 7/7] fix Yaml / Joi validation issues --- .../src/__tests__/validationUtils.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/docusaurus-utils-validation/src/__tests__/validationUtils.test.ts b/packages/docusaurus-utils-validation/src/__tests__/validationUtils.test.ts index cb4a0235ef2c..45bc5ec1386b 100644 --- a/packages/docusaurus-utils-validation/src/__tests__/validationUtils.test.ts +++ b/packages/docusaurus-utils-validation/src/__tests__/validationUtils.test.ts @@ -35,7 +35,8 @@ describe('validateFrontMatter', () => { ); }); - // Fix Yaml trying to auto-convert strings to numbers automatically + // Fix Yaml trying to convert strings to numbers automatically + // We only want to deal with a single type in the final frontmatter (not string | number) test('should convert number values to string when string schema', () => { const schema = Joi.object<{test: string}>({ test: JoiFrontMatter.string(), @@ -46,7 +47,8 @@ describe('validateFrontMatter', () => { expect(validateFrontMatter(frontMatter, schema)).toEqual({test: '42'}); }); - // Helps to fix Yaml trying to auto-convert strings to dates automatically + // Helps to fix Yaml trying to convert strings to dates automatically + // We only want to deal with a single type in the final frontmatter (not string | Date) test('should convert date values when string schema', () => { const schema = Joi.object<{test: string}>({ test: JoiFrontMatter.string(),