feat(v2): generalize usage of _ prefix convention to exclude content files/folders

packages/docusaurus-mdx-loader/src/index.js

@@ -24,6 +24,19 @@ const DEFAULT_OPTIONS = {
   remarkPlugins: [unwrapMdxCodeBlocks, emoji, headings, toc],
 };
 
+// When this throws, it generally means that there's no metadata file associated with this MDX document
+// It can happen when using MDX partials (usually starting with _)
+// That's why it's important to provide the "isMDXPartial" function in config
+async function readMetadataPath(metadataPath) {
+  try {
+    return await readFile(metadataPath, 'utf8');
+  } catch (e) {
+    throw new Error(
+      `MDX loader can't read MDX metadata file for path ${metadataPath}. Maybe the isMDXPartial option function was not provided?`,
+    );
+  }
+}
+
 module.exports = async function docusaurusMdxLoader(fileString) {
   const callback = this.async();
 
@@ -37,19 +50,15 @@ module.exports = async function docusaurusMdxLoader(fileString) {
 
   const hasFrontMatter = Object.keys(frontMatter).length > 0;
 
+  const filePath = this.resourcePath;
+
   const options = {
     ...reqOptions,
     remarkPlugins: [
       ...(reqOptions.beforeDefaultRemarkPlugins || []),
       ...DEFAULT_OPTIONS.remarkPlugins,
-      [
-        transformImage,
-        {staticDir: reqOptions.staticDir, filePath: this.resourcePath},
-      ],
-      [
-        transformLinks,
-        {staticDir: reqOptions.staticDir, filePath: this.resourcePath},
-      ],
+      [transformImage, {staticDir: reqOptions.staticDir, filePath}],
+      [transformLinks, {staticDir: reqOptions.staticDir, filePath}],
       ...(reqOptions.remarkPlugins || []),
     ],
     rehypePlugins: [
@@ -58,7 +67,7 @@ module.exports = async function docusaurusMdxLoader(fileString) {
 
       ...(reqOptions.rehypePlugins || []),
     ],
-    filepath: this.resourcePath,
+    filepath: filePath,
   };
   let result;
 
@@ -74,27 +83,33 @@ module.exports = async function docusaurusMdxLoader(fileString) {
     contentTitle,
   )};`;
 
-  // Read metadata for this MDX and export it.
-  if (options.metadataPath && typeof options.metadataPath === 'function') {
-    const metadataPath = options.metadataPath(this.resourcePath);
+  // MDX partials are MDX files starting with _ or in a folder starting with _
+  // Partial are not expected to have an associated metadata file or frontmatter
+  const isMDXPartial = options.isMDXPartial
+    ? options.isMDXPartial(filePath)
+    : false;
 
-    if (metadataPath) {
-      // Add as dependency of this loader result so that we can
-      // recompile if metadata is changed.
-      this.addDependency(metadataPath);
-      const metadata = await readFile(metadataPath, 'utf8');
-      exportStr += `\nexport const metadata = ${metadata};`;
-    }
+  if (isMDXPartial && hasFrontMatter) {
+    return callback(
+      new Error(`MDX partial should not contain FrontMatter: ${filePath}`),
+    );
   }
 
-  if (
-    options.forbidFrontMatter &&
-    typeof options.forbidFrontMatter === 'function'
-  ) {
-    if (options.forbidFrontMatter(this.resourcePath) && hasFrontMatter) {
-      return callback(new Error(`Front matter is forbidden in this file`));
+  if (!isMDXPartial) {
+    // Read metadata for this MDX and export it.
+    if (options.metadataPath && typeof options.metadataPath === 'function') {
+      const metadataPath = options.metadataPath(filePath);
+
+      if (metadataPath) {
+        const metadata = await readMetadataPath(metadataPath);
+        exportStr += `\nexport const metadata = ${metadata};`;
+        // Add as dependency of this loader result so that we can
+        // recompile if metadata is changed.
+        this.addDependency(metadataPath);
+      }
     }
   }
+
   const code = `
   import React from 'react';
   import { mdx } from '@mdx-js/react';

packages/docusaurus-plugin-content-blog/src/__tests__/__fixtures__/website/blog/_partials/somePartial.md

@@ -0,0 +1,3 @@
+This is a partial in file/folder starting with _:
+
+It should be excluded by default

packages/docusaurus-plugin-content-blog/src/__tests__/__fixtures__/website/blog/_partials/subfolder/somePartial.md

@@ -0,0 +1,3 @@
+This is a partial in file/folder starting with _:
+
+It should be excluded by default

packages/docusaurus-plugin-content-blog/src/__tests__/__fixtures__/website/blog/_somePartial.md

@@ -0,0 +1,3 @@
+This is a partial in file/folder starting with \_:
+
+It should be excluded by default

packages/docusaurus-plugin-content-blog/src/__tests__/__snapshots__/generateBlogFeed.test.ts.snap

@@ -7,19 +7,39 @@ exports[`blogFeed atom shows feed item for each post 1`] = `
 <feed xmlns=\\"http://www.w3.org/2005/Atom\\">
     <id>https://docusaurus.io/myBaseUrl/blog</id>
     <title>Hello Blog</title>
-    <updated>2020-02-27T00:00:00.000Z</updated>
+    <updated>2020-08-16T00:00:00.000Z</updated>
     <generator>https://github.com/jpmonette/feed</generator>
     <link rel=\\"alternate\\" href=\\"https://docusaurus.io/myBaseUrl/blog\\"/>
     <subtitle>Hello Blog</subtitle>
     <icon>https://docusaurus.io/myBaseUrl/image/favicon.ico</icon>
     <rights>Copyright</rights>
+    <entry>
+        <title type=\\"html\\"><![CDATA[Complex Slug]]></title>
+        <id>/hey/my super path/héllô</id>
+        <link href=\\"https://docusaurus.io/myBaseUrl/blog/hey/my super path/héllô\\"/>
+        <updated>2020-08-16T00:00:00.000Z</updated>
+        <summary type=\\"html\\"><![CDATA[complex url slug]]></summary>
+    </entry>
+    <entry>
+        <title type=\\"html\\"><![CDATA[Simple Slug]]></title>
+        <id>/simple/slug</id>
+        <link href=\\"https://docusaurus.io/myBaseUrl/blog/simple/slug\\"/>
+        <updated>2020-08-15T00:00:00.000Z</updated>
+        <summary type=\\"html\\"><![CDATA[simple url slug]]></summary>
+    </entry>
     <entry>
         <title type=\\"html\\"><![CDATA[draft]]></title>
         <id>draft</id>
         <link href=\\"https://docusaurus.io/myBaseUrl/blog/draft\\"/>
         <updated>2020-02-27T00:00:00.000Z</updated>
         <summary type=\\"html\\"><![CDATA[this post should not be published yet]]></summary>
     </entry>
+    <entry>
+        <title type=\\"html\\"><![CDATA[some heading]]></title>
+        <id>some heading</id>
+        <link href=\\"https://docusaurus.io/myBaseUrl/blog/heading-as-title\\"/>
+        <updated>2019-01-02T00:00:00.000Z</updated>
+    </entry>
     <entry>
         <title type=\\"html\\"><![CDATA[date-matter]]></title>
         <id>date-matter</id>
@@ -46,17 +66,37 @@ exports[`blogFeed rss shows feed item for each post 1`] = `
         <title>Hello Blog</title>
         <link>https://docusaurus.io/myBaseUrl/blog</link>
         <description>Hello Blog</description>
-        <lastBuildDate>Thu, 27 Feb 2020 00:00:00 GMT</lastBuildDate>
+        <lastBuildDate>Sun, 16 Aug 2020 00:00:00 GMT</lastBuildDate>
         <docs>https://validator.w3.org/feed/docs/rss2.html</docs>
         <generator>https://github.com/jpmonette/feed</generator>
         <copyright>Copyright</copyright>
+        <item>
+            <title><![CDATA[Complex Slug]]></title>
+            <link>https://docusaurus.io/myBaseUrl/blog/hey/my super path/héllô</link>
+            <guid>/hey/my super path/héllô</guid>
+            <pubDate>Sun, 16 Aug 2020 00:00:00 GMT</pubDate>
+            <description><![CDATA[complex url slug]]></description>
+        </item>
+        <item>
+            <title><![CDATA[Simple Slug]]></title>
+            <link>https://docusaurus.io/myBaseUrl/blog/simple/slug</link>
+            <guid>/simple/slug</guid>
+            <pubDate>Sat, 15 Aug 2020 00:00:00 GMT</pubDate>
+            <description><![CDATA[simple url slug]]></description>
+        </item>
         <item>
             <title><![CDATA[draft]]></title>
             <link>https://docusaurus.io/myBaseUrl/blog/draft</link>
             <guid>draft</guid>
             <pubDate>Thu, 27 Feb 2020 00:00:00 GMT</pubDate>
             <description><![CDATA[this post should not be published yet]]></description>
         </item>
+        <item>
+            <title><![CDATA[some heading]]></title>
+            <link>https://docusaurus.io/myBaseUrl/blog/heading-as-title</link>
+            <guid>some heading</guid>
+            <pubDate>Wed, 02 Jan 2019 00:00:00 GMT</pubDate>
+        </item>
         <item>
             <title><![CDATA[date-matter]]></title>
             <link>https://docusaurus.io/myBaseUrl/blog/date-matter</link>

packages/docusaurus-plugin-content-blog/src/__tests__/generateBlogFeed.test.ts

@@ -9,6 +9,7 @@ import path from 'path';
 import {generateBlogFeed} from '../blogUtils';
 import {LoadContext, I18n} from '@docusaurus/types';
 import {PluginOptions, BlogContentPaths} from '../types';
+import {DEFAULT_OPTIONS} from '../pluginOptionSchema';
 
 const DefaultI18N: I18n = {
   currentLocale: 'en',
@@ -84,7 +85,8 @@ describe('blogFeed', () => {
           {
             path: 'blog',
             routeBasePath: 'blog',
-            include: ['*r*.md', '*.mdx'], // skip no-date.md - it won't play nice with snapshots
+            include: DEFAULT_OPTIONS.include,
+            exclude: DEFAULT_OPTIONS.exclude,
             feedOptions: {
               type: [feedType],
               copyright: 'Copyright',

packages/docusaurus-plugin-content-blog/src/blogUtils.ts

@@ -6,7 +6,6 @@
  */
 
 import fs from 'fs-extra';
-import globby from 'globby';
 import chalk from 'chalk';
 import path from 'path';
 import readingTime from 'reading-time';
@@ -27,6 +26,7 @@ import {
   getFolderContainingFile,
   posixPath,
   replaceMarkdownLinks,
+  Globby,
 } from '@docusaurus/utils';
 import {LoadContext} from '@docusaurus/types';
 import {validateBlogPostFrontMatter} from './blogFrontMatter';
@@ -127,6 +127,7 @@ export async function generateBlogPosts(
 ): Promise<BlogPost[]> {
   const {
     include,
+    exclude,
     routeBasePath,
     truncateMarker,
     showReadingTime,
@@ -138,8 +139,9 @@ export async function generateBlogPosts(
   }
 
   const {baseUrl = ''} = siteConfig;
-  const blogSourceFiles = await globby(include, {
+  const blogSourceFiles = await Globby(include, {
     cwd: contentPaths.contentPath,
+    ignore: exclude,
   });
 
   const blogPosts: BlogPost[] = [];

packages/docusaurus-plugin-content-blog/src/index.ts

@@ -16,6 +16,7 @@ import {
   reportMessage,
   posixPath,
   addTrailingPathSeparator,
+  createMatcher,
 } from '@docusaurus/utils';
 import {
   STATIC_DIR_NAME,
@@ -459,9 +460,10 @@ export default function pluginContentBlog(
                     beforeDefaultRemarkPlugins,
                     beforeDefaultRehypePlugins,
                     staticDir: path.join(siteDir, STATIC_DIR_NAME),
-                    // Note that metadataPath must be the same/in-sync as
-                    // the path from createData for each MDX.
+                    isMDXPartial: createMatcher(options.exclude),
                     metadataPath: (mdxPath: string) => {
+                      // Note that metadataPath must be the same/in-sync as
+                      // the path from createData for each MDX.
                       const aliasedPath = aliasedSitePath(mdxPath, siteDir);
                       return path.join(
                         dataDir,

packages/docusaurus-plugin-content-blog/src/pluginOptionSchema.ts

@@ -12,6 +12,7 @@ import {
   AdmonitionsSchema,
   URISchema,
 } from '@docusaurus/utils-validation';
+import {GlobExcludeDefault} from '@docusaurus/utils';
 
 export const DEFAULT_OPTIONS = {
   feedOptions: {type: ['rss', 'atom']},
@@ -32,6 +33,7 @@ export const DEFAULT_OPTIONS = {
   blogSidebarTitle: 'Recent posts',
   postsPerPage: 10,
   include: ['*.md', '*.mdx'],
+  exclude: GlobExcludeDefault,
   routeBasePath: 'blog',
   path: 'blog',
   editLocalizedFiles: false,
@@ -44,6 +46,7 @@ export const PluginOptionSchema = Joi.object({
     // .allow('')
     .default(DEFAULT_OPTIONS.routeBasePath),
   include: Joi.array().items(Joi.string()).default(DEFAULT_OPTIONS.include),
+  exclude: Joi.array().items(Joi.string()).default(DEFAULT_OPTIONS.exclude),
   postsPerPage: Joi.number()
     .integer()
     .min(1)

packages/docusaurus-plugin-content-blog/src/types.ts

@@ -39,6 +39,7 @@ export interface PluginOptions extends RemarkAndRehypePluginOptions {
   path: string;
   routeBasePath: string;
   include: string[];
+  exclude: string[];
   postsPerPage: number;
   blogListComponent: string;
   blogPostComponent: string;

packages/docusaurus-plugin-content-docs/src/__tests__/__fixtures__/simple-site/docs/_partials/somePartial.md

@@ -0,0 +1,3 @@
+This is a partial in file/folder starting with _:
+
+It should be excluded by default

packages/docusaurus-plugin-content-docs/src/__tests__/__fixtures__/simple-site/docs/_partials/subfolder/somePartial.md

@@ -0,0 +1,3 @@
+This is a partial in file/folder starting with _:
+
+It should be excluded by default

packages/docusaurus-plugin-content-docs/src/__tests__/__fixtures__/simple-site/docs/_somePartial.md

@@ -0,0 +1,3 @@
+This is a partial in file/folder starting with \_:
+
+It should be excluded by default

packages/docusaurus-plugin-content-docs/src/__tests__/__fixtures__/versioned-site/versioned_docs/version-1.0.1/_partials/somePartial.md

@@ -0,0 +1,3 @@
+This is a partial in file/folder starting with _:
+
+It should be excluded by default

packages/docusaurus-plugin-content-docs/src/__tests__/__fixtures__/versioned-site/versioned_docs/version-1.0.1/_partials/subfolder/somePartial.md

@@ -0,0 +1,3 @@
+This is a partial in file/folder starting with _:
+
+It should be excluded by default

packages/docusaurus-plugin-content-docs/src/__tests__/__fixtures__/versioned-site/versioned_docs/version-1.0.1/_somePartial.md

@@ -0,0 +1,3 @@
+This is a partial in file/folder starting with \_:
+
+It should be excluded by default

packages/docusaurus-plugin-content-docs/src/__tests__/options.test.ts

@@ -12,6 +12,7 @@ import {
   DefaultNumberPrefixParser,
   DisabledNumberPrefixParser,
 } from '../numberPrefix';
+import {GlobExcludeDefault} from '@docusaurus/utils';
 
 // the type of remark/rehype plugins is function
 const markdownPluginsFunctionStub = () => {};
@@ -30,6 +31,7 @@ describe('normalizeDocsPluginOptions', () => {
       routeBasePath: 'my-docs', // URL Route.
       homePageId: 'home', // Document id for docs home page.
       include: ['**/*.{md,mdx}'], // Extensions to include.
+      exclude: GlobExcludeDefault,
       sidebarPath: 'my-sidebar', // Path to sidebar configuration for showing a list of markdown pages.
       sidebarItemsGenerator: DefaultSidebarItemsGenerator,
       numberPrefixParser: DefaultNumberPrefixParser,

packages/docusaurus-plugin-content-docs/src/docs.ts

@@ -14,6 +14,7 @@ import {
   normalizeUrl,
   parseMarkdownString,
   posixPath,
+  Globby,
 } from '@docusaurus/utils';
 import {LoadContext} from '@docusaurus/types';
 
@@ -28,7 +29,6 @@ import {
 } from './types';
 import getSlug from './slug';
 import {CURRENT_VERSION_NAME} from './constants';
-import globby from 'globby';
 import {getDocsDirPaths} from './versions';
 import {stripPathNumberPrefixes} from './numberPrefix';
 import {validateDocFrontMatter} from './docFrontMatter';
@@ -92,11 +92,12 @@ export async function readVersionDocs(
   versionMetadata: VersionMetadata,
   options: Pick<
     PluginOptions,
-    'include' | 'showLastUpdateAuthor' | 'showLastUpdateTime'
+    'include' | 'exclude' | 'showLastUpdateAuthor' | 'showLastUpdateTime'
   >,
 ): Promise<DocFile[]> {
-  const sources = await globby(options.include, {
+  const sources = await Globby(options.include, {
     cwd: versionMetadata.contentPath,
+    ignore: options.exclude,
   });
   return Promise.all(
     sources.map((source) => readDocFile(versionMetadata, source, options)),

packages/docusaurus-plugin-content-docs/src/index.ts

@@ -18,6 +18,7 @@ import {
   reportMessage,
   posixPath,
   addTrailingPathSeparator,
+  createMatcher,
 } from '@docusaurus/utils';
 import {LoadContext, Plugin, RouteConfig} from '@docusaurus/types';
 import {loadSidebars, createSidebarsUtils, processSidebars} from './sidebars';
@@ -408,6 +409,7 @@ export default function pluginContentDocs(
                 beforeDefaultRehypePlugins,
                 beforeDefaultRemarkPlugins,
                 staticDir: path.join(siteDir, STATIC_DIR_NAME),
+                isMDXPartial: createMatcher(options.exclude),
                 metadataPath: (mdxPath: string) => {
                   // Note that metadataPath must be the same/in-sync as
                   // the path from createData for each MDX.