Add experimental content layer flag (#11652)

* Add experimental content layer flag

* Syntax and format

* Aside

* Format

* Reset content config between runs

* Update fixture

* Update terminology

* Lint

* wut

packages/astro/src/@types/astro.ts

@@ -124,6 +124,7 @@ export type TransitionAnimationValue =
 	| TransitionDirectionalAnimations;
 
 // Allow users to extend this for astro-jsx.d.ts
+
 // eslint-disable-next-line @typescript-eslint/no-empty-object-type
 export interface AstroClientDirectives {}
 
@@ -2184,6 +2185,258 @@ export interface AstroUserConfig {
 		 * For a complete overview, and to give feedback on this experimental API, see the [Server Islands RFC](https://github.com/withastro/roadmap/pull/963).
 		 */
 		serverIslands?: boolean;
+
+		/**
+		 * @docs
+		 * @name experimental.contentLayer
+		 * @type {boolean}
+		 * @default `false`
+		 * @version 4.16.0
+		 * @description
+		 *
+		 * The Content Layer API is a new way to handle content and data in Astro. It takes [content collections](https://docs.astro.build/en/guides/content-collections/) beyond local files in `src/content` and allowing you to fetch content from anywhere, including remote APIs, or files anywhere in your project. As well as being more powerful, the Content Layer API is designed to be more performant, helping sites scale to tens of thousands of pages. Data is cached between builds and updated incrementally. Markdown parsing is also 5-10x faster, with similar scale reductions in memory. While the feature is experimental and subject to breaking changes, we invite you to try it today and let us know how it works for you.
+		 *
+		 * #### Enabling the Content Layer API
+		 *
+		 * To enable, add the `contentLayer` flag to the `experimental` object in your Astro config:
+		 *
+		 * ```js
+		 * {
+		 * 	experimental: {
+		 * 		contentLayer: true,
+		 * 	}
+		 * }
+		 * ```
+		 *
+		 * #### Using the Content Layer API
+		 *
+		 * :::tip
+		 * The Content Layer API is a new way to define content collections, but many of the APIs are the same. It will be helpful to refer to the current [content collection docs](https://docs.astro.build/en/guides/content-collections/) for more information. Any differences in the API usage are highlighted below.
+		 * :::
+		 *
+		 * To use the Content Layer API, create a collection in `src/content/config.ts` with a `loader` property. For local files where there is one entry per file, use the `glob()` loader. You can put your content files anywhere, but *not* in `src/content` because these would be handled by the current content collections APIs instead. In this example the files are in `src/data`.
+		 *
+		 * ```ts
+		 * import { defineCollection, z } from 'astro:content';
+		 * import { glob } from 'astro/loaders';
+		 *
+		 * const blog = defineCollection({
+		 *   // By default the ID is a slug, generated from the path of the file relative to `base`
+		 *   loader: glob({ pattern: "**\/*.md", base: "./src/data/blog" }),
+		 *   schema: z.object({
+		 *     title: z.string(),
+		 *     description: z.string(),
+		 *     pubDate: z.coerce.date(),
+		 *     updatedDate: z.coerce.date().optional(),
+		 *   }),
+		 * });
+		 *
+		 * export const collections = { blog };
+		 * ```
+		 *
+		 * You can load multiple entries from a single JSON file using the `file()` loader. In this case the data must either be an array of objects, which each contain an `id` property, or an object where each key is the ID.
+		 *
+		 * **Array syntax:**
+		 *
+		 * ```json
+		 * [
+		 *   {
+		 *     "id": "labrador-retriever",
+		 *     "breed": "Labrador Retriever",
+		 *     "size": "Large",
+		 *     "origin": "Canada",
+		 *     "lifespan": "10-12 years",
+		 *     "temperament": [
+		 *       "Friendly",
+		 *       "Active",
+		 *       "Outgoing"
+		 *     ]
+		 *   },
+		 *   {
+		 *     "id": "german-shepherd",
+		 *     "breed": "German Shepherd",
+		 *     "size": "Large",
+		 *     "origin": "Germany",
+		 *     "lifespan": "9-13 years",
+		 *     "temperament": [
+		 *       "Loyal",
+		 *       "Intelligent",
+		 *       "Confident"
+		 *     ]
+		 *   }
+		 * ]
+		 * ```
+		 *
+		 * **Object syntax:**
+		 *
+		 * ```json
+		 * {
+		 *   "labrador-retriever": {
+		 *     "breed": "Labrador Retriever",
+		 *     "size": "Large",
+		 *     "origin": "Canada",
+		 *     "lifespan": "10-12 years",
+		 *     "temperament": [
+		 *       "Friendly",
+		 *       "Active",
+		 *       "Outgoing"
+		 *     ]
+		 *   },
+		 *   "german-shepherd": {
+		 *     "breed": "German Shepherd",
+		 *     "size": "Large",
+		 *     "origin": "Germany",
+		 *     "lifespan": "9-13 years",
+		 *     "temperament": [
+		 *       "Loyal",
+		 *       "Intelligent",
+		 *       "Confident"
+		 *     ]
+		 *   }
+		 * }
+		 * ```
+		 *
+		 * The collection is then defined using the `file()` loader:
+		 *
+		 * ```ts
+		 * import { defineCollection, z } from 'astro:content';
+		 * import { file } from 'astro/loaders';
+		 *
+		 * const dogs = defineCollection({
+		 * 	loader: file('src/data/dogs.json'),
+		 * 	schema: z.object({
+		 * 		id: z.string(),
+		 *     breed: z.string(),
+		 * 		size: z.string(),
+		 * 		origin: z.string(),
+		 * 		lifespan: z.string(),
+		 * 		temperament: z.array(z.string()),
+		 * 	}),
+		 * });
+		 *
+		 * export const collections = { dogs };
+		 * ```
+		 *
+		 * The collection can be queried in the same way as existing content collections:
+		 *
+		 * ```ts
+		 * import { getCollection, getEntry } from 'astro:content';
+		 *
+		 * // Get all entries from a collection.
+		 * // Requires the name of the collection as an argument.
+		 * const allBlogPosts = await getCollection('blog');
+		 *
+		 * // Get a single entry from a collection.
+		 * // Requires the name of the collection and ID
+		 * const labradorData = await getEntry('dogs', 'labrador-retriever');
+		 * ```
+		 *
+		 * #### Rendering content
+		 *
+		 * Entries generated from markdown or MDX can be rendered directly to a page using the `render()` function.
+		 *
+		 * :::caution
+		 * The syntax for rendering entries from collections that use the Content Layer is different from current content collections syntax.
+		 * :::
+		 *
+		 * ```astro
+		 * ---
+		 * import { getEntry, render } from 'astro:content';
+		 *
+		 * const post = await getEntry('blog', Astro.params.slug);
+		 *
+		 * const { Content, headings } = await render(entry);
+		 * ---
+		 *
+		 * <Content />
+		 * ```
+		 *
+		 * #### Creating a loader
+		 *
+		 * Content loaders aren't restricted to just loading local files. You can also use loaders to fetch or generate content from anywhere. The simplest type of loader is an async function that returns an array of objects, each of which has an `id`:
+		 *
+		 * ```ts
+		 * const countries = defineCollection({
+		 *   loader: async () => {
+		 *     const response = await fetch("https://restcountries.com/v3.1/all");
+		 *     const data = await response.json();
+		 *     // Must return an array of entries with an id property, or an object with IDs as keys and entries as values
+		 *     return data.map((country) => ({
+		 *       id: country.cca3,
+		 *       ...country,
+		 *     }));
+		 *   },
+		 *   // optionally add a schema
+		 *   // schema: z.object...
+		 * });
+		 *
+		 * export const collections = { countries };
+		 * ```
+		 *
+		 * For more advanced loading logic, you can define an object loader. This allows incremental updates and conditional loading, and gives full access to the data store. See the API in [the RFC](https://github.com/withastro/roadmap/blob/content-layer/proposals/content-layer.md#loaders).
+		 *
+		 * ### Migrating an existing content collection to use the Content Layer API
+		 *
+		 * You can convert an existing content collection to use the Content Layer API if it uses markdown, MDX or JSON, with these steps:
+		 *
+		 * 1. **Move the collection folder out of `src/content`.** This is so it won't be handled using the existing content collection APIs. This example assumes the content has been moved to `src/data`. The `config.ts` file must remain in `src/content`.
+		 * 2. **Edit the collection definition**. The collection should not have `type` set, and needs a `loader` defined.
+		 *
+		 * ```diff
+		 * import { defineCollection, z } from 'astro:content';
+		 * + import { glob } from 'astro/loaders';
+		 *
+		 * const blog = defineCollection({
+		 *   // For content layer you do not define a `type`
+		 * - type: 'content',
+		 * + loader: glob({ pattern: "**\/*.md", base: "./src/data/blog" }),
+		 *   schema: z.object({
+		 * 		title: z.string(),
+		 * 		description: z.string(),
+		 * 		pubDate: z.coerce.date(),
+		 * 		updatedDate: z.coerce.date().optional(),
+		 * 	}),
+		 * });
+		 * ```
+		 *
+		 * 3. **Change references from `slug` to `id`**. Content collections created with the Content Layer API do not have a `slug` field. You should use `id` instead, which has the same syntax.
+		 *
+		 * ```diff
+		 * ---
+		 * export async function getStaticPaths() {
+		 * 	const posts = await getCollection('blog');
+		 * 	return posts.map((post) => ({
+		 * -   params: { slug: post.slug },
+		 * +   params: { slug: post.id },
+		 *     props: post,
+		 * 	}));
+		 * }
+		 * ---
+		 * ```
+		 *
+		 * 4. **Switch to the new `render()` function**. Entries no longer have a `render()` method, as they are now serializable plain objects. Instead, import the `render()` function from `astro:content`.
+		 *
+		 * ```diff
+		 * ---
+		 * - import { getEntry } from 'astro:content';
+		 * + import { getEntry, render } from 'astro:content';
+		 *
+		 *   const post = await getEntry('blog', params.slug);
+		 *
+		 * - const { Content, headings } = await post.render();
+		 * + const { Content, headings } = await render(post);
+		 * ---
+		 *
+		 * <Content />
+		 * ```
+		 *
+		 * The `getEntryBySlug` and `getDataEntryByID` functions are deprecated and cannot be used with collections that use the Content Layer API. Instead, use `getEntry`, which is a drop-in replacement for both.
+		 *
+		 * #### Learn more
+		 *
+		 * To see the full API look at [the RFC](https://github.com/withastro/roadmap/blob/content-layer/proposals/content-layer.md) and [share your feedback on the feature and API](https://github.com/withastro/roadmap/pull/982).
+		 */
+		contentLayer?: boolean;
 	};
 }
 

packages/astro/src/content/consts.ts

@@ -40,4 +40,4 @@ export const DATA_STORE_FILE = 'data-store.json';
 export const ASSET_IMPORTS_FILE = 'assets.mjs';
 export const MODULES_IMPORTS_FILE = 'modules.mjs';
 
-export const CONTENT_LAYER_TYPE = 'experimental_content';
+export const CONTENT_LAYER_TYPE = 'content_layer';

packages/astro/src/content/content-layer.ts

@@ -4,6 +4,7 @@ import { fileURLToPath } from 'node:url';
 import type { FSWatcher } from 'vite';
 import xxhash from 'xxhash-wasm';
 import type { AstroSettings } from '../@types/astro.js';
+import { AstroUserError } from '../core/errors/errors.js';
 import type { Logger } from '../core/logger/core.js';
 import {
 	ASSET_IMPORTS_FILE,
@@ -125,13 +126,26 @@ export class ContentLayer {
 	}
 
 	async #doSync() {
-		const logger = this.#logger.forkIntegrationLogger('content');
-		logger.info('Syncing content');
 		const contentConfig = globalContentConfigObserver.get();
+		const logger = this.#logger.forkIntegrationLogger('content');
 		if (contentConfig?.status !== 'loaded') {
 			logger.debug('Content config not loaded, skipping sync');
 			return;
 		}
+		if (!this.#settings.config.experimental.contentLayer) {
+			const contentLayerCollections = Object.entries(contentConfig.config.collections).filter(
+				([_, collection]) => collection.type === CONTENT_LAYER_TYPE,
+			);
+			if (contentLayerCollections.length > 0) {
+				throw new AstroUserError(
+					`The following collections have a loader defined, but the content layer is not enabled: ${contentLayerCollections.map(([title]) => title).join(', ')}.`,
+					'To enable the Content Layer API, set `experimental: { contentLayer: true }` in your Astro config file.',
+				);
+			}
+			return;
+		}
+
+		logger.info('Syncing content');
 		const { digest: currentConfigDigest } = contentConfig.config;
 		this.#lastConfigDigest = currentConfigDigest;
 

packages/astro/src/content/runtime.ts

@@ -27,15 +27,14 @@ type CollectionToEntryMap = Record<string, GlobResult>;
 type GetEntryImport = (collection: string, lookupId: string) => Promise<LazyImport>;
 
 export function defineCollection(config: any) {
-	if (
-		('loader' in config && config.type !== CONTENT_LAYER_TYPE) ||
-		(config.type === CONTENT_LAYER_TYPE && !('loader' in config))
-	) {
-		// TODO: when this moves out of experimental, we will set the type automatically
-		throw new AstroUserError(
-			'Collections that use the content layer must have a `loader` defined and `type` set to `experimental_content`',
-			"Check your collection definitions in `src/content/config.*`.'",
-		);
+	if ('loader' in config) {
+		if (config.type && config.type !== CONTENT_LAYER_TYPE) {
+			throw new AstroUserError(
+				'Collections that use the Content Layer API must have a `loader` defined and no `type` set.',
+				"Check your collection definitions in `src/content/config.*`.'",
+			);
+		}
+		config.type = CONTENT_LAYER_TYPE;
 	}
 	if (!config.type) config.type = 'content';
 	return config;

packages/astro/src/core/config/schema.ts

@@ -92,6 +92,7 @@ export const ASTRO_CONFIG_DEFAULTS = {
 		env: {
 			validateSecrets: false,
 		},
+		contentLayer: false,
 	},
 } satisfies AstroUserConfig & { server: { open: boolean } };
 
@@ -538,6 +539,7 @@ export const AstroConfigSchema = z.object({
 				.boolean()
 				.optional()
 				.default(ASTRO_CONFIG_DEFAULTS.experimental.serverIslands),
+			contentLayer: z.boolean().optional().default(ASTRO_CONFIG_DEFAULTS.experimental.contentLayer),
 		})
 		.strict(
 			`Invalid or outdated experimental feature.\nCheck for incorrect spelling or outdated Astro version.\nSee https://docs.astro.build/en/reference/configuration-reference/#experimental-flags for a list of all current experiments.`,

packages/astro/src/core/dev/dev.ts

@@ -122,6 +122,9 @@ export default async function dev(inlineConfig: AstroInlineConfig): Promise<DevS
 	}
 
 	const config = globalContentConfigObserver.get();
+	if (config.status === 'error') {
+		logger.error('content', config.error.message);
+	}
 	if (config.status === 'loaded') {
 		const contentLayer = globalContentLayer.init({
 			settings: restart.container.settings,