feat: add glob loader

packages/astro/package.json

@@ -67,6 +67,7 @@
     "./assets/services/sharp": "./dist/assets/services/sharp.js",
     "./assets/services/squoosh": "./dist/assets/services/squoosh.js",
     "./assets/services/noop": "./dist/assets/services/noop.js",
+    "./loaders": "./dist/content/loaders/index.js",
     "./content/runtime": "./dist/content/runtime.js",
     "./content/runtime-assets": "./dist/content/runtime-assets.js",
     "./debug": "./components/Debug.astro",
@@ -165,6 +166,7 @@
     "js-yaml": "^4.1.0",
     "kleur": "^4.1.5",
     "magic-string": "^0.30.10",
+    "micromatch": "^4.0.7",
     "mrmime": "^2.0.0",
     "ora": "^8.0.1",
     "p-limit": "^5.0.0",
@@ -208,6 +210,7 @@
     "@types/html-escaper": "^3.0.2",
     "@types/http-cache-semantics": "^4.0.4",
     "@types/js-yaml": "^4.0.9",
+    "@types/micromatch": "^4.0.9",
     "@types/probe-image-size": "^7.2.4",
     "@types/prompts": "^2.4.9",
     "@types/semver": "^7.5.8",
@@ -240,4 +243,4 @@
   "publishConfig": {
     "provenance": true
   }
-}
+}

packages/astro/src/@types/astro.ts

@@ -46,7 +46,7 @@ import type {
 	TransitionBeforePreparationEvent,
 	TransitionBeforeSwapEvent,
 } from '../transitions/events.js';
-import type { DeepPartial, OmitIndexSignature, Simplify, WithRequired } from '../type-utils.js';
+import type { DeepPartial, OmitIndexSignature, Simplify } from '../type-utils.js';
 import type { SUPPORTED_MARKDOWN_FILE_EXTENSIONS } from './../core/constants.js';
 
 export type { AstroIntegrationLogger, ToolbarServerHelpers };

packages/astro/src/content/data-store.ts

@@ -1,6 +1,14 @@
 import { promises as fs, type PathLike, existsSync } from 'fs';
 
 const SAVE_DEBOUNCE_MS = 500;
+
+export interface DataEntry {
+	id: string;
+	data: Record<string, unknown>;
+	filePath?: string;
+	body?: string;
+}
+
 export class DataStore {
 	#collections = new Map<string, Map<string, any>>();
 
@@ -13,14 +21,14 @@ export class DataStore {
 	constructor() {
 		this.#collections = new Map();
 	}
-	get(collectionName: string, key: string) {
+	get<T = unknown>(collectionName: string, key: string): T | undefined {
 		return this.#collections.get(collectionName)?.get(String(key));
 	}
-	entries(collectionName: string): Array<[id: string, any]> {
+	entries<T = unknown>(collectionName: string): Array<[id: string, T]> {
 		const collection = this.#collections.get(collectionName) ?? new Map();
 		return [...collection.entries()];
 	}
-	values(collectionName: string): Array<unknown> {
+	values<T = unknown>(collectionName: string): Array<T> {
 		const collection = this.#collections.get(collectionName) ?? new Map();
 		return [...collection.values()];
 	}
@@ -78,19 +86,44 @@ export class DataStore {
 
 	scopedStore(collectionName: string): ScopedDataStore {
 		return {
-			get: (key: string) => this.get(collectionName, key),
+			get: (key: string) => this.get<DataEntry>(collectionName, key),
 			entries: () => this.entries(collectionName),
 			values: () => this.values(collectionName),
 			keys: () => this.keys(collectionName),
-			set: (key: string, value: any) => this.set(collectionName, key, value),
+			set: (key, data, body, filePath) => {
+				if (!key) {
+					throw new Error(`Key must be a non-empty string`);
+				}
+				const id = String(key);
+				const entry: DataEntry = {
+					id,
+					data,
+				};
+				// We do it like this so we don't waste space stringifying
+				// the body and filePath if they are not set
+				if (body) {
+					entry.body = body;
+				}
+				if (filePath) {
+					entry.filePath = filePath;
+				}
+
+				this.set(collectionName, id, entry);
+			},
 			delete: (key: string) => this.delete(collectionName, key),
 			clear: () => this.clear(collectionName),
 			has: (key: string) => this.has(collectionName, key),
 		};
 	}
 
 	metaStore(collectionName: string): MetaStore {
-		return this.scopedStore(`meta:${collectionName}`) as MetaStore;
+		const collectionKey = `meta:${collectionName}`;
+		return {
+			get: (key: string) => this.get(collectionKey, key),
+			set: (key: string, data: string) => this.set(collectionKey, key, data),
+			delete: (key: string) => this.delete(collectionKey, key),
+			has: (key: string) => this.has(collectionKey, key),
+		};
 	}
 
 	toString() {
@@ -148,10 +181,10 @@ export class DataStore {
 }
 
 export interface ScopedDataStore {
-	get: (key: string) => unknown;
-	entries: () => Array<[id: string, unknown]>;
-	set: (key: string, value: unknown) => void;
-	values: () => Array<unknown>;
+	get: (key: string) => DataEntry | undefined;
+	entries: () => Array<[id: string, DataEntry]>;
+	set: (key: string, data: Record<string, unknown>, body?: string, filePath?: string) => void;
+	values: () => Array<DataEntry>;
 	keys: () => Array<string>;
 	delete: (key: string) => void;
 	clear: () => void;
@@ -166,6 +199,7 @@ export interface MetaStore {
 	get: (key: string) => string | undefined;
 	set: (key: string, value: string) => void;
 	has: (key: string) => boolean;
+	delete: (key: string) => void;
 }
 
 function dataStoreSingleton() {

packages/astro/src/content/file.ts → packages/astro/src/content/loaders/file.ts

@@ -1,6 +1,6 @@
-import { fileURLToPath } from 'url';
-import type { Loader, LoaderContext } from './loaders.js';
 import { promises as fs, existsSync } from 'fs';
+import { fileURLToPath } from 'url';
+import type { Loader, LoaderContext } from './types.js';
 
 /**
  * Loads entries from a JSON file. The file must contain an array of objects that contain unique `id` fields, or an object with string keys.
@@ -38,7 +38,7 @@ export function file(fileName: string): Loader {
 					continue;
 				}
 				const item = await parseData({ id, data: rawItem, filePath });
-				store.set(id, item);
+				store.set(id, item, undefined, filePath);
 			}
 		} else if (typeof json === 'object') {
 			const entries = Object.entries<Record<string, unknown>>(json);
@@ -57,9 +57,8 @@ export function file(fileName: string): Loader {
 		name: 'file-loader',
 		load: async (options) => {
 			const { settings, logger, watcher } = options;
-			const contentDir = new URL('./content/', settings.config.srcDir);
 			logger.debug(`Loading data from ${fileName}`);
-			const url = new URL(fileName, contentDir);
+			const url = new URL(fileName, settings.config.root);
 			if (!existsSync(url)) {
 				logger.error(`File not found: ${fileName}`);
 				return;

packages/astro/src/content/loaders/glob.ts

@@ -0,0 +1,176 @@
+import { promises as fs } from 'fs';
+import { fileURLToPath, pathToFileURL } from 'url';
+import fastGlob from 'fast-glob';
+import { green } from 'kleur/colors';
+import micromatch from 'micromatch';
+import pLimit from 'p-limit';
+import { relative } from 'path/posix';
+import type { ContentEntryType } from '../../@types/astro.js';
+import { getContentEntryIdAndSlug, getEntryConfigByExtMap } from '../utils.js';
+import type { Loader, LoaderContext } from './types.js';
+
+export interface GenerateIdOptions {
+	/** The path to the entry file, relative to the base directory. */
+	entry: string;
+
+	/** The base directory URL. */
+	base: URL;
+	/** The parsed, unvalidated data of the entry. */
+	data: Record<string, unknown>;
+}
+export interface GlobOptions {
+	/** The glob pattern to match files, relative to the base directory */
+	pattern: string;
+	/** The base directory to resolve the glob pattern from. Relative to the root directory, or an absolute file URL. Defaults to `.` */
+	base?: string | URL;
+	/**
+	 * Function that generates an ID for an entry. Default implementation generates a slug from the entry path.
+	 * @returns The ID of the entry. Must be unique per collection.
+	 **/
+	generateId?: (options: GenerateIdOptions) => string;
+}
+
+function generateIdDefault({ entry, base, data }: GenerateIdOptions): string {
+	if (data.slug) {
+		return data.slug as string;
+	}
+	const entryURL = new URL(entry, base);
+	const { slug } = getContentEntryIdAndSlug({
+		entry: entryURL,
+		contentDir: base,
+		collection: '',
+	});
+	return slug;
+}
+
+/**
+ * Loads multiple entries, using a glob pattern to match files.
+ * @param pattern A glob pattern to match files, relative to the content directory.
+ */
+export function glob(globOptions: GlobOptions): Loader {
+	if (globOptions.pattern.startsWith('../')) {
+		throw new Error(
+			'Glob patterns cannot start with `../`. Set the `base` option to a parent directory instead.'
+		);
+	}
+	if (globOptions.pattern.startsWith('/')) {
+		throw new Error(
+			'Glob patterns cannot start with `/`. Set the `base` option to a parent directory or use a relative path instead.'
+		);
+	}
+
+	const generateId = globOptions?.generateId ?? generateIdDefault;
+
+	const fileToIdMap = new Map<string, string>();
+
+	async function syncData(
+		entry: string,
+		base: URL,
+		{ logger, parseData, store }: LoaderContext,
+		entryType?: ContentEntryType
+	) {
+		if (!entryType) {
+			logger.warn(`No entry type found for ${entry}`);
+			return;
+		}
+		const fileUrl = new URL(entry, base);
+		const { body, data } = await entryType.getEntryInfo({
+			contents: await fs.readFile(fileUrl, 'utf-8'),
+			fileUrl,
+		});
+
+		const id = generateId({ entry, base, data });
+
+		const filePath = fileURLToPath(fileUrl);
+
+		const parsedData = await parseData({
+			id,
+			data,
+			filePath,
+		});
+
+		store.set(id, parsedData, body, filePath);
+
+		fileToIdMap.set(filePath, id);
+	}
+
+	return {
+		name: 'glob-loader',
+		load: async (options) => {
+			const { settings, logger, watcher } = options;
+
+			const entryConfigByExt = getEntryConfigByExtMap([
+				...settings.contentEntryTypes,
+				...settings.dataEntryTypes,
+			] as Array<ContentEntryType>);
+
+			const baseDir = globOptions.base
+				? new URL(globOptions.base, settings.config.root)
+				: settings.config.root;
+
+			if (!baseDir.pathname.endsWith('/')) {
+				baseDir.pathname = `${baseDir.pathname}/`;
+			}
+
+			const files = await fastGlob(globOptions.pattern, {
+				cwd: fileURLToPath(baseDir),
+			});
+
+			function configForFile(file: string) {
+				const ext = file.split('.').at(-1);
+				if (!ext) {
+					logger.warn(`No extension found for ${file}`);
+					return;
+				}
+				return entryConfigByExt.get(`.${ext}`);
+			}
+
+			const limit = pLimit(10);
+			options.store.clear();
+			await Promise.all(
+				files.map((entry) =>
+					limit(async () => {
+						const entryType = configForFile(entry);
+						await syncData(entry, baseDir, options, entryType);
+					})
+				)
+			);
+
+			if (!watcher) {
+				return;
+			}
+
+			const matcher: RegExp = micromatch.makeRe(globOptions.pattern);
+
+			const matchesGlob = (entry: string) => !entry.startsWith('../') && matcher.test(entry);
+
+			const basePath = fileURLToPath(baseDir);
+
+			async function onChange(changedPath: string) {
+				const entry = relative(basePath, changedPath);
+				if (!matchesGlob(entry)) {
+					return;
+				}
+				const entryType = configForFile(changedPath);
+				const baseUrl = pathToFileURL(basePath);
+				await syncData(entry, baseUrl, options, entryType);
+				logger.info(`Reloaded data from ${green(entry)}`);
+			}
+			watcher.on('change', onChange);
+
+			watcher.on('add', onChange);
+
+			watcher.on('unlink', async (deletedPath) => {
+				const entry = relative(basePath, deletedPath);
+				if (!matchesGlob(entry)) {
+					return;
+				}
+				const id = fileToIdMap.get(deletedPath);
+				if (id) {
+					options.store.delete(id);
+					fileToIdMap.delete(deletedPath);
+				}
+			});
+		},
+	};
+}

packages/astro/src/content/loaders/index.ts

@@ -0,0 +1,3 @@
+export { file } from './file.js';
+export { glob } from './glob.js';
+export * from './types.js';