feat: updated loaders

packages/astro/src/content/content-layer.ts

@@ -17,7 +17,7 @@ import {
 } from './consts.js';
 import type { RenderedContent } from './data-store.js';
 import type { LoaderContext } from './loaders/types.js';
-import type { MutableDataStore } from './mutable-data-store.js';
+import { InMemoryDataStore, type MutableDataStore } from './mutable-data-store.js';
 import {
 	type ContentObservable,
 	getEntryConfigByExtMap,
@@ -114,21 +114,19 @@ class ContentLayer {
 	async #getLoaderContext({
 		collectionName,
 		loaderName = 'content',
-		parseData,
 		refreshContextData,
 	}: {
 		collectionName: string;
 		loaderName: string;
-		parseData: LoaderContext['parseData'];
 		refreshContextData?: Record<string, unknown>;
-	}): Promise<LoaderContext> {
+	}): Promise<Omit<LoaderContext, 'store'>> {
 		return {
 			collection: collectionName,
-			store: this.#store.scopedStore(collectionName),
 			meta: this.#store.metaStore(collectionName),
 			logger: this.#logger.forkIntegrationLogger(loaderName),
 			config: this.#settings.config,
-			parseData,
+			// TODO: remove in v7
+			parseData: async (props) => props.data,
 			renderMarkdown: this.#processMarkdown.bind(this),
 			generateDigest: await this.#getGenerateDigest(),
 			watcher: this.#watcher,
@@ -269,40 +267,84 @@ class ContentLayer {
 
 				const collectionWithResolvedSchema = { ...collection, schema };
 
-				const parseData: LoaderContext['parseData'] = async ({ id, data, filePath = '' }) => {
-					const { data: parsedData } = await getEntryDataAndImages(
-						{
-							id,
-							collection: name,
-							unvalidatedData: data,
-							_internal: {
-								rawData: undefined,
-								filePath,
-							},
-						},
-						collectionWithResolvedSchema,
-						false,
-					);
-
-					return parsedData;
-				};
-
 				const context = await this.#getLoaderContext({
 					collectionName: name,
-					parseData,
 					loaderName: collection.loader.name,
 					refreshContextData: options?.context,
 				});
 
+				const realStore = this.#store.scopedStore(name);
+
 				if (typeof collection.loader === 'function') {
-					return simpleLoader(collection.loader as CollectionLoader<{ id: string }>, context);
+					return simpleLoader(collection.loader as CollectionLoader<{ id: string }>, {
+						...context,
+						store: realStore,
+					});
 				}
 
 				if (!collection.loader.load) {
 					throw new Error(`Collection loader for ${name} does not have a load method`);
 				}
 
-				return collection.loader.load(context);
+				const fakeStore = new InMemoryDataStore();
+
+				const result = collection.loader.load({
+					...context,
+					store: fakeStore,
+				});
+
+				// TODO: handle types
+
+				if (!schema && result?.schema) {
+					schema = result.schema;
+				}
+
+				for (const event of fakeStore.events) {
+					switch (event.type) {
+						case 'set': {
+							let data = event.entry.data;
+							if (schema) {
+								({ data } = await getEntryDataAndImages(
+									{
+										id: event.entry.id,
+										collection: name,
+										unvalidatedData: data,
+										_internal: {
+											rawData: undefined,
+											filePath: event.entry.filePath!,
+										},
+									},
+									collectionWithResolvedSchema,
+									false,
+								));
+							}
+							realStore.set({ ...event.entry, data });
+							break;
+						}
+						case 'delete': {
+							realStore.delete(event.key);
+							break;
+						}
+						case 'clear': {
+							realStore.clear();
+							break;
+						}
+						case 'addAssetImport': {
+							realStore.addAssetImport(event.assetImport, event.filePath);
+							break;
+						}
+						case 'addAssetImports': {
+							realStore.addAssetImports(event.assets, event.filePath);
+							break;
+						}
+						case 'addModuleImport': {
+							realStore.addModuleImport(event.filePath);
+							break;
+						}
+					}
+				}
+
+				return result;
 			}),
 		);
 		await fs.mkdir(this.#settings.config.cacheDir, { recursive: true });
@@ -394,8 +436,7 @@ async function simpleLoader<TData extends { id: string }>(
 					),
 				});
 			}
-			const item = await context.parseData({ id: raw.id, data: raw });
-			context.store.set({ id: raw.id, data: item });
+			context.store.set({ id: raw.id, data: raw });
 		}
 		return;
 	}

packages/astro/src/content/loaders/types.ts

@@ -31,7 +31,10 @@ export interface LoaderContext {
 	logger: AstroIntegrationLogger;
 	/** Astro config, with user config and merged defaults */
 	config: AstroConfig;
-	/** Validates and parses the data according to the collection schema */
+	/**
+	 * Validates and parses the data according to the collection schema
+	 * @deprecated Pass raw, unvalidated data to store.set({ data }) instead. Validation will occur after load()
+	 * */
 	parseData<TData extends Record<string, unknown>>(props: ParseDataOptions<TData>): Promise<TData>;
 
 	/** Renders markdown content to HTML and metadata */
@@ -49,14 +52,22 @@ export interface LoaderContext {
 	entryTypes: Map<string, ContentEntryType>;
 }
 
-export interface Loader {
+export type Loader = {
 	/** Unique name of the loader, e.g. the npm package name */
 	name: string;
-	/** Do the actual loading of the data */
-	load: (context: LoaderContext) => Promise<void>;
-	/** Optionally, define the schema of the data. Will be overridden by user-defined schema */
-	schema?: ZodSchema | Promise<ZodSchema> | (() => ZodSchema | Promise<ZodSchema>);
-}
+} & (
+	| {
+			/** Do the actual loading of the data */
+			load: (context: LoaderContext) => Promise<void>;
+			/** Optionally, define the schema of the data. Will be overridden by user-defined schema */
+			schema?: ZodSchema;
+	  }
+	| {
+			/** Do the actual loading of the data */
+			load: (context: LoaderContext) => Promise<{ schema?: ZodSchema; types?: string }>;
+			schema?: never;
+	  }
+);
 
 export interface LoadEntryContext<TEntryFilter = never> {
 	filter: TEntryFilter extends never ? { id: string } : TEntryFilter;

packages/astro/src/content/mutable-data-store.ts

@@ -485,3 +485,162 @@ export interface MetaStore {
 	has: (key: string) => boolean;
 	delete: (key: string) => void;
 }
+
+export class InMemoryDataStore implements DataStore {
+	#collection = new Map<string, any>();
+	#events: Array<
+		| {
+				type: 'set';
+				entry: DataEntry;
+		  }
+		| {
+				type: 'delete';
+				key: string;
+		  }
+		| {
+				type: 'clear';
+		  }
+		| {
+				type: 'addAssetImport';
+				assetImport: string;
+				filePath: string;
+		  }
+		| {
+				type: 'addAssetImports';
+				assets: Array<string>;
+				filePath: string;
+		  }
+		| {
+				type: 'addModuleImport';
+				filePath: string;
+		  }
+	> = [];
+
+	get events() {
+		return this.#events;
+	}
+
+	get<TData extends Record<string, unknown> = Record<string, unknown>>(
+		key: string,
+	): DataEntry<TData> {
+		return this.#collection.get(key);
+	}
+
+	entries() {
+		return [...this.#collection.entries()];
+	}
+
+	values() {
+		return [...this.#collection.values()];
+	}
+
+	keys() {
+		return [...this.#collection.keys()];
+	}
+
+	set<TData extends Record<string, unknown>>(inputEntry: DataEntry<TData>) {
+		this.#events.push({
+			type: 'set',
+			entry: inputEntry,
+		});
+		const {
+			id: key,
+			data,
+			body,
+			filePath,
+			deferredRender,
+			digest,
+			rendered,
+			assetImports,
+		} = inputEntry;
+		if (!key) {
+			throw new Error(`ID must be a non-empty string`);
+		}
+		const id = String(key);
+		if (digest) {
+			const existing = this.#collection.get(id) as DataEntry | undefined;
+			if (existing && existing.digest === digest) {
+				return false;
+			}
+		}
+		const foundAssets = new Set<string>(assetImports);
+		// Check for image imports in the data. These will have been prefixed during schema parsing
+		new Traverse(data).forEach((_, val) => {
+			if (typeof val === 'string' && val.startsWith(IMAGE_IMPORT_PREFIX)) {
+				const src = val.replace(IMAGE_IMPORT_PREFIX, '');
+				foundAssets.add(src);
+			}
+		});
+
+		const entry: DataEntry = {
+			id,
+			data,
+		};
+		// We do it like this so we don't waste space stringifying
+		// the fields if they are not set
+		if (body) {
+			entry.body = body;
+		}
+		if (filePath) {
+			if (filePath.startsWith('/')) {
+				throw new Error(`File path must be relative to the site root. Got: ${filePath}`);
+			}
+			entry.filePath = filePath;
+		}
+
+		if (foundAssets.size) {
+			entry.assetImports = Array.from(foundAssets);
+		}
+
+		if (digest) {
+			entry.digest = digest;
+		}
+		if (rendered) {
+			entry.rendered = rendered;
+		}
+		if (deferredRender) {
+			entry.deferredRender = deferredRender;
+		}
+		this.#collection.set(id, entry);
+		return true;
+	}
+
+	delete(key: string) {
+		this.#events.push({
+			type: 'delete',
+			key,
+		});
+		return this.#collection.delete(key);
+	}
+
+	clear() {
+		this.#collection = new Map();
+	}
+
+	has(key: string) {
+		return this.#collection.has(key);
+	}
+
+	addAssetImport(assetImport: string, filePath: string) {
+		this.#events.push({
+			type: 'addAssetImport',
+			assetImport,
+			filePath,
+		});
+	}
+
+	addAssetImports(assets: Array<string>, filePath: string) {
+		this.#events.push({
+			type: 'addAssetImports',
+			assets,
+			filePath,
+		});
+	}
+
+	addModuleImport(filePath: string) {
+		this.#events.push({
+			type: 'addModuleImport',
+			filePath,
+		});
+	}
+}