feat: updated loaders

packages/astro/src/content/consts.ts

@@ -42,3 +42,5 @@ export const COLLECTIONS_DIR = 'collections/';
 
 export const CONTENT_LAYER_TYPE = 'content_layer';
 export const LIVE_CONTENT_TYPE = 'live';
+
+export const REFERENCE_SYMBOL = Symbol.for('astro.content.reference');

packages/astro/src/content/content-layer.ts

@@ -1,5 +1,6 @@
 import { existsSync, promises as fs } from 'node:fs';
 import { createMarkdownProcessor, type MarkdownProcessor } from '@astrojs/markdown-remark';
+import { Traverse } from 'neotraverse/modern';
 import PQueue from 'p-queue';
 import type { FSWatcher } from 'vite';
 import xxhash from 'xxhash-wasm';
@@ -17,12 +18,13 @@ import {
 } from './consts.js';
 import type { RenderedContent } from './data-store.js';
 import type { LoaderContext } from './loaders/types.js';
-import type { MutableDataStore } from './mutable-data-store.js';
+import { InMemoryDataStore, type MutableDataStore } from './mutable-data-store.js';
 import {
 	type ContentObservable,
 	getEntryConfigByExtMap,
-	getEntryDataAndImages,
+	getEntryData,
 	globalContentConfigObserver,
+	isReference,
 	loaderReturnSchema,
 	safeStringify,
 } from './utils.js';
@@ -114,21 +116,19 @@ class ContentLayer {
 	async #getLoaderContext({
 		collectionName,
 		loaderName = 'content',
-		parseData,
 		refreshContextData,
 	}: {
 		collectionName: string;
 		loaderName: string;
-		parseData: LoaderContext['parseData'];
 		refreshContextData?: Record<string, unknown>;
-	}): Promise<LoaderContext> {
+	}): Promise<Omit<LoaderContext, 'store'>> {
 		return {
 			collection: collectionName,
-			store: this.#store.scopedStore(collectionName),
 			meta: this.#store.metaStore(collectionName),
 			logger: this.#logger.forkIntegrationLogger(loaderName),
 			config: this.#settings.config,
-			parseData,
+			// TODO: remove in v7
+			parseData: async (props) => props.data,
 			renderMarkdown: this.#processMarkdown.bind(this),
 			generateDigest: await this.#getGenerateDigest(),
 			watcher: this.#watcher,
@@ -243,7 +243,7 @@ class ContentLayer {
 			this.#watcher?.removeAllTrackedListeners();
 		}
 
-		await Promise.all(
+		const rawLoaderResults = await Promise.all(
 			Object.entries(contentConfig.config.collections).map(async ([name, collection]) => {
 				if (collection.type !== CONTENT_LAYER_TYPE) {
 					return;
@@ -267,44 +267,121 @@ class ContentLayer {
 					return;
 				}
 
-				const collectionWithResolvedSchema = { ...collection, schema };
-
-				const parseData: LoaderContext['parseData'] = async ({ id, data, filePath = '' }) => {
-					const { data: parsedData } = await getEntryDataAndImages(
-						{
-							id,
-							collection: name,
-							unvalidatedData: data,
-							_internal: {
-								rawData: undefined,
-								filePath,
-							},
-						},
-						collectionWithResolvedSchema,
-						false,
-					);
-
-					return parsedData;
-				};
-
 				const context = await this.#getLoaderContext({
 					collectionName: name,
-					parseData,
 					loaderName: collection.loader.name,
 					refreshContextData: options?.context,
 				});
 
+				const realStore = this.#store.scopedStore(name);
+
 				if (typeof collection.loader === 'function') {
-					return simpleLoader(collection.loader as CollectionLoader<{ id: string }>, context);
+					await simpleLoader(collection.loader as CollectionLoader<{ id: string }>, {
+						...context,
+						store: realStore,
+					});
+					return;
 				}
 
 				if (!collection.loader.load) {
 					throw new Error(`Collection loader for ${name} does not have a load method`);
 				}
 
-				return collection.loader.load(context);
+				const memoryStore = new InMemoryDataStore(new Map(realStore.entries()));
+
+				const result = collection.loader.load({
+					...context,
+					store: memoryStore,
+				});
+
+				if (!schema && result?.schema) {
+					schema = result.schema;
+				}
+
+				return {
+					schema,
+					types: result?.types,
+					realStore,
+					memoryStore,
+					name,
+					collection,
+				};
 			}),
 		);
+
+		const filteredRawLoaderResults = rawLoaderResults.filter((result) => !!result);
+
+		await Promise.all(
+			filteredRawLoaderResults.map(async ({ memoryStore, realStore, schema, name, collection }) => {
+				for (const event of memoryStore.events) {
+					switch (event.type) {
+						case 'set': {
+							let data = event.entry.data;
+							if (schema) {
+								data = await getEntryData(
+									{
+										id: event.entry.id,
+										collection: name,
+										unvalidatedData: data,
+										_internal: {
+											rawData: undefined,
+											filePath: event.entry.filePath!,
+										},
+									},
+									{
+										...collection,
+										schema,
+									},
+									false,
+								);
+
+								new Traverse(data).forEach((ctx, value) => {
+									if (!isReference(value)) {
+										return;
+									}
+									// TODO: better data structure
+									const collectionStore = filteredRawLoaderResults.find(
+										(e) => e.name === value.collection,
+									)!.memoryStore;
+									const id = 'id' in value ? value.id : value.slug;
+									if (!collectionStore.has(id)) {
+										// TODO: AstroError
+										throw new Error(
+											`Invalid reference for ${name}.${ctx.keys!.map((key) => key.toString()).join('.')}: cannot find entry ${id}`,
+										);
+									}
+								});
+							}
+							realStore.set({ ...event.entry, data });
+							break;
+						}
+						case 'delete': {
+							realStore.delete(event.key);
+							break;
+						}
+						case 'clear': {
+							realStore.clear();
+							break;
+						}
+						case 'addAssetImport': {
+							realStore.addAssetImport(event.assetImport, event.filePath);
+							break;
+						}
+						case 'addAssetImports': {
+							realStore.addAssetImports(event.assets, event.filePath);
+							break;
+						}
+						case 'addModuleImport': {
+							realStore.addModuleImport(event.filePath);
+							break;
+						}
+					}
+				}
+
+				// TODO: handle types
+			}),
+		);
+
 		await fs.mkdir(this.#settings.config.cacheDir, { recursive: true });
 		await fs.mkdir(this.#settings.dotAstroDir, { recursive: true });
 		const assetImportsFile = new URL(ASSET_IMPORTS_FILE, this.#settings.dotAstroDir);
@@ -394,8 +471,7 @@ async function simpleLoader<TData extends { id: string }>(
 					),
 				});
 			}
-			const item = await context.parseData({ id: raw.id, data: raw });
-			context.store.set({ id: raw.id, data: item });
+			context.store.set({ id: raw.id, data: raw });
 		}
 		return;
 	}

packages/astro/src/content/loaders/types.ts

@@ -31,7 +31,10 @@ export interface LoaderContext {
 	logger: AstroIntegrationLogger;
 	/** Astro config, with user config and merged defaults */
 	config: AstroConfig;
-	/** Validates and parses the data according to the collection schema */
+	/**
+	 * Validates and parses the data according to the collection schema
+	 * @deprecated Pass raw, unvalidated data to store.set({ data }) instead. Validation will occur after load()
+	 * */
 	parseData<TData extends Record<string, unknown>>(props: ParseDataOptions<TData>): Promise<TData>;
 
 	/** Renders markdown content to HTML and metadata */
@@ -49,14 +52,22 @@ export interface LoaderContext {
 	entryTypes: Map<string, ContentEntryType>;
 }
 
-export interface Loader {
+export type Loader = {
 	/** Unique name of the loader, e.g. the npm package name */
 	name: string;
-	/** Do the actual loading of the data */
-	load: (context: LoaderContext) => Promise<void>;
-	/** Optionally, define the schema of the data. Will be overridden by user-defined schema */
-	schema?: ZodSchema | Promise<ZodSchema> | (() => ZodSchema | Promise<ZodSchema>);
-}
+} & (
+	| {
+			/** Do the actual loading of the data */
+			load: (context: LoaderContext) => Promise<void>;
+			/** Optionally, define the schema of the data. Will be overridden by user-defined schema */
+			schema?: ZodSchema;
+	  }
+	| {
+			/** Do the actual loading of the data */
+			load: (context: LoaderContext) => Promise<{ schema?: ZodSchema; types?: string } | void>;
+			schema?: never;
+	  }
+);
 
 export interface LoadEntryContext<TEntryFilter = never> {
 	filter: TEntryFilter extends never ? { id: string } : TEntryFilter;