feat: updated loaders

packages/astro/src/content/consts.ts

@@ -42,3 +42,5 @@ export const COLLECTIONS_DIR = 'collections/';
 
 export const CONTENT_LAYER_TYPE = 'content_layer';
 export const LIVE_CONTENT_TYPE = 'live';
+
+export const REFERENCE_SYMBOL = Symbol.for('astro.content.reference');

packages/astro/src/content/content-layer.ts

@@ -1,5 +1,6 @@
 import { existsSync, promises as fs } from 'node:fs';
 import { createMarkdownProcessor, type MarkdownProcessor } from '@astrojs/markdown-remark';
+import { Traverse } from 'neotraverse/modern';
 import PQueue from 'p-queue';
 import type { FSWatcher } from 'vite';
 import xxhash from 'xxhash-wasm';
@@ -16,13 +17,15 @@ import {
 	MODULES_IMPORTS_FILE,
 } from './consts.js';
 import type { RenderedContent } from './data-store.js';
+import { InMemoryDataStore } from './in-memory-data-store.js';
 import type { LoaderContext } from './loaders/types.js';
 import type { MutableDataStore } from './mutable-data-store.js';
 import {
 	type ContentObservable,
 	getEntryConfigByExtMap,
-	getEntryDataAndImages,
+	getEntryData,
 	globalContentConfigObserver,
+	isReference,
 	loaderReturnSchema,
 	safeStringify,
 } from './utils.js';
@@ -114,21 +117,19 @@ class ContentLayer {
 	async #getLoaderContext({
 		collectionName,
 		loaderName = 'content',
-		parseData,
 		refreshContextData,
 	}: {
 		collectionName: string;
 		loaderName: string;
-		parseData: LoaderContext['parseData'];
 		refreshContextData?: Record<string, unknown>;
-	}): Promise<LoaderContext> {
+	}): Promise<Omit<LoaderContext, 'store'>> {
 		return {
 			collection: collectionName,
-			store: this.#store.scopedStore(collectionName),
 			meta: this.#store.metaStore(collectionName),
 			logger: this.#logger.forkIntegrationLogger(loaderName),
 			config: this.#settings.config,
-			parseData,
+			// TODO: remove in v7
+			parseData: async (props) => props.data,
 			renderMarkdown: this.#processMarkdown.bind(this),
 			generateDigest: await this.#getGenerateDigest(),
 			watcher: this.#watcher,
@@ -243,7 +244,7 @@ class ContentLayer {
 			this.#watcher?.removeAllTrackedListeners();
 		}
 
-		await Promise.all(
+		const rawLoaderResults = await Promise.all(
 			Object.entries(contentConfig.config.collections).map(async ([name, collection]) => {
 				if (collection.type !== CONTENT_LAYER_TYPE) {
 					return;
@@ -267,44 +268,127 @@ class ContentLayer {
 					return;
 				}
 
-				const collectionWithResolvedSchema = { ...collection, schema };
-
-				const parseData: LoaderContext['parseData'] = async ({ id, data, filePath = '' }) => {
-					const { data: parsedData } = await getEntryDataAndImages(
-						{
-							id,
-							collection: name,
-							unvalidatedData: data,
-							_internal: {
-								rawData: undefined,
-								filePath,
-							},
-						},
-						collectionWithResolvedSchema,
-						false,
-					);
-
-					return parsedData;
-				};
-
 				const context = await this.#getLoaderContext({
 					collectionName: name,
-					parseData,
 					loaderName: collection.loader.name,
 					refreshContextData: options?.context,
 				});
 
+				const realStore = this.#store.scopedStore(name);
+
 				if (typeof collection.loader === 'function') {
-					return simpleLoader(collection.loader as CollectionLoader<{ id: string }>, context);
+					await simpleLoader(collection.loader as CollectionLoader<{ id: string }>, {
+						...context,
+						store: realStore,
+					});
+					return;
 				}
 
 				if (!collection.loader.load) {
 					throw new Error(`Collection loader for ${name} does not have a load method`);
 				}
 
-				return collection.loader.load(context);
+				const memoryStore = new InMemoryDataStore(new Map(realStore.entries()));
+
+				const result = collection.loader.load({
+					...context,
+					store: memoryStore,
+				});
+
+				if (!schema && result?.schema) {
+					schema = result.schema;
+				}
+
+				return [
+					name,
+					{
+						schema,
+						types: result?.types,
+						realStore,
+						memoryStore,
+						collection,
+					},
+				] as const;
 			}),
 		);
+
+		const filteredRawLoaderResults = Object.fromEntries(
+			rawLoaderResults.filter((result) => !!result),
+		);
+
+		await Promise.all(
+			Object.entries(filteredRawLoaderResults).map(
+				async ([name, { memoryStore, realStore, schema, collection }]) => {
+					for (const event of memoryStore.events) {
+						switch (event.type) {
+							case 'set': {
+								let data = event.entry.data;
+								if (schema) {
+									data = await getEntryData(
+										{
+											id: event.entry.id,
+											collection: name,
+											unvalidatedData: data,
+											_internal: {
+												rawData: undefined,
+												filePath: event.entry.filePath!,
+											},
+										},
+										{
+											...collection,
+											schema,
+										},
+										false,
+									);
+
+									new Traverse(data).forEach((ctx, value) => {
+										if (!isReference(value)) {
+											return;
+										}
+										const loaderResult = filteredRawLoaderResults[value.collection];
+										if (!loaderResult) {
+											// TODO: throw error
+										}
+										const id = 'id' in value ? value.id : value.slug;
+										if (!loaderResult.memoryStore.has(id)) {
+											// TODO: AstroError
+											throw new Error(
+												`Invalid reference for ${name}.${ctx.keys!.map((key) => key.toString()).join('.')}: cannot find entry ${id}`,
+											);
+										}
+									});
+								}
+								realStore.set({ ...event.entry, data });
+								break;
+							}
+							case 'delete': {
+								realStore.delete(event.key);
+								break;
+							}
+							case 'clear': {
+								realStore.clear();
+								break;
+							}
+							case 'addAssetImport': {
+								realStore.addAssetImport(event.assetImport, event.filePath);
+								break;
+							}
+							case 'addAssetImports': {
+								realStore.addAssetImports(event.assets, event.filePath);
+								break;
+							}
+							case 'addModuleImport': {
+								realStore.addModuleImport(event.filePath);
+								break;
+							}
+						}
+					}
+
+					// TODO: handle types
+				},
+			),
+		);
+
 		await fs.mkdir(this.#settings.config.cacheDir, { recursive: true });
 		await fs.mkdir(this.#settings.dotAstroDir, { recursive: true });
 		const assetImportsFile = new URL(ASSET_IMPORTS_FILE, this.#settings.dotAstroDir);
@@ -394,8 +478,7 @@ async function simpleLoader<TData extends { id: string }>(
 					),
 				});
 			}
-			const item = await context.parseData({ id: raw.id, data: raw });
-			context.store.set({ id: raw.id, data: item });
+			context.store.set({ id: raw.id, data: raw });
 		}
 		return;
 	}

packages/astro/src/content/in-memory-data-store.ts

@@ -0,0 +1,167 @@
+import { Traverse } from 'neotraverse/modern';
+import { IMAGE_IMPORT_PREFIX } from './consts.js';
+import type { DataEntry } from './data-store.js';
+import type { DataStore } from './mutable-data-store.js';
+
+export class InMemoryDataStore implements DataStore {
+	#collection: Map<string, any>;
+	#events: Array<
+		| {
+				type: 'set';
+				entry: DataEntry;
+		  }
+		| {
+				type: 'delete';
+				key: string;
+		  }
+		| {
+				type: 'clear';
+		  }
+		| {
+				type: 'addAssetImport';
+				assetImport: string;
+				filePath: string;
+		  }
+		| {
+				type: 'addAssetImports';
+				assets: Array<string>;
+				filePath: string;
+		  }
+		| {
+				type: 'addModuleImport';
+				filePath: string;
+		  }
+	> = [];
+
+	constructor(collection: Map<string, any>) {
+		this.#collection = collection;
+	}
+
+	get events() {
+		return this.#events;
+	}
+
+	get<TData extends Record<string, unknown> = Record<string, unknown>>(
+		key: string,
+	): DataEntry<TData> {
+		return this.#collection.get(key);
+	}
+
+	entries() {
+		return [...this.#collection.entries()];
+	}
+
+	values() {
+		return [...this.#collection.values()];
+	}
+
+	keys() {
+		return [...this.#collection.keys()];
+	}
+
+	set<TData extends Record<string, unknown>>(inputEntry: DataEntry<TData>) {
+		this.#events.push({
+			type: 'set',
+			entry: inputEntry,
+		});
+		const {
+			id: key,
+			data,
+			body,
+			filePath,
+			deferredRender,
+			digest,
+			rendered,
+			assetImports,
+		} = inputEntry;
+		if (!key) {
+			throw new Error(`ID must be a non-empty string`);
+		}
+		const id = String(key);
+		if (digest) {
+			const existing = this.#collection.get(id) as DataEntry | undefined;
+			if (existing && existing.digest === digest) {
+				return false;
+			}
+		}
+		const foundAssets = new Set<string>(assetImports);
+		// Check for image imports in the data. These will have been prefixed during schema parsing
+		new Traverse(data).forEach((_, val) => {
+			if (typeof val === 'string' && val.startsWith(IMAGE_IMPORT_PREFIX)) {
+				const src = val.replace(IMAGE_IMPORT_PREFIX, '');
+				foundAssets.add(src);
+			}
+		});
+
+		const entry: DataEntry = {
+			id,
+			data,
+		};
+		// We do it like this so we don't waste space stringifying
+		// the fields if they are not set
+		if (body) {
+			entry.body = body;
+		}
+		if (filePath) {
+			if (filePath.startsWith('/')) {
+				throw new Error(`File path must be relative to the site root. Got: ${filePath}`);
+			}
+			entry.filePath = filePath;
+		}
+
+		if (foundAssets.size) {
+			entry.assetImports = Array.from(foundAssets);
+		}
+
+		if (digest) {
+			entry.digest = digest;
+		}
+		if (rendered) {
+			entry.rendered = rendered;
+		}
+		if (deferredRender) {
+			entry.deferredRender = deferredRender;
+		}
+		this.#collection.set(id, entry);
+		return true;
+	}
+
+	delete(key: string) {
+		this.#events.push({
+			type: 'delete',
+			key,
+		});
+		return this.#collection.delete(key);
+	}
+
+	clear() {
+		this.#collection = new Map();
+	}
+
+	has(key: string) {
+		return this.#collection.has(key);
+	}
+
+	addAssetImport(assetImport: string, filePath: string) {
+		this.#events.push({
+			type: 'addAssetImport',
+			assetImport,
+			filePath,
+		});
+	}
+
+	addAssetImports(assets: Array<string>, filePath: string) {
+		this.#events.push({
+			type: 'addAssetImports',
+			assets,
+			filePath,
+		});
+	}
+
+	addModuleImport(filePath: string) {
+		this.#events.push({
+			type: 'addModuleImport',
+			filePath,
+		});
+	}
+}