Merge pull request #60 from 11ty/remove-flat-cache

Fixes #59 and #56

Author: zachleat

package.json

@@ -41,13 +41,14 @@
 	},
 	"homepage": "https://github.com/11ty/eleventy-fetch#readme",
 	"devDependencies": {
-		"ava": "^6.1.3",
-		"prettier": "^3.3.3"
+		"ava": "^6.2.0",
+		"prettier": "^3.5.3"
 	},
 	"dependencies": {
+		"@11ty/eleventy-utils": "^2.0.1",
 		"@rgrove/parse-xml": "^4.2.0",
-		"debug": "^4.3.7",
-		"flat-cache": "^6.1.1",
+		"debug": "^4.4.0",
+		"flatted": "^3.3.3",
 		"graceful-fs": "^4.2.11",
 		"p-queue": "6.6.2"
 	},

src/AssetCache.js

@@ -1,14 +1,13 @@
 const fs = require("graceful-fs");
 const path = require("path");
-const { create: FlatCacheCreate } = require("flat-cache");
 const { createHash } = require("crypto");
-const debugUtil = require("debug");
+const { DateCompare } = require("@11ty/eleventy-utils");
 
+const FileCache = require("./FileCache.js");
 const Sources = require("./Sources.js");
-const DirectoryManager = require("./DirectoryManager.js");
 
+const debugUtil = require("debug");
 const debug = debugUtil("Eleventy:Fetch");
-const debugAssets = debugUtil("Eleventy:Assets");
 
 class AssetCache {
 	#source;
@@ -18,7 +17,6 @@ class AssetCache {
 	#cacheDirectory;
 	#cacheLocationDirty = false;
 	#directoryManager;
-	#rawContents = {}
 
 	constructor(source, cacheDirectory, options = {}) {
 		if(!Sources.isValidSource(source)) {
@@ -173,10 +171,12 @@ class AssetCache {
 
 	get cache() {
 		if (!this.#cache || this.#cacheLocationDirty) {
-			let cache = FlatCacheCreate({
-				cacheId: this.cacheFilename,
-				cacheDir: this.rootDir,
+			let cache = new FileCache(this.cacheFilename, {
+				dir: this.rootDir,
+				source: this.source,
 			});
+			cache.setDryRun(this.options.dryRun);
+			cache.setDirectoryManager(this.#directoryManager);
 
 			this.#cache = cache;
 			this.#cacheLocationDirty = false;
@@ -185,181 +185,58 @@ class AssetCache {
 	}
 
 	getDurationMs(duration = "0s") {
-		let durationUnits = duration.slice(-1);
-		let durationMultiplier;
-		if (durationUnits === "s") {
-			durationMultiplier = 1;
-		} else if (durationUnits === "m") {
-			durationMultiplier = 60;
-		} else if (durationUnits === "h") {
-			durationMultiplier = 60 * 60;
-		} else if (durationUnits === "d") {
-			durationMultiplier = 60 * 60 * 24;
-		} else if (durationUnits === "w") {
-			durationMultiplier = 60 * 60 * 24 * 7;
-		} else if (durationUnits === "y") {
-			durationMultiplier = 60 * 60 * 24 * 365;
-		}
-
-		let durationValue = parseInt(duration.slice(0, duration.length - 1), 10);
-		return durationValue * durationMultiplier * 1000;
-	}
-
-	getCachedContentsPath(type = "buffer") {
-		if(type === "xml") {
-			type = "text";
-		} else if(type === "parsed-xml") {
-			type = "json";
-		}
-
-		return `${this.cachePath}.${type}`;
+		return DateCompare.getDurationMs(duration);
 	}
 
 	setDirectoryManager(manager) {
 		this.#directoryManager = manager;
 	}
 
-	ensureDir() {
-		if (this.options.dryRun) {
-			return;
-		}
-
-		if(!this.#directoryManager) {
-			// standalone fallback (for tests)
-			this.#directoryManager = new DirectoryManager();
-		}
-
-		this.#directoryManager.create(this.cacheDirectory);
-	}
-
 	async save(contents, type = "buffer", metadata = {}) {
 		if(!contents) {
 			throw new Error("save(contents) expects contents (was falsy)");
 		}
 
-		this.cache.set(this.hash, {
-			cachedAt: Date.now(),
-			type: type,
-			metadata,
-		});
-
-		let contentPath = this.getCachedContentsPath(type);
-
-		if (type === "json" || type === "parsed-xml") {
-			contents = JSON.stringify(contents);
-		}
-
-		this.#rawContents[type] = contents;
-
-		if(this.options.dryRun) {
-			debug(`Dry run writing ${contentPath}`);
-			return;
-		}
-
-		this.ensureDir();
-
-		debugAssets("[11ty/eleventy-fetch] Writing %o from %o", contentPath, this.source);
-
-		// the contents must exist before the cache metadata are saved below
-		fs.writeFileSync(contentPath, contents);
-		debug(`Writing ${contentPath}`);
+		this.cache.set(type, contents, metadata);
 
+		// Dry-run handled downstream
 		this.cache.save();
 	}
 
-	async #getCachedContents(type) {
-		let contentPath = this.getCachedContentsPath(type);
-
-		debug(`Fetching from cache ${contentPath}`);
-
-		if(this.source) {
-			debugAssets("[11ty/eleventy-fetch] Reading via %o", this.source);
-		} else {
-			debugAssets("[11ty/eleventy-fetch] Reading %o", contentPath);
-		}
-
-		if (type === "json" || type === "parsed-xml") {
-			return require(contentPath);
-		}
-
-		return fs.readFileSync(contentPath, type !== "buffer" ? "utf8" : null);
-	}
-
-	async getCachedContents(type) {
-		if(!this.#rawContents[type]) {
-			this.#rawContents[type] = this.#getCachedContents(type);
-		}
-
-		// already saved on this instance in-memory
-		return this.#rawContents[type];
-	}
-
-	_backwardsCompatibilityGetCachedValue(type) {
-		if (type === "json") {
-			return this.cachedObject.contents;
-		} else if (type === "text") {
-			return this.cachedObject.contents.toString();
-		}
-
-		// buffer
-		return Buffer.from(this.cachedObject.contents);
+	getCachedContents() {
+		return this.cache.getContents();
 	}
 
-	async getCachedValue() {
-		let type = this.cachedObject.type;
-
-		// backwards compat with old caches
-		if (this.cachedObject.contents) {
-			return this._backwardsCompatibilityGetCachedValue(type);
-		}
-
+	getCachedValue() {
 		if(this.options.returnType === "response") {
 			return {
 				...this.cachedObject.metadata?.response,
-				body: await this.getCachedContents(type),
+				body: this.getCachedContents(),
 				cache: "hit",
 			}
 		}
 
-		// promise
-		return this.getCachedContents(type);
+		return this.getCachedContents();
 	}
 
 	getCachedTimestamp() {
 		return this.cachedObject?.cachedAt;
 	}
 
 	isCacheValid(duration = this.duration) {
-		if (!this.cachedObject) {
-			// not cached
+		if(!this.cachedObject || !this.cachedObject?.cachedAt) {
 			return false;
 		}
 
-		// in the cache and no duration
-		if (!duration || duration === "*") {
-			// no duration specified (plugin default is 1d, but if this is falsy assume infinite)
-			// "*" is infinite duration
-			return true;
-		}
-
-		debug("Cache check for: %o %o (duration: %o)", this.hash, this.source, duration);
-		debug("Cache object: %o", this.cachedObject);
-
-		let compareDuration = this.getDurationMs(duration);
-		let expiration = this.cachedObject.cachedAt + compareDuration;
-		let expirationRelative = Math.abs(Date.now() - expiration);
-
-		if (expiration > Date.now()) {
-			debug("Cache okay, expires in %o s (%o)", expirationRelative / 1000, new Date(expiration));
-			return true;
+		if(DateCompare.isTimestampWithinDuration(this.cachedObject?.cachedAt, duration)) {
+			return this.cache.hasContents(); // check file system to make files haven’t been purged.
 		}
 
-		debug("Cache expired %o s ago (%o)", expirationRelative / 1000, new Date(expiration));
 		return false;
 	}
 
 	get cachedObject() {
-		return this.cache.get(this.hash);
+		return this.cache.get();
 	}
 
 	// Deprecated
@@ -382,17 +259,18 @@ class AssetCache {
 	}
 
 	// for testing
-	hasCacheFiles() {
-		return fs.existsSync(this.cachePath) || fs.existsSync(this.getCachedContentsPath());
+	hasAnyCacheFiles() {
+		for(let p of this.cache.getFilePaths()) {
+			if(fs.existsSync(p)) {
+				return true;
+			}
+		}
+		return false;
 	}
 
 	// for testing
 	async destroy() {
-		let paths = [];
-		paths.push(this.cachePath);
-		paths.push(this.getCachedContentsPath("json"));
-		paths.push(this.getCachedContentsPath("text"));
-		paths.push(this.getCachedContentsPath("buffer"));
+		let paths = this.cache.getFilePaths();
 
 		await Promise.all(paths.map(path => {
 			if (fs.existsSync(path)) {

src/DirectoryManager.js

@@ -1,7 +1,5 @@
 const fs = require("node:fs");
-const path = require("node:path");
-const debugUtil = require("debug");
-const debugAssets = debugUtil("Eleventy:Assets");
+const debugAssets = require("debug")("Eleventy:Assets");
 
 class DirectoryManager {
   #dirs = new Set();
@@ -16,7 +14,7 @@ class DirectoryManager {
     }
 
     this.#dirs.add(dir);
-    debugAssets("[11ty/eleventy-fetch] Creating directory %o", dir);
+    debugAssets("Creating directory %o", dir);
     fs.mkdirSync(dir, { recursive: true });
   }
 }

src/ExistsCache.js

@@ -0,0 +1,24 @@
+const fs = require("node:fs");
+// const debug = require("debug")("Eleventy:Assets");
+
+class ExistsCache {
+	#checks = new Map();
+	#count = 0;
+
+	set(target, value) {
+		this.#checks.set(target, Boolean(value));
+	}
+
+	exists(target) {
+		if(this.#checks.has(target)) {
+			return this.#checks.get(target);
+		}
+
+		let exists = fs.existsSync(target);
+		this.#count++;
+		this.#checks.set(target, exists);
+		return exists;
+	}
+}
+
+module.exports = ExistsCache;