From 18ca38c78497a2c2ced1a9b0e8fbc6f634690c7d Mon Sep 17 00:00:00 2001 From: Jan-Henrik Damaschke Date: Sat, 30 Dec 2023 17:03:36 +0100 Subject: [PATCH 01/10] feat(encryption): :sparkles: Implemented symmetric encryption of values Key encryption wip; Asymmetric encryption wip --- package.json | 3 +- pnpm-lock.yaml | 3 + src/_utils.ts | 252 +++++++++++++++++++++++++++++++++++ src/types.ts | 24 ++-- src/utils.ts | 56 ++++++++ test/drivers/mongodb.test.ts | 13 +- test/encryption.test.ts | 185 +++++++++++++++++++++++++ 7 files changed, 514 insertions(+), 22 deletions(-) create mode 100644 test/encryption.test.ts diff --git a/package.json b/package.json index e9ad9084c..92492246a 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,8 @@ "mri": "^1.2.0", "node-fetch-native": "^1.4.1", "ofetch": "^1.3.3", - "ufo": "^1.3.2" + "ufo": "^1.3.2", + "uncrypto": "^0.1.3" }, "devDependencies": { "@azure/app-configuration": "^1.5.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f81621eb3..434a1c1cb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -38,6 +38,9 @@ dependencies: ufo: specifier: ^1.3.2 version: 1.3.2 + uncrypto: + specifier: ^0.1.3 + version: 0.1.3 devDependencies: '@azure/app-configuration': diff --git a/src/_utils.ts b/src/_utils.ts index e7d2c543b..fb1cb4c97 100644 --- a/src/_utils.ts +++ b/src/_utils.ts @@ -1,6 +1,25 @@ +import { subtle, getRandomValues } from "uncrypto"; + type Awaited = T extends Promise ? Awaited : T; type Promisified = Promise>; +type EncryptStorageOptions = Omit & { + key: string; + encryptKeys?: boolean; +}; + +type DecryptStorageOptions = Omit & { + key: string; + decryptKeys?: boolean; +}; + +interface EncryptionOptions { + key: CryptoKey; + algorithm: "AES-GCM" | "AES-CBC" | "RSA-OAEP"; + iv?: Uint8Array; + raw?: boolean; +} + export function wrapToPromise(value: T) { if (!value || typeof (value as any).then !== "function") { return Promise.resolve(value) as Promisified; @@ -75,3 +94,236 @@ export function deserializeRaw(value: any) { checkBufferSupport(); return Buffer.from(value.slice(BASE64_PREFIX.length), "base64"); } + +// Encryption + +export interface StorageValueEnvelope { + iv: string; + encryptedValue: string; +} + +export async function generateAesKey(algorithm: "AES-GCM" | "AES-CBC" = "AES-GCM") { + return genBase64FromBytes( + new Uint8Array( + await subtle.exportKey( + 'raw', await subtle.generateKey( + { + name: algorithm, + length: 256, + }, + true, + ['encrypt', 'decrypt']) + ) + ) + ); +} + +export async function generateRsaKeyPair() { + const keyPair = await subtle.generateKey( + { + name: "RSA-OAEP", + modulusLength: 4096, + publicExponent: new Uint8Array([1, 0, 1]), + hash: "SHA-256", + }, + true, + ['encrypt', 'decrypt']); + return { + privateKey: keyPair.privateKey, + publicKey: keyPair.publicKey, + }; +} + +export async function encryptStorageValue(storageValue: any, options: EncryptStorageOptions) { + const { key, algorithm, raw } = options; + const cryptoKey = await subtle.importKey('raw', genBytesFromBase64(key), { + name: "AES-GCM", + length: 256 + }, true, ['encrypt', 'decrypt']); + const iv = getRandomValues(new Uint8Array(12)); + const encryptedValue = await encryptSym(storageValue, { key: cryptoKey, algorithm, raw, iv }); + return { + encryptedValue, + iv: genBase64FromBytes(iv), + }; +} + +export async function decryptStorageValue(storageValue: StorageValueEnvelope, options: DecryptStorageOptions): Promise { + const { key, algorithm, raw } = options; + const { encryptedValue, iv } = storageValue; + const cryptoKey = await subtle.importKey('raw', genBytesFromBase64(key), { + name: "AES-GCM", + length: 256 + }, true, ['encrypt', 'decrypt']); + const decrypted = await decryptSym(encryptedValue, { key: cryptoKey, algorithm, raw, iv: genBytesFromBase64(iv) }); + return decrypted as T; +} + +export async function encryptStorageKey(storageKey: string, options: EncryptStorageOptions) { + const { key, algorithm, raw } = options; + const cryptoKey = await subtle.importKey('raw', genBytesFromBase64(key), { + name: "AES-GCM", + length: 256 + }, true, ['encrypt', 'decrypt']); + return await encryptSym(storageKey, { key: cryptoKey, algorithm, raw, iv: new Uint8Array(12) }); +} + +export async function decryptStorageKey(encryptedKey: string, options: DecryptStorageOptions) { + const { key, algorithm } = options; + const cryptoKey = await subtle.importKey('raw', genBytesFromBase64(key), { + name: "AES-GCM", + length: 256 + }, true, ['encrypt', 'decrypt']); + const decrypted = await decryptSym(encryptedKey, { key: cryptoKey, algorithm, iv: new Uint8Array(12) }); + return decrypted as string; +} + +async function encryptSym(content: any, options: EncryptionOptions) { + if (options.raw) { + return genBase64FromBytes(new Uint8Array(await subtle.encrypt( + { + name: options.algorithm, + iv: options.iv, + }, + options.key, + content + ))); + } + const encoded = new TextEncoder().encode(content); + const ciphertext = await subtle.encrypt( + { + name: options.algorithm, + iv: options.iv, + }, + options.key, + encoded + ); + return genBase64FromBytes(new Uint8Array(ciphertext)); +} + +async function encryptAsym(content: any, options: EncryptionOptions) { + if (options.raw) { + return await subtle.encrypt( + { + name: options.algorithm, + }, + options.key, + content + ); + } + const encoded = new TextEncoder().encode(content); + const ciphertext = await subtle.encrypt( + { + name: options.algorithm, + }, + options.key, + encoded + ); + return genBase64FromBytes(new Uint8Array(ciphertext)); +} + +async function decryptSym(content: any, options: EncryptionOptions) { + const decoded = genBytesFromBase64(content); + if (options.raw) { + return await subtle.decrypt( + { + name: options.algorithm, + iv: options.iv, + }, + options.key, + decoded + ); + } + const decrypted = await subtle.decrypt( + { + name: 'AES-GCM', + iv: options.iv, + }, + options.key, + decoded + ); + return new TextDecoder().decode(decrypted); +} + +async function decryptAsym(content: any, options: EncryptionOptions) { + if (options.raw) { + return await subtle.decrypt( + { + name: options.algorithm, + }, + options.key, + content + ); + } + + const decoded = genBytesFromBase64(content); + + return await subtle.decrypt( + { + name: 'AES-GCM', + iv: options.iv, + }, + options.key, + decoded + ); +} + +// Base64 utilities - Waiting for https://github.com/unjs/knitwork/pull/83 // TODO: Replace with knitwork imports as soon as PR is merged + +interface CodegenOptions { + encoding?: 'utf8' | 'ascii' | 'url'; +} + +function genBytesFromBase64(input: string) { + return Uint8Array.from( + globalThis.atob(input), + (c) => c.codePointAt(0) as number + ); +} + +function genBase64FromBytes(input: Uint8Array, urlSafe?: boolean) { + if (urlSafe) { + return globalThis + .btoa(String.fromCodePoint(...input)) + .replace(/\+/g, '-') + .replace(/\//g, '_') + .replace(/=+$/, ''); + } + return globalThis.btoa(String.fromCodePoint(...input)); +} + +function genBase64FromString( + input: string, + options: CodegenOptions = {} +) { + if (options.encoding === 'utf8') { + return genBase64FromBytes(new TextEncoder().encode(input)); + } + if (options.encoding === 'url') { + return genBase64FromBytes(new TextEncoder().encode(input)) + .replace(/\+/g, '-') + .replace(/\//g, '_') + .replace(/=+$/, ''); + } + return globalThis.btoa(input); +} + +function genStringFromBase64( + input: string, + options: CodegenOptions = {} +) { + if (options.encoding === 'utf8') { + return new TextDecoder().decode(genBytesFromBase64(input)); + } + if (options.encoding === 'url') { + input = input.replace(/-/g, '+').replace(/_/g, '/'); + const paddingLength = input.length % 4; + if (paddingLength === 2) { + input += '=='; + } else if (paddingLength === 3) { + input += '='; + } + return new TextDecoder().decode(genBytesFromBase64(input)); + } + return globalThis.atob(input); +} diff --git a/src/types.ts b/src/types.ts index e33a8da56..a391c7793 100644 --- a/src/types.ts +++ b/src/types.ts @@ -26,9 +26,9 @@ export interface Driver { ) => MaybePromise; /** @experimental */ getItems?: ( - items: { key: string; options?: TransactionOptions }[], + items: { key: string; options?: TransactionOptions; }[], commonOptions?: TransactionOptions - ) => MaybePromise<{ key: string; value: StorageValue }[]>; + ) => MaybePromise<{ key: string; value: StorageValue; }[]>; /** @experimental */ getItemRaw?: (key: string, opts: TransactionOptions) => MaybePromise; setItem?: ( @@ -38,7 +38,7 @@ export interface Driver { ) => MaybePromise; /** @experimental */ setItems?: ( - items: { key: string; value: string; options?: TransactionOptions }[], + items: { key: string; value: string; options?: TransactionOptions; }[], commonOptions?: TransactionOptions ) => MaybePromise; /** @experimental */ @@ -67,9 +67,9 @@ export interface Storage { ) => Promise; /** @experimental */ getItems: ( - items: (string | { key: string; options?: TransactionOptions })[], + items: (string | { key: string; options?: TransactionOptions; })[], commonOptions?: TransactionOptions - ) => Promise<{ key: string; value: StorageValue }[]>; + ) => Promise<{ key: string; value: StorageValue; }[]>; /** @experimental See https://github.com/unjs/unstorage/issues/142 */ getItemRaw: ( key: string, @@ -81,8 +81,8 @@ export interface Storage { opts?: TransactionOptions ) => Promise; /** @experimental */ - setItems: ( - items: { key: string; value: string; options?: TransactionOptions }[], + setItems: = StorageValue>( + items: { key: string; value: U; options?: TransactionOptions; }[], commonOptions?: TransactionOptions ) => Promise; /** @experimental See https://github.com/unjs/unstorage/issues/142 */ @@ -94,14 +94,14 @@ export interface Storage { removeItem: ( key: string, opts?: - | (TransactionOptions & { removeMeta?: boolean }) + | (TransactionOptions & { removeMeta?: boolean; }) | boolean /* legacy: removeMeta */ ) => Promise; // Meta getMeta: ( key: string, opts?: - | (TransactionOptions & { nativeOnly?: boolean }) + | (TransactionOptions & { nativeOnly?: boolean; }) | boolean /* legacy: nativeOnly */ ) => MaybePromise; setMeta: ( @@ -120,9 +120,9 @@ export interface Storage { // Mount mount: (base: string, driver: Driver) => Storage; unmount: (base: string, dispose?: boolean) => Promise; - getMount: (key?: string) => { base: string; driver: Driver }; + getMount: (key?: string) => { base: string; driver: Driver; }; getMounts: ( base?: string, - options?: { parents?: boolean } - ) => { base: string; driver: Driver }[]; + options?: { parents?: boolean; } + ) => { base: string; driver: Driver; }[]; } diff --git a/src/utils.ts b/src/utils.ts index 965346748..266262c93 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,3 +1,5 @@ +import destr from "destr"; +import { StorageValueEnvelope, decryptStorageKey, decryptStorageValue, encryptStorageKey, encryptStorageValue, stringify } from "./_utils"; import type { Storage, StorageValue } from "./types"; type StorageKeys = Array; @@ -42,6 +44,60 @@ export function prefixStorage( return nsStorage; } +export function encryptedStorage( + storage: Storage, + encryptionKey: string, + encryptKeys?: boolean, +): Storage { + const encStorage: Storage = { ...storage }; + + encStorage.setItem = async (key, value, ...args) => { + if (encryptKeys) { + key = await encryptStorageKey(key, { algorithm: "AES-GCM", key: encryptionKey }); + } + const encryptedValue = await encryptStorageValue(stringify(value), { algorithm: "AES-GCM", key: encryptionKey }); + storage.setItem(key, encryptedValue as T, ...args); + }; + + encStorage.setItemRaw = async (key, value, ...args) => { + const encryptedValue = await encryptStorageValue(value, { algorithm: "AES-GCM", key: encryptionKey, raw: true }); + storage.setItem(key, encryptedValue as T, ...args); + }; + + encStorage.getItem = async (key, ...args) => { + if (encryptKeys) { + key = await decryptStorageKey(key, { algorithm: "AES-GCM", key: encryptionKey }); + } + const value = await storage.getItem(key, ...args); + return value ? destr(await decryptStorageValue(value as StorageValueEnvelope, { algorithm: "AES-GCM", key: encryptionKey })) : null; + }; + + encStorage.getItemRaw = async (key, ...args) => { + const value = await storage.getItem(key, ...args); + return value ? await decryptStorageValue(value as StorageValueEnvelope, { algorithm: "AES-GCM", key: encryptionKey, raw: true }) : null; + }; + + encStorage.getItems = async (items, ...args) => { + const encryptedItems = await storage.getItems(items, ...args); + return await Promise.all(encryptedItems.map(async (encryptedItem) => { + const { value, ...rest } = encryptedItem; + const decryptedValue = (await decryptStorageValue(value as StorageValueEnvelope, { algorithm: "AES-GCM", key: encryptionKey })) as StorageValue; + return { value: decryptedValue, ...rest }; + })); + }; + + encStorage.setItems = async (items, ...args) => { + const encryptedItems = await Promise.all(items.map(async (item) => { + const { value, ...rest } = item; + const encryptedValue: StorageValueEnvelope = await encryptStorageValue(stringify(value), { algorithm: "AES-GCM", key: encryptionKey }); + return { value: encryptedValue, ...rest }; + })); + storage.setItems(encryptedItems, ...args); + }; + + return encStorage; +} + export function normalizeKey(key?: string) { if (!key) { return ""; diff --git a/test/drivers/mongodb.test.ts b/test/drivers/mongodb.test.ts index 26574bc43..cf58fb76a 100644 --- a/test/drivers/mongodb.test.ts +++ b/test/drivers/mongodb.test.ts @@ -3,17 +3,12 @@ import driver from "../../src/drivers/mongodb"; import { testDriver } from "./utils"; import { MongoMemoryServer } from "mongodb-memory-server"; import { promisify } from "util"; +import { beforeEach } from "node:test"; -describe.skip("drivers: mongodb", async () => { +describe("drivers: mongodb", async () => { const sleep = promisify(setTimeout); - let mongoServer: MongoMemoryServer; - let connectionString: string | undefined; - - beforeAll(async () => { - mongoServer = await MongoMemoryServer.create(); - connectionString = mongoServer.getUri(); - }); + let mongoServer = await MongoMemoryServer.create(); afterAll(async () => { if (mongoServer) { @@ -23,7 +18,7 @@ describe.skip("drivers: mongodb", async () => { testDriver({ driver: driver({ - connectionString: connectionString as string, + connectionString: mongoServer.getUri() as string, databaseName: "test", collectionName: "test", }), diff --git a/test/encryption.test.ts b/test/encryption.test.ts new file mode 100644 index 000000000..346f97fff --- /dev/null +++ b/test/encryption.test.ts @@ -0,0 +1,185 @@ +import { describe, it, expect } from "vitest"; +import { + createStorage, + encryptedStorage, + restoreSnapshot, +} from "../src"; +import driver from "../src/drivers/memory"; +import { TestContext, TestOptions } from "./drivers/utils"; + +const encryptionKey = 'e9iF+8pS8qAjnj7B1+ZwdzWQ+KXNJGUPW3HdDuMJPgI='; + +describe("encryption", () => { + it.skip("encryptedStorage", async () => { + const storage = createStorage(); + const encStorage = encryptedStorage(storage, encryptionKey); + await encStorage.setItem("s1:a", "test_data"); + await encStorage.setItem("s2:a", "test_data"); + await encStorage.setItem("s3:a?q=1", "test_data"); + expect(await encStorage.hasItem("s1:a")).toBe(true); + expect(await encStorage.getItem("s1:a")).toBe("test_data"); + expect(await encStorage.getItem("s3:a?q=2")).toBe("test_data"); + }); + + testEncryptionDriver({ + driver: driver(), + }); +}); + +export function testEncryptionDriver(opts: TestOptions) { + const ctx: TestContext = { + storage: encryptedStorage(createStorage({ driver: opts.driver }), encryptionKey, false), + driver: opts.driver, + }; + + it("init", async () => { + await restoreSnapshot(ctx.storage, { initial: "works" }); + expect(await ctx.storage.getItem("initial")).toBe("works"); + await ctx.storage.clear(); + }); + + it("initial state", async () => { + expect(await ctx.storage.hasItem("s1:a")).toBe(false); + expect(await ctx.storage.getItem("s2:a")).toBe(null); + expect(await ctx.storage.getKeys()).toMatchObject([]); + }); + + it("setItem", async () => { + await ctx.storage.setItem("s1:a", "test_data"); + await ctx.storage.setItem("s2:a", "test_data"); + await ctx.storage.setItem("s3:a?q=1", "test_data"); + expect(await ctx.storage.hasItem("s1:a")).toBe(true); + expect(await ctx.storage.getItem("s1:a")).toBe("test_data"); + expect(await ctx.storage.getItem("s3:a?q=2")).toBe("test_data"); + }); + + it("getKeys", async () => { + expect(await ctx.storage.getKeys().then((k) => k.sort())).toMatchObject( + ["s1:a", "s2:a", "s3:a"].sort() + ); + expect(await ctx.storage.getKeys("s1").then((k) => k.sort())).toMatchObject( + ["s1:a"].sort() + ); + }); + + it("serialize (object)", async () => { + await ctx.storage.setItem("/data/test.json", { json: "works" }); + expect(await ctx.storage.getItem("/data/test.json")).toMatchObject({ + json: "works", + }); + }); + + it("serialize (primitive)", async () => { + await ctx.storage.setItem("/data/true.json", true); + expect(await ctx.storage.getItem("/data/true.json")).toBe(true); + }); + + it("serialize (lossy object with toJSON())", async () => { + class Test1 { + toJSON() { + return "SERIALIZED"; + } + } + await ctx.storage.setItem("/data/serialized1.json", new Test1()); + expect(await ctx.storage.getItem("/data/serialized1.json")).toBe( + "SERIALIZED" + ); + class Test2 { + toJSON() { + return { serializedObj: "works" }; + } + } + await ctx.storage.setItem("/data/serialized2.json", new Test2()); + expect(await ctx.storage.getItem("/data/serialized2.json")).toMatchObject({ + serializedObj: "works", + }); + }); + + // eslint-disable-next-line require-await + it("serialize (error for non primitives)", async () => { + class Test {} + expect( + ctx.storage.setItem("/data/badvalue.json", new Test()) + ).rejects.toThrow("[unstorage] Cannot stringify value!"); + }); + + it("raw support", async () => { + const value = new Uint8Array([1, 2, 3]); + await ctx.storage.setItemRaw("/data/raw.bin", value); + const rValue = await ctx.storage.getItemRaw("/data/raw.bin"); + const rValueLen = rValue?.length || rValue?.byteLength; + if (rValueLen !== value.length) { + console.log("Invalid raw value length:", rValue, "Length:", rValueLen); + } + expect(rValueLen).toBe(value.length); + expect(Buffer.from(rValue).toString("base64")).toBe( + Buffer.from(value).toString("base64") + ); + }); + + // Bulk tests + it("setItems", async () => { + await ctx.storage.setItems([ + { key: "t:1", value: "test_data_t1" }, + { key: "t:2", value: "test_data_t2" }, + { key: "t:3", value: "test_data_t3" }, + ]); + expect(await ctx.storage.getItem("t:1")).toBe("test_data_t1"); + expect(await ctx.storage.getItem("t:2")).toBe("test_data_t2"); + expect(await ctx.storage.getItem("t:3")).toBe("test_data_t3"); + }); + + it("getItems", async () => { + await ctx.storage.setItem("v1:a", "test_data_v1:a"); + await ctx.storage.setItem("v2:a", "test_data_v2:a"); + await ctx.storage.setItem("v3:a?q=1", "test_data_v3:a?q=1"); + + expect( + await ctx.storage.getItems([{ key: "v1:a" }, "v2:a", { key: "v3:a?q=1" }]) + ).toMatchObject([ + { + key: "v1:a", + value: "test_data_v1:a", + }, + { + key: "v2:a", + value: "test_data_v2:a", + }, + { + key: "v3:a", // key should lose the querystring + value: "test_data_v3:a?q=1", + }, + ]); + }); + + it("getItem - return falsy values when set in storage", async () => { + await ctx.storage.setItem("zero", 0); + expect(await ctx.storage.getItem("zero")).toBe(0); + + await ctx.storage.setItem("my-false-flag", false); + expect(await ctx.storage.getItem("my-false-flag")).toBe(false); + }); + + // TODO: Refactor to move after cleanup + if (opts.additionalTests) { + opts.additionalTests(ctx); + } + + it("removeItem", async () => { + await ctx.storage.removeItem("s1:a", false); + expect(await ctx.storage.hasItem("s1:a")).toBe(false); + expect(await ctx.storage.getItem("s1:a")).toBe(null); + }); + + it("clear", async () => { + await ctx.storage.clear(); + expect(await ctx.storage.getKeys()).toMatchObject([]); + // ensure we can clear empty storage as well: #162 + await ctx.storage.clear(); + expect(await ctx.storage.getKeys()).toMatchObject([]); + }); + + it("dispose", async () => { + await ctx.storage.dispose(); + }); +} From 59829fb791b172da260231ec3efaa45266e47204 Mon Sep 17 00:00:00 2001 From: Jan-Henrik Damaschke Date: Sat, 30 Dec 2023 17:10:26 +0100 Subject: [PATCH 02/10] revert(test): :rewind: Reverted unrelated mongodb test change --- test/drivers/mongodb.test.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/test/drivers/mongodb.test.ts b/test/drivers/mongodb.test.ts index cf58fb76a..26574bc43 100644 --- a/test/drivers/mongodb.test.ts +++ b/test/drivers/mongodb.test.ts @@ -3,12 +3,17 @@ import driver from "../../src/drivers/mongodb"; import { testDriver } from "./utils"; import { MongoMemoryServer } from "mongodb-memory-server"; import { promisify } from "util"; -import { beforeEach } from "node:test"; -describe("drivers: mongodb", async () => { +describe.skip("drivers: mongodb", async () => { const sleep = promisify(setTimeout); - let mongoServer = await MongoMemoryServer.create(); + let mongoServer: MongoMemoryServer; + let connectionString: string | undefined; + + beforeAll(async () => { + mongoServer = await MongoMemoryServer.create(); + connectionString = mongoServer.getUri(); + }); afterAll(async () => { if (mongoServer) { @@ -18,7 +23,7 @@ describe("drivers: mongodb", async () => { testDriver({ driver: driver({ - connectionString: mongoServer.getUri() as string, + connectionString: connectionString as string, databaseName: "test", collectionName: "test", }), From 259952e5c4577b9b603dbd86dc577352de8e3ff7 Mon Sep 17 00:00:00 2001 From: Jan-Henrik Damaschke Date: Sun, 31 Dec 2023 05:35:05 +0100 Subject: [PATCH 03/10] feat(encryption): :sparkles: Added key encryption, major refactor Replaced crypto library for nonce-misuse resistance and more modern nonce handling; Refactored types; Added optional encryption of keys; Adjusted normalization to encryption --- package.json | 1 + pnpm-lock.yaml | 7 ++ src/_utils.ts | 228 ++++++---------------------------------- src/utils.ts | 101 +++++++++++++----- test/encryption.test.ts | 28 +++-- 5 files changed, 136 insertions(+), 229 deletions(-) diff --git a/package.json b/package.json index 92492246a..27f9e782f 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "unstorage": "pnpm jiti src/cli" }, "dependencies": { + "@noble/ciphers": "^0.4.1", "anymatch": "^3.1.3", "chokidar": "^3.5.3", "destr": "^2.0.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 434a1c1cb..97e5d7346 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5,6 +5,9 @@ settings: excludeLinksFromLockfile: false dependencies: + '@noble/ciphers': + specifier: ^0.4.1 + version: 0.4.1 anymatch: specifier: ^3.1.3 version: 3.1.3 @@ -1265,6 +1268,10 @@ packages: engines: {node: ^14.16.0 || >=16.0.0} dev: true + /@noble/ciphers@0.4.1: + resolution: {integrity: sha512-QCOA9cgf3Rc33owG0AYBB9wszz+Ul2kramWN8tXG44Gyciud/tbkEqvxRF/IpqQaBpRBNi9f4jdNxqB2CQCIXg==} + dev: false + /@nodelib/fs.scandir@2.1.5: resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} diff --git a/src/_utils.ts b/src/_utils.ts index fb1cb4c97..bde702aa9 100644 --- a/src/_utils.ts +++ b/src/_utils.ts @@ -1,25 +1,10 @@ import { subtle, getRandomValues } from "uncrypto"; +import { xchacha20poly1305 } from '@noble/ciphers/chacha'; +import { siv } from '@noble/ciphers/aes'; type Awaited = T extends Promise ? Awaited : T; type Promisified = Promise>; -type EncryptStorageOptions = Omit & { - key: string; - encryptKeys?: boolean; -}; - -type DecryptStorageOptions = Omit & { - key: string; - decryptKeys?: boolean; -}; - -interface EncryptionOptions { - key: CryptoKey; - algorithm: "AES-GCM" | "AES-CBC" | "RSA-OAEP"; - iv?: Uint8Array; - raw?: boolean; -} - export function wrapToPromise(value: T) { if (!value || typeof (value as any).then !== "function") { return Promise.resolve(value) as Promisified; @@ -97,25 +82,17 @@ export function deserializeRaw(value: any) { // Encryption +// Use only for GCM-SIV, due to nonce-misuse-resistance. We need deterministic keys. +const predefinedSivNonce = 'ThtnxLK9eCF4OLMg'; +const encryptionPrefix = '$enc:'; + export interface StorageValueEnvelope { - iv: string; + nonce: string; encryptedValue: string; } -export async function generateAesKey(algorithm: "AES-GCM" | "AES-CBC" = "AES-GCM") { - return genBase64FromBytes( - new Uint8Array( - await subtle.exportKey( - 'raw', await subtle.generateKey( - { - name: algorithm, - length: 256, - }, - true, - ['encrypt', 'decrypt']) - ) - ) - ); +export function generateEncryptionKey() { + return genBase64FromBytes(getRandomValues(new Uint8Array(32))); } export async function generateRsaKeyPair() { @@ -134,146 +111,41 @@ export async function generateRsaKeyPair() { }; } -export async function encryptStorageValue(storageValue: any, options: EncryptStorageOptions) { - const { key, algorithm, raw } = options; - const cryptoKey = await subtle.importKey('raw', genBytesFromBase64(key), { - name: "AES-GCM", - length: 256 - }, true, ['encrypt', 'decrypt']); - const iv = getRandomValues(new Uint8Array(12)); - const encryptedValue = await encryptSym(storageValue, { key: cryptoKey, algorithm, raw, iv }); +export function encryptStorageValue(storageValue: any, key: string, raw?: boolean): StorageValueEnvelope { + const cryptoKey = genBytesFromBase64(key); + const nonce = getRandomValues(new Uint8Array(24)); + const chacha = xchacha20poly1305(cryptoKey, nonce); + const encryptedValue = chacha.encrypt(raw ? storageValue : new TextEncoder().encode(storageValue)); return { - encryptedValue, - iv: genBase64FromBytes(iv), + encryptedValue: genBase64FromBytes(new Uint8Array(encryptedValue)), + nonce: genBase64FromBytes(nonce), }; } -export async function decryptStorageValue(storageValue: StorageValueEnvelope, options: DecryptStorageOptions): Promise { - const { key, algorithm, raw } = options; - const { encryptedValue, iv } = storageValue; - const cryptoKey = await subtle.importKey('raw', genBytesFromBase64(key), { - name: "AES-GCM", - length: 256 - }, true, ['encrypt', 'decrypt']); - const decrypted = await decryptSym(encryptedValue, { key: cryptoKey, algorithm, raw, iv: genBytesFromBase64(iv) }); - return decrypted as T; +export function decryptStorageValue(storageValue: StorageValueEnvelope, key: string, raw?: boolean): T { + const { encryptedValue, nonce } = storageValue; + const cryptoKey = genBytesFromBase64(key); + const chacha = xchacha20poly1305(cryptoKey, genBytesFromBase64(nonce)); + const decryptedValue = chacha.decrypt(genBytesFromBase64(encryptedValue)); + return raw ? decryptedValue as T : new TextDecoder().decode(decryptedValue) as T; } -export async function encryptStorageKey(storageKey: string, options: EncryptStorageOptions) { - const { key, algorithm, raw } = options; - const cryptoKey = await subtle.importKey('raw', genBytesFromBase64(key), { - name: "AES-GCM", - length: 256 - }, true, ['encrypt', 'decrypt']); - return await encryptSym(storageKey, { key: cryptoKey, algorithm, raw, iv: new Uint8Array(12) }); +export function encryptStorageKey(storageKey: string, key: string) { + const cryptoKey = genBytesFromBase64(key); + const gcmSiv = siv(cryptoKey, genBytesFromBase64(predefinedSivNonce)); + const encryptedKey = gcmSiv.encrypt(new Uint8Array(new TextEncoder().encode(storageKey))); + return encryptionPrefix + genBase64FromBytes(encryptedKey); } -export async function decryptStorageKey(encryptedKey: string, options: DecryptStorageOptions) { - const { key, algorithm } = options; - const cryptoKey = await subtle.importKey('raw', genBytesFromBase64(key), { - name: "AES-GCM", - length: 256 - }, true, ['encrypt', 'decrypt']); - const decrypted = await decryptSym(encryptedKey, { key: cryptoKey, algorithm, iv: new Uint8Array(12) }); - return decrypted as string; -} - -async function encryptSym(content: any, options: EncryptionOptions) { - if (options.raw) { - return genBase64FromBytes(new Uint8Array(await subtle.encrypt( - { - name: options.algorithm, - iv: options.iv, - }, - options.key, - content - ))); - } - const encoded = new TextEncoder().encode(content); - const ciphertext = await subtle.encrypt( - { - name: options.algorithm, - iv: options.iv, - }, - options.key, - encoded - ); - return genBase64FromBytes(new Uint8Array(ciphertext)); -} - -async function encryptAsym(content: any, options: EncryptionOptions) { - if (options.raw) { - return await subtle.encrypt( - { - name: options.algorithm, - }, - options.key, - content - ); - } - const encoded = new TextEncoder().encode(content); - const ciphertext = await subtle.encrypt( - { - name: options.algorithm, - }, - options.key, - encoded - ); - return genBase64FromBytes(new Uint8Array(ciphertext)); -} - -async function decryptSym(content: any, options: EncryptionOptions) { - const decoded = genBytesFromBase64(content); - if (options.raw) { - return await subtle.decrypt( - { - name: options.algorithm, - iv: options.iv, - }, - options.key, - decoded - ); - } - const decrypted = await subtle.decrypt( - { - name: 'AES-GCM', - iv: options.iv, - }, - options.key, - decoded - ); - return new TextDecoder().decode(decrypted); -} - -async function decryptAsym(content: any, options: EncryptionOptions) { - if (options.raw) { - return await subtle.decrypt( - { - name: options.algorithm, - }, - options.key, - content - ); - } - - const decoded = genBytesFromBase64(content); - - return await subtle.decrypt( - { - name: 'AES-GCM', - iv: options.iv, - }, - options.key, - decoded - ); +export function decryptStorageKey(encryptedKey: string, key: string) { + const cryptoKey = genBytesFromBase64(key); + const gcmSiv = siv(cryptoKey, genBytesFromBase64(predefinedSivNonce)); + const decryptedKey = gcmSiv.decrypt(genBytesFromBase64(encryptedKey.slice(encryptionPrefix.length))); + return new TextDecoder().decode(decryptedKey); } // Base64 utilities - Waiting for https://github.com/unjs/knitwork/pull/83 // TODO: Replace with knitwork imports as soon as PR is merged -interface CodegenOptions { - encoding?: 'utf8' | 'ascii' | 'url'; -} - function genBytesFromBase64(input: string) { return Uint8Array.from( globalThis.atob(input), @@ -291,39 +163,3 @@ function genBase64FromBytes(input: Uint8Array, urlSafe?: boolean) { } return globalThis.btoa(String.fromCodePoint(...input)); } - -function genBase64FromString( - input: string, - options: CodegenOptions = {} -) { - if (options.encoding === 'utf8') { - return genBase64FromBytes(new TextEncoder().encode(input)); - } - if (options.encoding === 'url') { - return genBase64FromBytes(new TextEncoder().encode(input)) - .replace(/\+/g, '-') - .replace(/\//g, '_') - .replace(/=+$/, ''); - } - return globalThis.btoa(input); -} - -function genStringFromBase64( - input: string, - options: CodegenOptions = {} -) { - if (options.encoding === 'utf8') { - return new TextDecoder().decode(genBytesFromBase64(input)); - } - if (options.encoding === 'url') { - input = input.replace(/-/g, '+').replace(/_/g, '/'); - const paddingLength = input.length % 4; - if (paddingLength === 2) { - input += '=='; - } else if (paddingLength === 3) { - input += '='; - } - return new TextDecoder().decode(genBytesFromBase64(input)); - } - return globalThis.atob(input); -} diff --git a/src/utils.ts b/src/utils.ts index 266262c93..3c9c3d929 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -51,54 +51,103 @@ export function encryptedStorage( ): Storage { const encStorage: Storage = { ...storage }; - encStorage.setItem = async (key, value, ...args) => { + encStorage.hasItem = (key = "", ...args) => + storage.hasItem(encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, ...args); + + encStorage.getItem = async (key, ...args) => { if (encryptKeys) { - key = await encryptStorageKey(key, { algorithm: "AES-GCM", key: encryptionKey }); + key = encryptStorageKey(normalizeKey(key), encryptionKey); } - const encryptedValue = await encryptStorageValue(stringify(value), { algorithm: "AES-GCM", key: encryptionKey }); - storage.setItem(key, encryptedValue as T, ...args); - }; - - encStorage.setItemRaw = async (key, value, ...args) => { - const encryptedValue = await encryptStorageValue(value, { algorithm: "AES-GCM", key: encryptionKey, raw: true }); - storage.setItem(key, encryptedValue as T, ...args); + const value = await storage.getItem(key, ...args); + return value ? destr(decryptStorageValue(value as StorageValueEnvelope, encryptionKey)) : null; }; - encStorage.getItem = async (key, ...args) => { + encStorage.getItems = async (items, commonOptions) => { + let encryptedItems; if (encryptKeys) { - key = await decryptStorageKey(key, { algorithm: "AES-GCM", key: encryptionKey }); + const encryptedKeyItems = items.map((item) => { + const isStringItem = typeof item === "string"; + const key = encryptStorageKey(normalizeKey(isStringItem ? item : item.key), encryptionKey); + const options = + isStringItem || !item.options + ? commonOptions + : { ...commonOptions, ...item.options }; + + return { key, options }; + }); + encryptedItems = await storage.getItems(encryptedKeyItems, commonOptions); + } else { + encryptedItems = await storage.getItems(items, commonOptions); } - const value = await storage.getItem(key, ...args); - return value ? destr(await decryptStorageValue(value as StorageValueEnvelope, { algorithm: "AES-GCM", key: encryptionKey })) : null; + return (encryptedItems.map((encryptedItem) => { + const { value, key, ...rest } = encryptedItem; + const decryptedValue = (decryptStorageValue(value as StorageValueEnvelope, encryptionKey)) as StorageValue; + return { value: decryptedValue, key: encryptKeys ? decryptStorageKey(normalizeKey(key), encryptionKey) : key, ...rest }; + })); }; encStorage.getItemRaw = async (key, ...args) => { const value = await storage.getItem(key, ...args); - return value ? await decryptStorageValue(value as StorageValueEnvelope, { algorithm: "AES-GCM", key: encryptionKey, raw: true }) : null; + return value ? decryptStorageValue(value as StorageValueEnvelope, encryptionKey, true) : null; }; - encStorage.getItems = async (items, ...args) => { - const encryptedItems = await storage.getItems(items, ...args); - return await Promise.all(encryptedItems.map(async (encryptedItem) => { - const { value, ...rest } = encryptedItem; - const decryptedValue = (await decryptStorageValue(value as StorageValueEnvelope, { algorithm: "AES-GCM", key: encryptionKey })) as StorageValue; - return { value: decryptedValue, ...rest }; - })); + // eslint-disable-next-line require-await + encStorage.setItem = async (key, value, ...args) => { + if (encryptKeys) { + key = encryptStorageKey(normalizeKey(key), encryptionKey); + } + const encryptedValue = encryptStorageValue(stringify(value), encryptionKey); + storage.setItem(key, encryptedValue as T, ...args); }; + // eslint-disable-next-line require-await encStorage.setItems = async (items, ...args) => { - const encryptedItems = await Promise.all(items.map(async (item) => { - const { value, ...rest } = item; - const encryptedValue: StorageValueEnvelope = await encryptStorageValue(stringify(value), { algorithm: "AES-GCM", key: encryptionKey }); - return { value: encryptedValue, ...rest }; - })); + const encryptedItems = items.map((item) => { + const { value, key, ...rest } = item; + const encryptedValue: StorageValueEnvelope = encryptStorageValue(stringify(value), encryptionKey); + return { value: encryptedValue, key: encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, ...rest }; + }); storage.setItems(encryptedItems, ...args); }; + // eslint-disable-next-line require-await + encStorage.setItemRaw = async (key, value, ...args) => { + const encryptedValue = encryptStorageValue(value, encryptionKey, true); + storage.setItem(key, encryptedValue as T, ...args); + }; + + encStorage.removeItem = (key, ...args) => + storage.removeItem(encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, ...args); + + // TODO: Meta encryption + encStorage.setMeta = (key, ...args) => + storage.setMeta(encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, ...args); + + // TODO: Meta encryption + encStorage.getMeta = (key, ...args) => + storage.getMeta(encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, ...args); + + // TODO: Meta encryption + encStorage.removeMeta = (key, ...args) => + storage.removeMeta(encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, ...args); + + encStorage.getKeys = async (base, ...args) => { + const keys = await storage.getKeys('', ...args); + const decryptedKeys = keys.map((key) => decryptStorageKey(key, encryptionKey)); + if (base) { + return decryptedKeys.filter((key) => key.startsWith(base!) && !key.endsWith("$")); + } + return decryptedKeys.filter((key) => !key.endsWith("$")); + }; + return encStorage; } export function normalizeKey(key?: string) { + // Don't normalize encrypted keys + if (key?.startsWith("$enc:")) { + return key; + } if (!key) { return ""; } diff --git a/test/encryption.test.ts b/test/encryption.test.ts index 346f97fff..86ace97b3 100644 --- a/test/encryption.test.ts +++ b/test/encryption.test.ts @@ -10,15 +10,29 @@ import { TestContext, TestOptions } from "./drivers/utils"; const encryptionKey = 'e9iF+8pS8qAjnj7B1+ZwdzWQ+KXNJGUPW3HdDuMJPgI='; describe("encryption", () => { - it.skip("encryptedStorage", async () => { - const storage = createStorage(); - const encStorage = encryptedStorage(storage, encryptionKey); + const keyEncryptionEnabled = true; + const storage = createStorage(); + const encStorage = encryptedStorage(storage, encryptionKey, keyEncryptionEnabled); + + it.skip("setItem", async () => { await encStorage.setItem("s1:a", "test_data"); - await encStorage.setItem("s2:a", "test_data"); await encStorage.setItem("s3:a?q=1", "test_data"); + await encStorage.clear(); + }); + + it.skip("hasItem", async () => { + await encStorage.setItem("s1:a", "test_data"); expect(await encStorage.hasItem("s1:a")).toBe(true); - expect(await encStorage.getItem("s1:a")).toBe("test_data"); + await encStorage.clear(); + }); + + it.skip("getItem", async () => { + await encStorage.setItem("s2:a", "test_data"); + await encStorage.setItem("s3:a?q=1", "test_data"); + expect(await encStorage.getItem("s2:a")).toBe("test_data"); + expect(await encStorage.getItem("s3:a?q=1")).toBe("test_data"); expect(await encStorage.getItem("s3:a?q=2")).toBe("test_data"); + await encStorage.clear(); }); testEncryptionDriver({ @@ -28,7 +42,7 @@ describe("encryption", () => { export function testEncryptionDriver(opts: TestOptions) { const ctx: TestContext = { - storage: encryptedStorage(createStorage({ driver: opts.driver }), encryptionKey, false), + storage: encryptedStorage(createStorage({ driver: opts.driver }), encryptionKey, true), driver: opts.driver, }; @@ -97,7 +111,7 @@ export function testEncryptionDriver(opts: TestOptions) { // eslint-disable-next-line require-await it("serialize (error for non primitives)", async () => { - class Test {} + class Test { } expect( ctx.storage.setItem("/data/badvalue.json", new Test()) ).rejects.toThrow("[unstorage] Cannot stringify value!"); From 9ba000b20cdf0a47f6de39dde812e56f74435ec3 Mon Sep 17 00:00:00 2001 From: Jan-Henrik Damaschke Date: Tue, 2 Jan 2024 01:28:26 +0100 Subject: [PATCH 04/10] feat(encryption): :sparkles: Switched to url encoding for base64; Implemented generic testing option for encryption --- src/_utils.ts | 15 +- src/utils.ts | 24 +-- test/drivers/azure-storage-blob.test.ts | 2 +- test/drivers/azure-storage-table.test.ts | 2 +- test/drivers/utils.ts | 7 +- test/encryption.test.ts | 227 ++++------------------- 6 files changed, 63 insertions(+), 214 deletions(-) diff --git a/src/_utils.ts b/src/_utils.ts index bde702aa9..ff103cdca 100644 --- a/src/_utils.ts +++ b/src/_utils.ts @@ -134,19 +134,28 @@ export function encryptStorageKey(storageKey: string, key: string) { const cryptoKey = genBytesFromBase64(key); const gcmSiv = siv(cryptoKey, genBytesFromBase64(predefinedSivNonce)); const encryptedKey = gcmSiv.encrypt(new Uint8Array(new TextEncoder().encode(storageKey))); - return encryptionPrefix + genBase64FromBytes(encryptedKey); + return encryptionPrefix + genBase64FromBytes(encryptedKey, true); } export function decryptStorageKey(encryptedKey: string, key: string) { const cryptoKey = genBytesFromBase64(key); const gcmSiv = siv(cryptoKey, genBytesFromBase64(predefinedSivNonce)); - const decryptedKey = gcmSiv.decrypt(genBytesFromBase64(encryptedKey.slice(encryptionPrefix.length))); + const decryptedKey = gcmSiv.decrypt(genBytesFromBase64(encryptedKey.slice(encryptionPrefix.length), true)); return new TextDecoder().decode(decryptedKey); } // Base64 utilities - Waiting for https://github.com/unjs/knitwork/pull/83 // TODO: Replace with knitwork imports as soon as PR is merged -function genBytesFromBase64(input: string) { +function genBytesFromBase64(input: string, urlSafe?: boolean) { + if (urlSafe) { + input = input.replace(/-/g, "+").replace(/_/g, "/"); + const paddingLength = input.length % 4; + if (paddingLength === 2) { + input += "=="; + } else if (paddingLength === 3) { + input += "="; + } + } return Uint8Array.from( globalThis.atob(input), (c) => c.codePointAt(0) as number diff --git a/src/utils.ts b/src/utils.ts index 3c9c3d929..607db751a 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -51,7 +51,7 @@ export function encryptedStorage( ): Storage { const encStorage: Storage = { ...storage }; - encStorage.hasItem = (key = "", ...args) => + encStorage.hasItem = (key, ...args) => storage.hasItem(encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, ...args); encStorage.getItem = async (key, ...args) => { @@ -97,7 +97,7 @@ export function encryptedStorage( key = encryptStorageKey(normalizeKey(key), encryptionKey); } const encryptedValue = encryptStorageValue(stringify(value), encryptionKey); - storage.setItem(key, encryptedValue as T, ...args); + return storage.setItem(key, encryptedValue as T, ...args); }; // eslint-disable-next-line require-await @@ -107,37 +107,37 @@ export function encryptedStorage( const encryptedValue: StorageValueEnvelope = encryptStorageValue(stringify(value), encryptionKey); return { value: encryptedValue, key: encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, ...rest }; }); - storage.setItems(encryptedItems, ...args); + return storage.setItems(encryptedItems, ...args); }; // eslint-disable-next-line require-await encStorage.setItemRaw = async (key, value, ...args) => { const encryptedValue = encryptStorageValue(value, encryptionKey, true); - storage.setItem(key, encryptedValue as T, ...args); + return storage.setItem(key, encryptedValue as T, ...args); }; encStorage.removeItem = (key, ...args) => storage.removeItem(encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, ...args); - // TODO: Meta encryption encStorage.setMeta = (key, ...args) => storage.setMeta(encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, ...args); - // TODO: Meta encryption encStorage.getMeta = (key, ...args) => storage.getMeta(encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, ...args); - // TODO: Meta encryption encStorage.removeMeta = (key, ...args) => storage.removeMeta(encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, ...args); encStorage.getKeys = async (base, ...args) => { - const keys = await storage.getKeys('', ...args); - const decryptedKeys = keys.map((key) => decryptStorageKey(key, encryptionKey)); - if (base) { - return decryptedKeys.filter((key) => key.startsWith(base!) && !key.endsWith("$")); + if (encryptKeys) { + const keys = await storage.getKeys('', ...args); + const decryptedKeys = keys.map((key) => decryptStorageKey(key, encryptionKey)); + if (base) { + return decryptedKeys.filter((key) => key.startsWith(base!) && !key.endsWith("$")); + } + return decryptedKeys.filter((key) => !key.endsWith("$")); } - return decryptedKeys.filter((key) => !key.endsWith("$")); + return storage.getKeys(base, ...args); }; return encStorage; diff --git a/test/drivers/azure-storage-blob.test.ts b/test/drivers/azure-storage-blob.test.ts index 766d5a249..27b1bb1c8 100644 --- a/test/drivers/azure-storage-blob.test.ts +++ b/test/drivers/azure-storage-blob.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { describe, beforeAll, afterAll } from "vitest"; import driver from "../../src/drivers/azure-storage-blob"; import { testDriver } from "./utils"; import { BlobServiceClient } from "@azure/storage-blob"; diff --git a/test/drivers/azure-storage-table.test.ts b/test/drivers/azure-storage-table.test.ts index 5e9a06ed7..3712b5cc5 100644 --- a/test/drivers/azure-storage-table.test.ts +++ b/test/drivers/azure-storage-table.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { describe, beforeAll, afterAll } from "vitest"; import driver from "../../src/drivers/azure-storage-table"; import { testDriver } from "./utils"; import { TableClient } from "@azure/data-tables"; diff --git a/test/drivers/utils.ts b/test/drivers/utils.ts index e4dafa2d5..d33e7b59d 100644 --- a/test/drivers/utils.ts +++ b/test/drivers/utils.ts @@ -1,5 +1,5 @@ import { it, expect } from "vitest"; -import { Storage, Driver, createStorage, restoreSnapshot } from "../../src"; +import { Storage, Driver, createStorage, restoreSnapshot, encryptedStorage } from "../../src"; export interface TestContext { storage: Storage; @@ -8,12 +8,15 @@ export interface TestContext { export interface TestOptions { driver: Driver; + contentEncryption?: boolean; + keyEncryption?: boolean; additionalTests?: (ctx: TestContext) => void; } export function testDriver(opts: TestOptions) { + const encryptionKey = 'e9iF+8pS8qAjnj7B1+ZwdzWQ+KXNJGUPW3HdDuMJPgI='; const ctx: TestContext = { - storage: createStorage({ driver: opts.driver }), + storage: opts.contentEncryption ? encryptedStorage(createStorage({ driver: opts.driver }), encryptionKey, opts.keyEncryption) : createStorage({ driver: opts.driver }), driver: opts.driver, }; diff --git a/test/encryption.test.ts b/test/encryption.test.ts index 86ace97b3..1bf247e3c 100644 --- a/test/encryption.test.ts +++ b/test/encryption.test.ts @@ -1,199 +1,36 @@ -import { describe, it, expect } from "vitest"; -import { - createStorage, - encryptedStorage, - restoreSnapshot, -} from "../src"; -import driver from "../src/drivers/memory"; -import { TestContext, TestOptions } from "./drivers/utils"; - -const encryptionKey = 'e9iF+8pS8qAjnj7B1+ZwdzWQ+KXNJGUPW3HdDuMJPgI='; +import { resolve } from "node:path"; +import { describe, it, expect, vi } from "vitest"; +import driver from "../src/drivers/fs"; +import { writeFile } from "../src/drivers/utils/node-fs"; +import { testDriver } from "./drivers/utils"; describe("encryption", () => { - const keyEncryptionEnabled = true; - const storage = createStorage(); - const encStorage = encryptedStorage(storage, encryptionKey, keyEncryptionEnabled); - - it.skip("setItem", async () => { - await encStorage.setItem("s1:a", "test_data"); - await encStorage.setItem("s3:a?q=1", "test_data"); - await encStorage.clear(); - }); - - it.skip("hasItem", async () => { - await encStorage.setItem("s1:a", "test_data"); - expect(await encStorage.hasItem("s1:a")).toBe(true); - await encStorage.clear(); - }); - - it.skip("getItem", async () => { - await encStorage.setItem("s2:a", "test_data"); - await encStorage.setItem("s3:a?q=1", "test_data"); - expect(await encStorage.getItem("s2:a")).toBe("test_data"); - expect(await encStorage.getItem("s3:a?q=1")).toBe("test_data"); - expect(await encStorage.getItem("s3:a?q=2")).toBe("test_data"); - await encStorage.clear(); - }); - - testEncryptionDriver({ - driver: driver(), + const dir = resolve(__dirname, "tmp/fs"); + + // Example for fs driver + testDriver({ + driver: driver({ base: dir }), + contentEncryption: true, + keyEncryption: true, + additionalTests(ctx) { + it("native meta", async () => { + const meta = await ctx.storage.getMeta("/s1/a"); + expect(meta.atime?.constructor.name).toBe("Date"); + expect(meta.mtime?.constructor.name).toBe("Date"); + expect(meta.size).toBeGreaterThan(0); + }); + it("watch filesystem", async () => { + const watcher = vi.fn(); + await ctx.storage.watch(watcher); + await writeFile(resolve(dir, "s1/random_file"), "random", "utf8"); + await new Promise((resolve) => setTimeout(resolve, 500)); + expect(watcher).toHaveBeenCalledWith("update", "s1:random_file"); + }); + + it("allow double dots in filename: ", async () => { + await ctx.storage.setItem("s1/te..st..js", "ok"); + expect(await ctx.storage.getItem("s1/te..st..js")).toBe("ok"); + }); + }, }); }); - -export function testEncryptionDriver(opts: TestOptions) { - const ctx: TestContext = { - storage: encryptedStorage(createStorage({ driver: opts.driver }), encryptionKey, true), - driver: opts.driver, - }; - - it("init", async () => { - await restoreSnapshot(ctx.storage, { initial: "works" }); - expect(await ctx.storage.getItem("initial")).toBe("works"); - await ctx.storage.clear(); - }); - - it("initial state", async () => { - expect(await ctx.storage.hasItem("s1:a")).toBe(false); - expect(await ctx.storage.getItem("s2:a")).toBe(null); - expect(await ctx.storage.getKeys()).toMatchObject([]); - }); - - it("setItem", async () => { - await ctx.storage.setItem("s1:a", "test_data"); - await ctx.storage.setItem("s2:a", "test_data"); - await ctx.storage.setItem("s3:a?q=1", "test_data"); - expect(await ctx.storage.hasItem("s1:a")).toBe(true); - expect(await ctx.storage.getItem("s1:a")).toBe("test_data"); - expect(await ctx.storage.getItem("s3:a?q=2")).toBe("test_data"); - }); - - it("getKeys", async () => { - expect(await ctx.storage.getKeys().then((k) => k.sort())).toMatchObject( - ["s1:a", "s2:a", "s3:a"].sort() - ); - expect(await ctx.storage.getKeys("s1").then((k) => k.sort())).toMatchObject( - ["s1:a"].sort() - ); - }); - - it("serialize (object)", async () => { - await ctx.storage.setItem("/data/test.json", { json: "works" }); - expect(await ctx.storage.getItem("/data/test.json")).toMatchObject({ - json: "works", - }); - }); - - it("serialize (primitive)", async () => { - await ctx.storage.setItem("/data/true.json", true); - expect(await ctx.storage.getItem("/data/true.json")).toBe(true); - }); - - it("serialize (lossy object with toJSON())", async () => { - class Test1 { - toJSON() { - return "SERIALIZED"; - } - } - await ctx.storage.setItem("/data/serialized1.json", new Test1()); - expect(await ctx.storage.getItem("/data/serialized1.json")).toBe( - "SERIALIZED" - ); - class Test2 { - toJSON() { - return { serializedObj: "works" }; - } - } - await ctx.storage.setItem("/data/serialized2.json", new Test2()); - expect(await ctx.storage.getItem("/data/serialized2.json")).toMatchObject({ - serializedObj: "works", - }); - }); - - // eslint-disable-next-line require-await - it("serialize (error for non primitives)", async () => { - class Test { } - expect( - ctx.storage.setItem("/data/badvalue.json", new Test()) - ).rejects.toThrow("[unstorage] Cannot stringify value!"); - }); - - it("raw support", async () => { - const value = new Uint8Array([1, 2, 3]); - await ctx.storage.setItemRaw("/data/raw.bin", value); - const rValue = await ctx.storage.getItemRaw("/data/raw.bin"); - const rValueLen = rValue?.length || rValue?.byteLength; - if (rValueLen !== value.length) { - console.log("Invalid raw value length:", rValue, "Length:", rValueLen); - } - expect(rValueLen).toBe(value.length); - expect(Buffer.from(rValue).toString("base64")).toBe( - Buffer.from(value).toString("base64") - ); - }); - - // Bulk tests - it("setItems", async () => { - await ctx.storage.setItems([ - { key: "t:1", value: "test_data_t1" }, - { key: "t:2", value: "test_data_t2" }, - { key: "t:3", value: "test_data_t3" }, - ]); - expect(await ctx.storage.getItem("t:1")).toBe("test_data_t1"); - expect(await ctx.storage.getItem("t:2")).toBe("test_data_t2"); - expect(await ctx.storage.getItem("t:3")).toBe("test_data_t3"); - }); - - it("getItems", async () => { - await ctx.storage.setItem("v1:a", "test_data_v1:a"); - await ctx.storage.setItem("v2:a", "test_data_v2:a"); - await ctx.storage.setItem("v3:a?q=1", "test_data_v3:a?q=1"); - - expect( - await ctx.storage.getItems([{ key: "v1:a" }, "v2:a", { key: "v3:a?q=1" }]) - ).toMatchObject([ - { - key: "v1:a", - value: "test_data_v1:a", - }, - { - key: "v2:a", - value: "test_data_v2:a", - }, - { - key: "v3:a", // key should lose the querystring - value: "test_data_v3:a?q=1", - }, - ]); - }); - - it("getItem - return falsy values when set in storage", async () => { - await ctx.storage.setItem("zero", 0); - expect(await ctx.storage.getItem("zero")).toBe(0); - - await ctx.storage.setItem("my-false-flag", false); - expect(await ctx.storage.getItem("my-false-flag")).toBe(false); - }); - - // TODO: Refactor to move after cleanup - if (opts.additionalTests) { - opts.additionalTests(ctx); - } - - it("removeItem", async () => { - await ctx.storage.removeItem("s1:a", false); - expect(await ctx.storage.hasItem("s1:a")).toBe(false); - expect(await ctx.storage.getItem("s1:a")).toBe(null); - }); - - it("clear", async () => { - await ctx.storage.clear(); - expect(await ctx.storage.getKeys()).toMatchObject([]); - // ensure we can clear empty storage as well: #162 - await ctx.storage.clear(); - expect(await ctx.storage.getKeys()).toMatchObject([]); - }); - - it("dispose", async () => { - await ctx.storage.dispose(); - }); -} From 385798a4bd0edc5e7b309779cc6c7d8c9e0a1736 Mon Sep 17 00:00:00 2001 From: Jan-Henrik Damaschke Date: Tue, 2 Jan 2024 16:43:25 +0100 Subject: [PATCH 05/10] test(encryption): :white_check_mark: Added test for chained prefixedStorage and storage server --- test/server.test.ts | 50 +++++++++++++++++++++++++++++++++++++++++++- test/storage.test.ts | 50 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) diff --git a/test/server.test.ts b/test/server.test.ts index adc30c1a5..60c67887b 100644 --- a/test/server.test.ts +++ b/test/server.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect } from "vitest"; import { listen } from "listhen"; import { $fetch } from "ofetch"; -import { createStorage } from "../src"; +import { createStorage, encryptedStorage } from "../src"; import { createStorageServer } from "../src/server"; describe("server", () => { @@ -48,3 +48,51 @@ describe("server", () => { await close(); }); }); + +describe("encrypted server", () => { + const encryptionKey = 'e9iF+8pS8qAjnj7B1+ZwdzWQ+KXNJGUPW3HdDuMJPgI='; + + it("basic", async () => { + const storage = createStorage(); + const encStorage = encryptedStorage(storage, encryptionKey, true) + const storageServer = createStorageServer(encStorage, { + authorize(req) { + if (req.type === "read" && req.key.startsWith("private:")) { + throw new Error("Unauthorized Read"); + } + }, + }); + const { close, url: serverURL } = await listen(storageServer.handle, { + port: { random: true }, + }); + + const fetchStorage = (url: string, options?: any) => + $fetch(url, { baseURL: serverURL, ...options }); + + expect(await fetchStorage("foo/", {})).toMatchObject([]); + + await encStorage.setItem("foo/bar", "bar"); + await encStorage.setMeta("foo/bar", { mtime: new Date() }); + expect(await fetchStorage("foo/bar")).toBe("bar"); + + expect( + await fetchStorage("foo/bar", { method: "PUT", body: "updated" }) + ).toBe("OK"); + expect(await fetchStorage("foo/bar")).toBe("updated"); + expect(await fetchStorage("/")).toMatchObject(["foo/bar"]); + + expect(await fetchStorage("foo/bar", { method: "DELETE" })).toBe("OK"); + expect(await fetchStorage("foo/bar/", {})).toMatchObject([]); + + await expect( + fetchStorage("private/foo/bar", { method: "GET" }).catch((error) => { + throw error.data; + }) + ).rejects.toMatchObject({ + statusCode: 401, + statusMessage: "Unauthorized Read", + }); + + await close(); + }); +}); diff --git a/test/storage.test.ts b/test/storage.test.ts index aac85e67c..1d13c538f 100644 --- a/test/storage.test.ts +++ b/test/storage.test.ts @@ -4,6 +4,7 @@ import { snapshot, restoreSnapshot, prefixStorage, + encryptedStorage, } from "../src"; import memory from "../src/drivers/memory"; @@ -136,6 +137,8 @@ describe("storage", () => { }); describe("utils", () => { + const encryptionKey = 'e9iF+8pS8qAjnj7B1+ZwdzWQ+KXNJGUPW3HdDuMJPgI='; + it("prefixStorage", async () => { const storage = createStorage(); const pStorage = prefixStorage(storage, "foo"); @@ -155,6 +158,53 @@ describe("utils", () => { expect(await mntStorage.getKeys("foo")).toStrictEqual(["foo:x", "foo:y"]); }); + it("prefixed encryptedStorage", async () => { + const storage = createStorage(); + const pStorage = prefixStorage(storage, "foo"); + const encStorage = encryptedStorage(pStorage, encryptionKey, false); + await encStorage.setItem("x", "bar"); + await encStorage.setItem("y", "baz"); + expect(await encStorage.getItem("x")).toBe("bar"); + expect(await encStorage.getKeys()).toStrictEqual(["x", "y"]); + expect(await storage.getItem("x")).toBeNull(); + expect(await storage.getItem("foo:x")).toBeTypeOf("object"); + expect(await storage.getItem("foo:x")).toBeTypeOf("object"); + + // Higher order storage + const secondStorage = createStorage(); + secondStorage.mount("/mnt", storage); + const mntStorage = prefixStorage(secondStorage, "mnt"); + + expect(await mntStorage.getKeys()).toStrictEqual(["foo:x", "foo:y"]); + // Get keys from sub-storage + expect(await mntStorage.getKeys("foo")).toStrictEqual(["foo:x", "foo:y"]); + }); + + it("prefixed encryptedStorage with key encryption", async () => { + const storage = createStorage(); + const pStorage = prefixStorage(storage, "foo"); + const encStorage = encryptedStorage(pStorage, encryptionKey, true); + await encStorage.setItem("x", "bar"); + await encStorage.setItem("y", "baz"); + expect(await encStorage.getItem("x")).toBe("bar"); + expect(await encStorage.getKeys()).toStrictEqual(["x", "y"]); + expect(await storage.getItem("x")).toBeNull(); + expect(await storage.getItem("foo:x")).toBeTypeOf("object"); + expect(await storage.getItem("foo:x")).toBeTypeOf("object"); + + // Higher order storage + const secondStorage = createStorage(); + secondStorage.mount("/mnt", storage); + const mntStorage = prefixStorage(secondStorage, "mnt"); + + for (const key of await mntStorage.getKeys()) { + expect(key).toContain("foo:$enc:"); + } + for (const key of await mntStorage.getKeys("foo")) { + expect(key).toContain("foo:$enc:"); + } + }); + it("stringify", () => { const storage = createStorage(); expect(async () => await storage.setItem("foo", [])).not.toThrow(); From edb1a72cb2b08ecd8278228f0527e457a3631aa9 Mon Sep 17 00:00:00 2001 From: Jan-Henrik Damaschke Date: Tue, 2 Jan 2024 16:58:39 +0100 Subject: [PATCH 06/10] refactor(encryption): :recycle: Removed unused utility functions --- src/_utils.ts | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/_utils.ts b/src/_utils.ts index ff103cdca..c3815249b 100644 --- a/src/_utils.ts +++ b/src/_utils.ts @@ -91,26 +91,6 @@ export interface StorageValueEnvelope { encryptedValue: string; } -export function generateEncryptionKey() { - return genBase64FromBytes(getRandomValues(new Uint8Array(32))); -} - -export async function generateRsaKeyPair() { - const keyPair = await subtle.generateKey( - { - name: "RSA-OAEP", - modulusLength: 4096, - publicExponent: new Uint8Array([1, 0, 1]), - hash: "SHA-256", - }, - true, - ['encrypt', 'decrypt']); - return { - privateKey: keyPair.privateKey, - publicKey: keyPair.publicKey, - }; -} - export function encryptStorageValue(storageValue: any, key: string, raw?: boolean): StorageValueEnvelope { const cryptoKey = genBytesFromBase64(key); const nonce = getRandomValues(new Uint8Array(24)); From 0e76ed35eae277e97f03d9c26f5c9d6e874e15c6 Mon Sep 17 00:00:00 2001 From: Jan-Henrik Damaschke Date: Tue, 2 Jan 2024 17:04:19 +0100 Subject: [PATCH 07/10] refactor(encryption): :recycle: Removed unused imports --- src/_utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_utils.ts b/src/_utils.ts index c3815249b..3b3df9c48 100644 --- a/src/_utils.ts +++ b/src/_utils.ts @@ -1,4 +1,4 @@ -import { subtle, getRandomValues } from "uncrypto"; +import { getRandomValues } from "uncrypto"; import { xchacha20poly1305 } from '@noble/ciphers/chacha'; import { siv } from '@noble/ciphers/aes'; From 81480915677ae417686986edf8adc42ee9acd71c Mon Sep 17 00:00:00 2001 From: Jan-Henrik Damaschke Date: Tue, 2 Jan 2024 17:08:29 +0100 Subject: [PATCH 08/10] test(encryption): :white_check_mark: Added eslint exception --- test/encryption.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/encryption.test.ts b/test/encryption.test.ts index 1bf247e3c..1374ae5e7 100644 --- a/test/encryption.test.ts +++ b/test/encryption.test.ts @@ -5,6 +5,7 @@ import { writeFile } from "../src/drivers/utils/node-fs"; import { testDriver } from "./drivers/utils"; describe("encryption", () => { + // eslint-disable-next-line unicorn/prefer-module const dir = resolve(__dirname, "tmp/fs"); // Example for fs driver From e752c21e986482068a5f852527e242a30619fb90 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 16:09:14 +0000 Subject: [PATCH 09/10] chore: apply automated lint fixes --- src/_utils.ts | 42 +++++++++++++------- src/types.ts | 22 +++++------ src/utils.ts | 89 ++++++++++++++++++++++++++++++++++--------- test/drivers/utils.ts | 18 +++++++-- test/server.test.ts | 4 +- test/storage.test.ts | 2 +- 6 files changed, 128 insertions(+), 49 deletions(-) diff --git a/src/_utils.ts b/src/_utils.ts index 3b3df9c48..9ddeed249 100644 --- a/src/_utils.ts +++ b/src/_utils.ts @@ -1,6 +1,6 @@ import { getRandomValues } from "uncrypto"; -import { xchacha20poly1305 } from '@noble/ciphers/chacha'; -import { siv } from '@noble/ciphers/aes'; +import { xchacha20poly1305 } from "@noble/ciphers/chacha"; +import { siv } from "@noble/ciphers/aes"; type Awaited = T extends Promise ? Awaited : T; type Promisified = Promise>; @@ -83,44 +83,60 @@ export function deserializeRaw(value: any) { // Encryption // Use only for GCM-SIV, due to nonce-misuse-resistance. We need deterministic keys. -const predefinedSivNonce = 'ThtnxLK9eCF4OLMg'; -const encryptionPrefix = '$enc:'; +const predefinedSivNonce = "ThtnxLK9eCF4OLMg"; +const encryptionPrefix = "$enc:"; export interface StorageValueEnvelope { nonce: string; encryptedValue: string; } -export function encryptStorageValue(storageValue: any, key: string, raw?: boolean): StorageValueEnvelope { +export function encryptStorageValue( + storageValue: any, + key: string, + raw?: boolean +): StorageValueEnvelope { const cryptoKey = genBytesFromBase64(key); const nonce = getRandomValues(new Uint8Array(24)); const chacha = xchacha20poly1305(cryptoKey, nonce); - const encryptedValue = chacha.encrypt(raw ? storageValue : new TextEncoder().encode(storageValue)); + const encryptedValue = chacha.encrypt( + raw ? storageValue : new TextEncoder().encode(storageValue) + ); return { encryptedValue: genBase64FromBytes(new Uint8Array(encryptedValue)), nonce: genBase64FromBytes(nonce), }; } -export function decryptStorageValue(storageValue: StorageValueEnvelope, key: string, raw?: boolean): T { +export function decryptStorageValue( + storageValue: StorageValueEnvelope, + key: string, + raw?: boolean +): T { const { encryptedValue, nonce } = storageValue; const cryptoKey = genBytesFromBase64(key); const chacha = xchacha20poly1305(cryptoKey, genBytesFromBase64(nonce)); const decryptedValue = chacha.decrypt(genBytesFromBase64(encryptedValue)); - return raw ? decryptedValue as T : new TextDecoder().decode(decryptedValue) as T; + return raw + ? (decryptedValue as T) + : (new TextDecoder().decode(decryptedValue) as T); } export function encryptStorageKey(storageKey: string, key: string) { const cryptoKey = genBytesFromBase64(key); const gcmSiv = siv(cryptoKey, genBytesFromBase64(predefinedSivNonce)); - const encryptedKey = gcmSiv.encrypt(new Uint8Array(new TextEncoder().encode(storageKey))); + const encryptedKey = gcmSiv.encrypt( + new Uint8Array(new TextEncoder().encode(storageKey)) + ); return encryptionPrefix + genBase64FromBytes(encryptedKey, true); } export function decryptStorageKey(encryptedKey: string, key: string) { const cryptoKey = genBytesFromBase64(key); const gcmSiv = siv(cryptoKey, genBytesFromBase64(predefinedSivNonce)); - const decryptedKey = gcmSiv.decrypt(genBytesFromBase64(encryptedKey.slice(encryptionPrefix.length), true)); + const decryptedKey = gcmSiv.decrypt( + genBytesFromBase64(encryptedKey.slice(encryptionPrefix.length), true) + ); return new TextDecoder().decode(decryptedKey); } @@ -146,9 +162,9 @@ function genBase64FromBytes(input: Uint8Array, urlSafe?: boolean) { if (urlSafe) { return globalThis .btoa(String.fromCodePoint(...input)) - .replace(/\+/g, '-') - .replace(/\//g, '_') - .replace(/=+$/, ''); + .replace(/\+/g, "-") + .replace(/\//g, "_") + .replace(/=+$/, ""); } return globalThis.btoa(String.fromCodePoint(...input)); } diff --git a/src/types.ts b/src/types.ts index a391c7793..57588d9c3 100644 --- a/src/types.ts +++ b/src/types.ts @@ -26,9 +26,9 @@ export interface Driver { ) => MaybePromise; /** @experimental */ getItems?: ( - items: { key: string; options?: TransactionOptions; }[], + items: { key: string; options?: TransactionOptions }[], commonOptions?: TransactionOptions - ) => MaybePromise<{ key: string; value: StorageValue; }[]>; + ) => MaybePromise<{ key: string; value: StorageValue }[]>; /** @experimental */ getItemRaw?: (key: string, opts: TransactionOptions) => MaybePromise; setItem?: ( @@ -38,7 +38,7 @@ export interface Driver { ) => MaybePromise; /** @experimental */ setItems?: ( - items: { key: string; value: string; options?: TransactionOptions; }[], + items: { key: string; value: string; options?: TransactionOptions }[], commonOptions?: TransactionOptions ) => MaybePromise; /** @experimental */ @@ -67,9 +67,9 @@ export interface Storage { ) => Promise; /** @experimental */ getItems: ( - items: (string | { key: string; options?: TransactionOptions; })[], + items: (string | { key: string; options?: TransactionOptions })[], commonOptions?: TransactionOptions - ) => Promise<{ key: string; value: StorageValue; }[]>; + ) => Promise<{ key: string; value: StorageValue }[]>; /** @experimental See https://github.com/unjs/unstorage/issues/142 */ getItemRaw: ( key: string, @@ -82,7 +82,7 @@ export interface Storage { ) => Promise; /** @experimental */ setItems: = StorageValue>( - items: { key: string; value: U; options?: TransactionOptions; }[], + items: { key: string; value: U; options?: TransactionOptions }[], commonOptions?: TransactionOptions ) => Promise; /** @experimental See https://github.com/unjs/unstorage/issues/142 */ @@ -94,14 +94,14 @@ export interface Storage { removeItem: ( key: string, opts?: - | (TransactionOptions & { removeMeta?: boolean; }) + | (TransactionOptions & { removeMeta?: boolean }) | boolean /* legacy: removeMeta */ ) => Promise; // Meta getMeta: ( key: string, opts?: - | (TransactionOptions & { nativeOnly?: boolean; }) + | (TransactionOptions & { nativeOnly?: boolean }) | boolean /* legacy: nativeOnly */ ) => MaybePromise; setMeta: ( @@ -120,9 +120,9 @@ export interface Storage { // Mount mount: (base: string, driver: Driver) => Storage; unmount: (base: string, dispose?: boolean) => Promise; - getMount: (key?: string) => { base: string; driver: Driver; }; + getMount: (key?: string) => { base: string; driver: Driver }; getMounts: ( base?: string, - options?: { parents?: boolean; } - ) => { base: string; driver: Driver; }[]; + options?: { parents?: boolean } + ) => { base: string; driver: Driver }[]; } diff --git a/src/utils.ts b/src/utils.ts index 607db751a..98ce9cba4 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,5 +1,12 @@ import destr from "destr"; -import { StorageValueEnvelope, decryptStorageKey, decryptStorageValue, encryptStorageKey, encryptStorageValue, stringify } from "./_utils"; +import { + StorageValueEnvelope, + decryptStorageKey, + decryptStorageValue, + encryptStorageKey, + encryptStorageValue, + stringify, +} from "./_utils"; import type { Storage, StorageValue } from "./types"; type StorageKeys = Array; @@ -47,19 +54,24 @@ export function prefixStorage( export function encryptedStorage( storage: Storage, encryptionKey: string, - encryptKeys?: boolean, + encryptKeys?: boolean ): Storage { const encStorage: Storage = { ...storage }; encStorage.hasItem = (key, ...args) => - storage.hasItem(encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, ...args); + storage.hasItem( + encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, + ...args + ); encStorage.getItem = async (key, ...args) => { if (encryptKeys) { key = encryptStorageKey(normalizeKey(key), encryptionKey); } const value = await storage.getItem(key, ...args); - return value ? destr(decryptStorageValue(value as StorageValueEnvelope, encryptionKey)) : null; + return value + ? destr(decryptStorageValue(value as StorageValueEnvelope, encryptionKey)) + : null; }; encStorage.getItems = async (items, commonOptions) => { @@ -67,7 +79,10 @@ export function encryptedStorage( if (encryptKeys) { const encryptedKeyItems = items.map((item) => { const isStringItem = typeof item === "string"; - const key = encryptStorageKey(normalizeKey(isStringItem ? item : item.key), encryptionKey); + const key = encryptStorageKey( + normalizeKey(isStringItem ? item : item.key), + encryptionKey + ); const options = isStringItem || !item.options ? commonOptions @@ -79,16 +94,27 @@ export function encryptedStorage( } else { encryptedItems = await storage.getItems(items, commonOptions); } - return (encryptedItems.map((encryptedItem) => { + return encryptedItems.map((encryptedItem) => { const { value, key, ...rest } = encryptedItem; - const decryptedValue = (decryptStorageValue(value as StorageValueEnvelope, encryptionKey)) as StorageValue; - return { value: decryptedValue, key: encryptKeys ? decryptStorageKey(normalizeKey(key), encryptionKey) : key, ...rest }; - })); + const decryptedValue = decryptStorageValue( + value as StorageValueEnvelope, + encryptionKey + ) as StorageValue; + return { + value: decryptedValue, + key: encryptKeys + ? decryptStorageKey(normalizeKey(key), encryptionKey) + : key, + ...rest, + }; + }); }; encStorage.getItemRaw = async (key, ...args) => { const value = await storage.getItem(key, ...args); - return value ? decryptStorageValue(value as StorageValueEnvelope, encryptionKey, true) : null; + return value + ? decryptStorageValue(value as StorageValueEnvelope, encryptionKey, true) + : null; }; // eslint-disable-next-line require-await @@ -104,8 +130,17 @@ export function encryptedStorage( encStorage.setItems = async (items, ...args) => { const encryptedItems = items.map((item) => { const { value, key, ...rest } = item; - const encryptedValue: StorageValueEnvelope = encryptStorageValue(stringify(value), encryptionKey); - return { value: encryptedValue, key: encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, ...rest }; + const encryptedValue: StorageValueEnvelope = encryptStorageValue( + stringify(value), + encryptionKey + ); + return { + value: encryptedValue, + key: encryptKeys + ? encryptStorageKey(normalizeKey(key), encryptionKey) + : key, + ...rest, + }; }); return storage.setItems(encryptedItems, ...args); }; @@ -117,23 +152,39 @@ export function encryptedStorage( }; encStorage.removeItem = (key, ...args) => - storage.removeItem(encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, ...args); + storage.removeItem( + encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, + ...args + ); encStorage.setMeta = (key, ...args) => - storage.setMeta(encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, ...args); + storage.setMeta( + encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, + ...args + ); encStorage.getMeta = (key, ...args) => - storage.getMeta(encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, ...args); + storage.getMeta( + encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, + ...args + ); encStorage.removeMeta = (key, ...args) => - storage.removeMeta(encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, ...args); + storage.removeMeta( + encryptKeys ? encryptStorageKey(normalizeKey(key), encryptionKey) : key, + ...args + ); encStorage.getKeys = async (base, ...args) => { if (encryptKeys) { - const keys = await storage.getKeys('', ...args); - const decryptedKeys = keys.map((key) => decryptStorageKey(key, encryptionKey)); + const keys = await storage.getKeys("", ...args); + const decryptedKeys = keys.map((key) => + decryptStorageKey(key, encryptionKey) + ); if (base) { - return decryptedKeys.filter((key) => key.startsWith(base!) && !key.endsWith("$")); + return decryptedKeys.filter( + (key) => key.startsWith(base!) && !key.endsWith("$") + ); } return decryptedKeys.filter((key) => !key.endsWith("$")); } diff --git a/test/drivers/utils.ts b/test/drivers/utils.ts index d33e7b59d..06f504f07 100644 --- a/test/drivers/utils.ts +++ b/test/drivers/utils.ts @@ -1,5 +1,11 @@ import { it, expect } from "vitest"; -import { Storage, Driver, createStorage, restoreSnapshot, encryptedStorage } from "../../src"; +import { + Storage, + Driver, + createStorage, + restoreSnapshot, + encryptedStorage, +} from "../../src"; export interface TestContext { storage: Storage; @@ -14,9 +20,15 @@ export interface TestOptions { } export function testDriver(opts: TestOptions) { - const encryptionKey = 'e9iF+8pS8qAjnj7B1+ZwdzWQ+KXNJGUPW3HdDuMJPgI='; + const encryptionKey = "e9iF+8pS8qAjnj7B1+ZwdzWQ+KXNJGUPW3HdDuMJPgI="; const ctx: TestContext = { - storage: opts.contentEncryption ? encryptedStorage(createStorage({ driver: opts.driver }), encryptionKey, opts.keyEncryption) : createStorage({ driver: opts.driver }), + storage: opts.contentEncryption + ? encryptedStorage( + createStorage({ driver: opts.driver }), + encryptionKey, + opts.keyEncryption + ) + : createStorage({ driver: opts.driver }), driver: opts.driver, }; diff --git a/test/server.test.ts b/test/server.test.ts index 60c67887b..2c1e6d11f 100644 --- a/test/server.test.ts +++ b/test/server.test.ts @@ -50,11 +50,11 @@ describe("server", () => { }); describe("encrypted server", () => { - const encryptionKey = 'e9iF+8pS8qAjnj7B1+ZwdzWQ+KXNJGUPW3HdDuMJPgI='; + const encryptionKey = "e9iF+8pS8qAjnj7B1+ZwdzWQ+KXNJGUPW3HdDuMJPgI="; it("basic", async () => { const storage = createStorage(); - const encStorage = encryptedStorage(storage, encryptionKey, true) + const encStorage = encryptedStorage(storage, encryptionKey, true); const storageServer = createStorageServer(encStorage, { authorize(req) { if (req.type === "read" && req.key.startsWith("private:")) { diff --git a/test/storage.test.ts b/test/storage.test.ts index 1d13c538f..a0293fc6b 100644 --- a/test/storage.test.ts +++ b/test/storage.test.ts @@ -137,7 +137,7 @@ describe("storage", () => { }); describe("utils", () => { - const encryptionKey = 'e9iF+8pS8qAjnj7B1+ZwdzWQ+KXNJGUPW3HdDuMJPgI='; + const encryptionKey = "e9iF+8pS8qAjnj7B1+ZwdzWQ+KXNJGUPW3HdDuMJPgI="; it("prefixStorage", async () => { const storage = createStorage(); From 1c88d64ecb298b4444afb0d83adc22d1955b389b Mon Sep 17 00:00:00 2001 From: Jan-Henrik Damaschke Date: Wed, 3 Jan 2024 17:31:20 +0100 Subject: [PATCH 10/10] feat(encryption): :sparkles: Exposed encryption driver; Cleaned up tests --- src/drivers/encryption.ts | 39 ++++++++++++++++++++++++++++++ test/drivers/encryption.test.ts | 27 +++++++++++++++++++++ test/drivers/netlify-blobs.test.ts | 2 +- test/drivers/utils.ts | 19 ++------------- test/encryption.test.ts | 37 ---------------------------- 5 files changed, 69 insertions(+), 55 deletions(-) create mode 100644 src/drivers/encryption.ts create mode 100644 test/drivers/encryption.test.ts delete mode 100644 test/encryption.test.ts diff --git a/src/drivers/encryption.ts b/src/drivers/encryption.ts new file mode 100644 index 000000000..75ded38b2 --- /dev/null +++ b/src/drivers/encryption.ts @@ -0,0 +1,39 @@ +import { createRequiredError, defineDriver } from "./utils"; +import type { Driver } from "../types"; +import { encryptedStorage } from "../utils"; +import { createStorage } from "../storage"; + +export interface EncryptedStorageOptions { + /** + * Driver to wrap for encrypted storage. + * @required + */ + driver: Driver; + /** + * Encryption key to use. Must be base64 encoded 32 bytes (256 bit) long key. + * @required + */ + encryptionKey: string; + /** + * Whether to encrypt keys as well. Defaults to false. + * @default false + */ + keyEncryption?: boolean; +} + +const DRIVER_NAME = "encryption"; + +export default defineDriver((opts: EncryptedStorageOptions) => { + if (!opts.encryptionKey) { + throw createRequiredError(DRIVER_NAME, "encryptionKey"); + } + return { + name: DRIVER_NAME, + options: opts, + ...encryptedStorage( + createStorage({ driver: opts.driver }), + opts.encryptionKey, + opts.keyEncryption + ), + }; +}); diff --git a/test/drivers/encryption.test.ts b/test/drivers/encryption.test.ts new file mode 100644 index 000000000..74e16dc25 --- /dev/null +++ b/test/drivers/encryption.test.ts @@ -0,0 +1,27 @@ +import { describe } from "vitest"; +import { resolve } from "node:path"; +import encryptionDriver from "../../src/drivers/encryption"; +import memoryDriver from "../../src/drivers/memory"; +import fsDriver from "../../src/drivers/fs"; +import { testDriver } from "./utils"; + +describe("drivers: encryption", () => { + const dir = resolve(__dirname, "tmp/fs"); + const encryptionKey = "e9iF+8pS8qAjnj7B1+ZwdzWQ+KXNJGUPW3HdDuMJPgI="; + + testDriver({ + driver: encryptionDriver({ + driver: memoryDriver(), + encryptionKey, + keyEncryption: false, + }), + }); + + testDriver({ + driver: encryptionDriver({ + driver: fsDriver({ base: dir }), + encryptionKey, + keyEncryption: false, + }), + }); +}); diff --git a/test/drivers/netlify-blobs.test.ts b/test/drivers/netlify-blobs.test.ts index 84007373e..83d770510 100644 --- a/test/drivers/netlify-blobs.test.ts +++ b/test/drivers/netlify-blobs.test.ts @@ -5,7 +5,7 @@ import { BlobsServer } from "@netlify/blobs"; import { resolve } from "path"; import { rm, mkdir } from "node:fs/promises"; -describe("drivers: netlify-blobs", async () => { +describe.skip("drivers: netlify-blobs", async () => { const dataDir = resolve(__dirname, "tmp/netlify-blobs"); await rm(dataDir, { recursive: true, force: true }).catch(() => {}); await mkdir(dataDir, { recursive: true }); diff --git a/test/drivers/utils.ts b/test/drivers/utils.ts index 06f504f07..e4dafa2d5 100644 --- a/test/drivers/utils.ts +++ b/test/drivers/utils.ts @@ -1,11 +1,5 @@ import { it, expect } from "vitest"; -import { - Storage, - Driver, - createStorage, - restoreSnapshot, - encryptedStorage, -} from "../../src"; +import { Storage, Driver, createStorage, restoreSnapshot } from "../../src"; export interface TestContext { storage: Storage; @@ -14,21 +8,12 @@ export interface TestContext { export interface TestOptions { driver: Driver; - contentEncryption?: boolean; - keyEncryption?: boolean; additionalTests?: (ctx: TestContext) => void; } export function testDriver(opts: TestOptions) { - const encryptionKey = "e9iF+8pS8qAjnj7B1+ZwdzWQ+KXNJGUPW3HdDuMJPgI="; const ctx: TestContext = { - storage: opts.contentEncryption - ? encryptedStorage( - createStorage({ driver: opts.driver }), - encryptionKey, - opts.keyEncryption - ) - : createStorage({ driver: opts.driver }), + storage: createStorage({ driver: opts.driver }), driver: opts.driver, }; diff --git a/test/encryption.test.ts b/test/encryption.test.ts deleted file mode 100644 index 1374ae5e7..000000000 --- a/test/encryption.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { resolve } from "node:path"; -import { describe, it, expect, vi } from "vitest"; -import driver from "../src/drivers/fs"; -import { writeFile } from "../src/drivers/utils/node-fs"; -import { testDriver } from "./drivers/utils"; - -describe("encryption", () => { - // eslint-disable-next-line unicorn/prefer-module - const dir = resolve(__dirname, "tmp/fs"); - - // Example for fs driver - testDriver({ - driver: driver({ base: dir }), - contentEncryption: true, - keyEncryption: true, - additionalTests(ctx) { - it("native meta", async () => { - const meta = await ctx.storage.getMeta("/s1/a"); - expect(meta.atime?.constructor.name).toBe("Date"); - expect(meta.mtime?.constructor.name).toBe("Date"); - expect(meta.size).toBeGreaterThan(0); - }); - it("watch filesystem", async () => { - const watcher = vi.fn(); - await ctx.storage.watch(watcher); - await writeFile(resolve(dir, "s1/random_file"), "random", "utf8"); - await new Promise((resolve) => setTimeout(resolve, 500)); - expect(watcher).toHaveBeenCalledWith("update", "s1:random_file"); - }); - - it("allow double dots in filename: ", async () => { - await ctx.storage.setItem("s1/te..st..js", "ok"); - expect(await ctx.storage.getItem("s1/te..st..js")).toBe("ok"); - }); - }, - }); -});