From 3a1491e7a13df85e126f5f503db65b16bc4abb7a Mon Sep 17 00:00:00 2001 From: kricsleo Date: Fri, 11 Apr 2025 22:03:15 +0800 Subject: [PATCH 1/2] feat: add optional error skipping for `getKeys()` --- docs/1.guide/1.index.md | 8 ++++++++ src/storage.ts | 32 ++++++++++++++++++++++++++++++-- src/types.ts | 27 +++++++++++++++++++++++++-- src/utils.ts | 11 ++++++++++- test/storage.test.ts | 23 +++++++++++++++++++++++ 5 files changed, 96 insertions(+), 5 deletions(-) diff --git a/docs/1.guide/1.index.md b/docs/1.guide/1.index.md index 2ea792524..d7ca8994b 100644 --- a/docs/1.guide/1.index.md +++ b/docs/1.guide/1.index.md @@ -182,6 +182,14 @@ You can also use the `keys` alias: await storage.keys(); ``` +By default, key retrieval errors halt the process. To ignore errors and fetch available keys, enable `try: true` in the options. +Failed retrievals will include error details in the result array's `errors` property. + +```js +const keys = await storage.getKeys("base", { try: true }); +// Errors are accessible via `keys.errors` +``` + ### `clear(base?, opts?)` Removes all stored key/values. If a base is provided, only mounts matching base will be cleared. diff --git a/src/storage.ts b/src/storage.ts index eb6578471..d61420b72 100644 --- a/src/storage.ts +++ b/src/storage.ts @@ -7,6 +7,7 @@ import type { StorageValue, WatchEvent, TransactionOptions, + StorageError, } from "./types"; import memory from "./drivers/memory"; import { asyncCall, deserializeRaw, serializeRaw, stringify } from "./_utils"; @@ -348,6 +349,7 @@ export function createStorage( const mounts = getMounts(base, true); let maskedMounts: string[] = []; const allKeys = []; + const errors: StorageError[] = []; let allMountsSupportMaxDepth = true; for (const mount of mounts) { if (!mount.driver.flags?.maxDepth) { @@ -357,7 +359,25 @@ export function createStorage( mount.driver.getKeys, mount.relativeBase, opts - ); + ).catch((error) => { + if (!opts.try) { + throw error; + } + + console.warn( + `[unstorage]: Failed to get keys for "${mount.mountpoint}":`, + error + ); + errors.push({ + error, + mount: { + base: mount.mountpoint, + driver: mount.driver, + }, + }); + return []; + }); + for (const key of rawKeys) { const fullKey = mount.mountpoint + normalizeKey(key); if (!maskedMounts.some((p) => fullKey.startsWith(p))) { @@ -374,11 +394,19 @@ export function createStorage( } const shouldFilterByDepth = opts.maxDepth !== undefined && !allMountsSupportMaxDepth; - return allKeys.filter( + const keys = allKeys.filter( (key) => (!shouldFilterByDepth || filterKeyByDepth(key, opts.maxDepth)) && filterKeyByBase(key, base) ); + if (opts.try) { + Object.defineProperty(keys, "errors", { + enumerable: false, + value: errors, + }); + } + + return keys; }, // Utils async clear(base, opts = {}) { diff --git a/src/types.ts b/src/types.ts index d347dbd06..972d85a87 100644 --- a/src/types.ts +++ b/src/types.ts @@ -20,6 +20,23 @@ export type TransactionOptions = Record; export type GetKeysOptions = TransactionOptions & { maxDepth?: number; + /** + * Whether to ignore errors during key retrieval and return available keys. + * @default false + */ + try?: boolean; +}; + +export interface StorageError { + error: unknown; + mount: { + base: string; + driver: Driver; + }; +} + +export type WithErrors = T & { + errors?: StorageError[]; }; export interface DriverFlags { @@ -65,7 +82,10 @@ export interface Driver { key: string, opts: TransactionOptions ) => MaybePromise; - getKeys: (base: string, opts: GetKeysOptions) => MaybePromise; + getKeys: ( + base: string, + opts: GetKeysOptions + ) => MaybePromise>; clear?: (base: string, opts: TransactionOptions) => MaybePromise; dispose?: () => MaybePromise; watch?: (callback: WatchCallback) => MaybePromise; @@ -173,7 +193,10 @@ export interface Storage { ) => Promise; removeMeta: (key: string, opts?: TransactionOptions) => Promise; // Keys - getKeys: (base?: string, opts?: GetKeysOptions) => Promise; + getKeys: ( + base?: string, + opts?: GetKeysOptions + ) => Promise>; // Utils clear: (base?: string, opts?: TransactionOptions) => Promise; dispose: () => Promise; diff --git a/src/utils.ts b/src/utils.ts index 98af01c0c..9073b4e7a 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -42,7 +42,16 @@ export function prefixStorage( storage .getKeys(base + key, ...arguments_) // Remove Prefix - .then((keys) => keys.map((key) => key.slice(base.length))); + .then((_keys) => { + const keys = _keys.map((key) => key.slice(base.length)); + if (_keys.errors) { + Object.defineProperty(keys, "errors", { + enumerable: false, + value: _keys.errors, + }); + } + return keys; + }); nsStorage.getItems = async ( items: (string | { key: string; options?: TransactionOptions })[], diff --git a/test/storage.test.ts b/test/storage.test.ts index d5a2542d7..f24ee0d11 100644 --- a/test/storage.test.ts +++ b/test/storage.test.ts @@ -7,6 +7,7 @@ import { prefixStorage, } from "../src"; import memory from "../src/drivers/memory"; +import redisDriver from "../src/drivers/redis"; import fs from "../src/drivers/fs"; const data = { @@ -250,6 +251,28 @@ describe("Regression", () => { } }); + it.only("getKeys - ignore errors and fetch available keys ", async () => { + const storage = createStorage(); + const invalidDriver = redisDriver({ + url: "ioredis://localhost:9999/0", + connectTimeout: 10, + retryStrategy: () => null, + }); + storage.mount("/invalid", invalidDriver); + + await storage.setItem("foo", "bar"); + + await expect(storage.getKeys()).rejects.toThrowError( + "Connection is closed" + ); + + const keys = await storage.getKeys(undefined, { try: true }); + expect(keys).toMatchObject(["foo"]); + expect(keys.errors![0]!.error).toMatchInlineSnapshot( + `[Error: Connection is closed.]` + ); + }); + it("prefixStorage getItems to not returns null (issue #396)", async () => { const storage = createStorage(); await storage.setItem("namespace:key", "value"); From 4021926dccbc15c2aed13c077042e26001853245 Mon Sep 17 00:00:00 2001 From: kricsleo Date: Fri, 11 Apr 2025 22:23:40 +0800 Subject: [PATCH 2/2] test: remove `.only` --- test/storage.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/storage.test.ts b/test/storage.test.ts index f24ee0d11..46810738f 100644 --- a/test/storage.test.ts +++ b/test/storage.test.ts @@ -251,7 +251,7 @@ describe("Regression", () => { } }); - it.only("getKeys - ignore errors and fetch available keys ", async () => { + it("getKeys - ignore errors and fetch available keys ", async () => { const storage = createStorage(); const invalidDriver = redisDriver({ url: "ioredis://localhost:9999/0",