From f47427e243c144e385bc27119c03cf971d9ff307 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Lucas?= Date: Mon, 7 Apr 2025 00:35:33 -0300 Subject: [PATCH 1/8] feat(storage): enhance get() method to support multiple data types --- src/storage.ts | 40 +++++++++++++++++++++++++++++++++++++++- src/types.ts | 34 +++++++++++++++++++++++++++++++++- test/storage.test.ts | 25 +++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 2 deletions(-) diff --git a/src/storage.ts b/src/storage.ts index eb6578471..e9ba31b90 100644 --- a/src/storage.ts +++ b/src/storage.ts @@ -473,7 +473,45 @@ export function createStorage( }, // Aliases keys: (base, opts = {}) => storage.getKeys(base, opts), - get: (key: string, opts = {}) => storage.getItem(key, opts), + get: async ( + key: string, + opts: TransactionOptions & { + type?: "json" | "text" | "bytes" | "stream" | "blob"; + } = {} + ) => { + key = normalizeKey(key); + const { relativeKey, driver } = getMount(key); + + const type = opts.type; + + if (type === "bytes" || type === "stream" || type === "blob") { + if (driver.getItemRaw) { + return asyncCall(driver.getItemRaw, relativeKey, opts); + } + const raw = await asyncCall(driver.getItem, relativeKey, opts); + return deserializeRaw(raw); + } + + const value = await asyncCall(driver.getItem, relativeKey, opts); + + if (value == null) { + return null; + } + + if (type === "text") { + return typeof value === "string" ? value : String(value); + } + + if (type === "json") { + if (typeof value === "string") { + return JSON.parse(value); + } + return value; + } + + // default behavior: try destr (safe JSON parse fallback to text) + return destr(value); + }, set: (key: string, value: T, opts = {}) => storage.setItem(key, value, opts), has: (key: string, opts = {}) => storage.hasItem(key, opts), diff --git a/src/types.ts b/src/types.ts index d347dbd06..0e28265ed 100644 --- a/src/types.ts +++ b/src/types.ts @@ -189,7 +189,39 @@ export interface Storage { ) => { base: string; driver: Driver }[]; // Aliases keys: Storage["getKeys"]; - get: Storage["getItem"]; + get< + U extends Extract, + K extends keyof StorageItemMap, + >( + key: K, + opts?: TransactionOptions & { type?: undefined } + ): Promise | null>; + + get( + key: string, + opts: { type: "text" } & TransactionOptions + ): Promise; + get( + key: string, + opts: { type: "json" } & TransactionOptions + ): Promise; + get( + key: string, + opts: { type: "bytes" } & TransactionOptions + ): Promise; + get( + key: string, + opts: { type: "stream" } & TransactionOptions + ): Promise; + get( + key: string, + opts: { type: "blob" } & TransactionOptions + ): Promise; + get( + key: string, + opts?: TransactionOptions & { type?: undefined } + ): Promise; + set: Storage["setItem"]; has: Storage["hasItem"]; del: Storage["removeItem"]; diff --git a/test/storage.test.ts b/test/storage.test.ts index 3857ce27f..701a689b8 100644 --- a/test/storage.test.ts +++ b/test/storage.test.ts @@ -250,3 +250,28 @@ describe("Regression", () => { } }); }); + +describe("get() with type option", () => { + const storage = createStorage(); + + it("should get JSON object with type=json", async () => { + const obj = { foo: "bar", num: 42 }; + await storage.setItem("json-key", obj); + const result = await storage.get("json-key", { type: "json" }); + expect(result).toEqual(obj); + }); + + it("should get string with type=text", async () => { + await storage.setItem("text-key", "hello world"); + const result = await storage.get("text-key", { type: "text" }); + expect(result).toBe("hello world"); + }); + + it("should get bytes with type=bytes", async () => { + const bytes = new Uint8Array([1, 2, 3, 4]); + await storage.setItemRaw("bytes-key", bytes); + const result = await storage.get("bytes-key", { type: "bytes" }); + const len = result?.length || result?.byteLength; + expect(len).toBe(4); + }); +}); From 3a362772ce140d855c24763b4c28403330521cda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Lucas?= Date: Mon, 7 Apr 2025 13:34:40 -0300 Subject: [PATCH 2/8] chore: pr feedback, move type from get to getItem --- src/storage.ts | 81 +++++++++++++++++++++++--------------------------- src/types.ts | 65 +++++++++++++++++++--------------------- 2 files changed, 68 insertions(+), 78 deletions(-) diff --git a/src/storage.ts b/src/storage.ts index e9ba31b90..243e26f75 100644 --- a/src/storage.ts +++ b/src/storage.ts @@ -172,12 +172,44 @@ export function createStorage( const { relativeKey, driver } = getMount(key); return asyncCall(driver.hasItem, relativeKey, opts); }, - getItem(key: string, opts = {}) { + getItem: async ( + key: string, + opts: TransactionOptions & { + type?: "json" | "text" | "bytes" | "stream" | "blob"; + } = {} + ) => { key = normalizeKey(key); const { relativeKey, driver } = getMount(key); - return asyncCall(driver.getItem, relativeKey, opts).then( - (value) => destr(value) as StorageValue - ); + + const type = opts.type; + + if (type === "bytes" || type === "stream" || type === "blob") { + if (driver.getItemRaw) { + return asyncCall(driver.getItemRaw, relativeKey, opts); + } + const raw = await asyncCall(driver.getItem, relativeKey, opts); + return deserializeRaw(raw); + } + + const value = await asyncCall(driver.getItem, relativeKey, opts); + + if (value == null) { + return null; + } + + if (type === "text") { + return typeof value === "string" ? value : String(value); + } + + if (type === "json") { + if (typeof value === "string") { + return JSON.parse(value); + } + return value; + } + + // default behavior: try destr (safe JSON parse fallback to text) + return destr(value); }, getItems( items: (string | { key: string; options?: TransactionOptions })[], @@ -473,45 +505,8 @@ export function createStorage( }, // Aliases keys: (base, opts = {}) => storage.getKeys(base, opts), - get: async ( - key: string, - opts: TransactionOptions & { - type?: "json" | "text" | "bytes" | "stream" | "blob"; - } = {} - ) => { - key = normalizeKey(key); - const { relativeKey, driver } = getMount(key); - - const type = opts.type; - - if (type === "bytes" || type === "stream" || type === "blob") { - if (driver.getItemRaw) { - return asyncCall(driver.getItemRaw, relativeKey, opts); - } - const raw = await asyncCall(driver.getItem, relativeKey, opts); - return deserializeRaw(raw); - } - - const value = await asyncCall(driver.getItem, relativeKey, opts); - - if (value == null) { - return null; - } - - if (type === "text") { - return typeof value === "string" ? value : String(value); - } - - if (type === "json") { - if (typeof value === "string") { - return JSON.parse(value); - } - return value; - } - - // default behavior: try destr (safe JSON parse fallback to text) - return destr(value); - }, + get: ((key: string, opts: TransactionOptions = {}) => + storage.getItem(key, opts)) as Storage["get"], set: (key: string, value: T, opts = {}) => storage.setItem(key, value, opts), has: (key: string, opts = {}) => storage.hasItem(key, opts), diff --git a/src/types.ts b/src/types.ts index 0e28265ed..2fdaa8bed 100644 --- a/src/types.ts +++ b/src/types.ts @@ -96,14 +96,40 @@ export interface Storage { getItem< U extends Extract, - K extends string & keyof StorageItemMap, + K extends keyof StorageItemMap, >( key: K, - ops?: TransactionOptions + opts?: TransactionOptions & { type?: undefined } ): Promise | null>; + + getItem( + key: string, + opts: { type: "text" } & TransactionOptions + ): Promise; + + getItem( + key: string, + opts: { type: "json" } & TransactionOptions + ): Promise; + + getItem( + key: string, + opts: { type: "bytes" } & TransactionOptions + ): Promise; + + getItem( + key: string, + opts: { type: "stream" } & TransactionOptions + ): Promise; + + getItem( + key: string, + opts: { type: "blob" } & TransactionOptions + ): Promise; + getItem>( key: string, - opts?: TransactionOptions + opts?: TransactionOptions & { type?: undefined } ): Promise; /** @experimental */ @@ -189,38 +215,7 @@ export interface Storage { ) => { base: string; driver: Driver }[]; // Aliases keys: Storage["getKeys"]; - get< - U extends Extract, - K extends keyof StorageItemMap, - >( - key: K, - opts?: TransactionOptions & { type?: undefined } - ): Promise | null>; - - get( - key: string, - opts: { type: "text" } & TransactionOptions - ): Promise; - get( - key: string, - opts: { type: "json" } & TransactionOptions - ): Promise; - get( - key: string, - opts: { type: "bytes" } & TransactionOptions - ): Promise; - get( - key: string, - opts: { type: "stream" } & TransactionOptions - ): Promise; - get( - key: string, - opts: { type: "blob" } & TransactionOptions - ): Promise; - get( - key: string, - opts?: TransactionOptions & { type?: undefined } - ): Promise; + get: Storage["getItem"]; set: Storage["setItem"]; has: Storage["hasItem"]; From f12eb950b4950334b5d5855f2bcf3fce2546934e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Lucas?= Date: Thu, 10 Apr 2025 16:30:01 -0300 Subject: [PATCH 3/8] feat(utils): add transformRawToType function for data type conversion --- src/storage.ts | 9 +- src/utils.ts | 82 +++++++++++++ test/transform-raw-to-type.test.ts | 183 +++++++++++++++++++++++++++++ 3 files changed, 271 insertions(+), 3 deletions(-) create mode 100644 test/transform-raw-to-type.test.ts diff --git a/src/storage.ts b/src/storage.ts index 243e26f75..eb232ba3e 100644 --- a/src/storage.ts +++ b/src/storage.ts @@ -16,6 +16,7 @@ import { joinKeys, filterKeyByDepth, filterKeyByBase, + transformRawToType, } from "./utils"; interface StorageCTX { @@ -185,10 +186,12 @@ export function createStorage( if (type === "bytes" || type === "stream" || type === "blob") { if (driver.getItemRaw) { - return asyncCall(driver.getItemRaw, relativeKey, opts); + const raw = await asyncCall(driver.getItemRaw, relativeKey, opts); + return transformRawToType(raw, type); } - const raw = await asyncCall(driver.getItem, relativeKey, opts); - return deserializeRaw(raw); + const rawValue = await asyncCall(driver.getItem, relativeKey, opts); + const raw = deserializeRaw(rawValue); + return transformRawToType(raw, type); } const value = await asyncCall(driver.getItem, relativeKey, opts); diff --git a/src/utils.ts b/src/utils.ts index 98af01c0c..518190109 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -124,3 +124,85 @@ export function filterKeyByBase( return key[key.length - 1] !== "$"; } + +/** + * Transform raw data into the expected type. + */ +export function transformRawToType( + raw: any, + type: "bytes" | "blob" | "stream" +) { + // Handle "bytes" + if (type === "bytes") { + if ( + raw instanceof Uint8Array || + (typeof Buffer !== "undefined" && raw instanceof Buffer) + ) { + return raw; + } + if (typeof raw === "string") { + return new TextEncoder().encode(raw); + } + // Try to convert ArrayBuffer + if (raw instanceof ArrayBuffer) { + return new Uint8Array(raw); + } + throw new Error("[unstorage] Cannot convert raw data to bytes"); + } + + // Handle "blob" + if (type === "blob") { + if (typeof Blob !== "undefined" && raw instanceof Blob) { + return raw; + } + if ( + raw instanceof Uint8Array || + (typeof Buffer !== "undefined" && raw instanceof Buffer) + ) { + return new Blob([raw]); + } + if (typeof raw === "string") { + return new Blob([new TextEncoder().encode(raw)]); + } + if (raw instanceof ArrayBuffer) { + return new Blob([new Uint8Array(raw)]); + } + throw new Error("[unstorage] Cannot convert raw data to Blob"); + } + + // Handle "stream" + if (type === "stream") { + if ( + typeof ReadableStream !== "undefined" && + raw instanceof ReadableStream + ) { + return raw; + } + // Convert Uint8Array, Buffer, string, or ArrayBuffer to stream + let uint8array: Uint8Array; + if ( + raw instanceof Uint8Array || + (typeof Buffer !== "undefined" && raw instanceof Buffer) + ) { + uint8array = raw; + } else if (typeof raw === "string") { + uint8array = new TextEncoder().encode(raw); + } else if (raw instanceof ArrayBuffer) { + uint8array = new Uint8Array(raw); + } else { + throw new TypeError( + "[unstorage] Cannot convert raw data to ReadableStream" + ); + } + + // Create a ReadableStream from Uint8Array + return new ReadableStream({ + start(controller) { + controller.enqueue(uint8array); + controller.close(); + }, + }); + } + + throw new Error("[unstorage] Unknown type for transformRawToType"); +} diff --git a/test/transform-raw-to-type.test.ts b/test/transform-raw-to-type.test.ts new file mode 100644 index 000000000..00ce617e6 --- /dev/null +++ b/test/transform-raw-to-type.test.ts @@ -0,0 +1,183 @@ +import { describe, it, expect } from "vitest"; +import { transformRawToType } from "../src/utils"; + +const isNode = typeof Buffer !== "undefined"; +const hasBlob = typeof Blob !== "undefined"; +const hasReadableStream = typeof ReadableStream !== "undefined"; + +describe("transformRawToType", () => { + describe('"bytes"', () => { + it("returns Uint8Array as-is", () => { + const input = new Uint8Array([1, 2, 3]); + const result = transformRawToType(input, "bytes"); + expect(result).toBe(input); + }); + + it("returns Buffer as-is", () => { + if (!isNode) return; + const input = Buffer.from([1, 2, 3]); + const result = transformRawToType(input, "bytes"); + expect(result).toBe(input); + }); + + it("converts string to Uint8Array", () => { + const input = "hello"; + const result = transformRawToType(input, "bytes"); + expect( + result instanceof Uint8Array || (isNode && result instanceof Buffer) + ).toBe(true); + if ( + result instanceof Uint8Array || + (isNode && result instanceof Buffer) + ) { + expect(new TextDecoder().decode(result)).toBe("hello"); + } + }); + + it("converts ArrayBuffer to Uint8Array", () => { + const input = new Uint8Array([1, 2, 3]).buffer; + const result = transformRawToType(input, "bytes"); + expect( + result instanceof Uint8Array || (isNode && result instanceof Buffer) + ).toBe(true); + if ( + result instanceof Uint8Array || + (isNode && result instanceof Buffer) + ) { + expect([...result]).toEqual([1, 2, 3]); + } + }); + + it('throws when "bytes" input is invalid', () => { + expect(() => transformRawToType(123, "bytes")).toThrow(); + expect(() => transformRawToType({}, "bytes")).toThrow(); + expect(() => transformRawToType(null, "bytes")).toThrow(); + expect(() => transformRawToType(undefined, "bytes")).toThrow(); + }); + + it("throws on unsupported input", () => { + expect(() => transformRawToType(123, "bytes")).toThrow(); + expect(() => transformRawToType({}, "bytes")).toThrow(); + }); + }); + + describe('"blob"', () => { + it("returns Blob as-is", () => { + if (!hasBlob) return; + const input = new Blob(["hello"]); + const result = transformRawToType(input, "blob"); + expect(result).toBe(input); + }); + + it("converts Uint8Array to Blob", () => { + if (!hasBlob) return; + const input = new Uint8Array([1, 2, 3]); + const result = transformRawToType(input, "blob"); + expect(result).toBeInstanceOf(Blob); + }); + + it("converts Buffer to Blob", () => { + if (!hasBlob || !isNode) return; + const input = Buffer.from([1, 2, 3]); + const result = transformRawToType(input, "blob"); + expect(result).toBeInstanceOf(Blob); + }); + + it("converts string to Blob", () => { + if (!hasBlob) return; + const input = "hello"; + const result = transformRawToType(input, "blob"); + expect(result).toBeInstanceOf(Blob); + }); + + it('throws when "blob" input is invalid', () => { + const originalBlob = globalThis.Blob; + // Force Blob to be defined to enter error branches + globalThis.Blob = class MockBlob {} as any; + try { + expect(() => transformRawToType(123, "blob")).toThrow(); + expect(() => transformRawToType({}, "blob")).toThrow(); + expect(() => transformRawToType(null, "blob")).toThrow(); + expect(() => transformRawToType(undefined, "blob")).toThrow(); + } finally { + globalThis.Blob = originalBlob; + } + }); + + it("converts ArrayBuffer to Blob", () => { + if (!hasBlob) return; + const input = new Uint8Array([1, 2, 3]).buffer; + const result = transformRawToType(input, "blob"); + expect(result).toBeInstanceOf(Blob); + }); + + it("throws on unsupported input", () => { + if (!hasBlob) return; + expect(() => transformRawToType(123, "blob")).toThrow(); + expect(() => transformRawToType({}, "blob")).toThrow(); + }); + }); + + describe('"stream"', () => { + it("returns ReadableStream as-is", () => { + if (!hasReadableStream) return; + const input = new ReadableStream({}); + const result = transformRawToType(input, "stream"); + expect(result).toBe(input); + }); + + it("converts Uint8Array to ReadableStream", async () => { + if (!hasReadableStream) return; + const input = new Uint8Array([1, 2, 3]); + const stream = transformRawToType(input, "stream"); + expect(stream).toBeInstanceOf(ReadableStream); + }); + + it("converts Buffer to ReadableStream", async () => { + if (!hasReadableStream || !isNode) return; + const input = Buffer.from([1, 2, 3]); + const stream = transformRawToType(input, "stream"); + expect(stream).toBeInstanceOf(ReadableStream); + }); + + it('throws when "stream" input is invalid', () => { + const originalStream = globalThis.ReadableStream; + // Force ReadableStream to be defined to enter error branches + globalThis.ReadableStream = class MockStream {} as any; + try { + expect(() => transformRawToType(123, "stream")).toThrow(); + expect(() => transformRawToType({}, "stream")).toThrow(); + expect(() => transformRawToType(null, "stream")).toThrow(); + expect(() => transformRawToType(undefined, "stream")).toThrow(); + } finally { + globalThis.ReadableStream = originalStream; + } + }); + + it("converts string to ReadableStream", async () => { + if (!hasReadableStream) return; + const input = "hello"; + const stream = transformRawToType(input, "stream"); + expect(stream).toBeInstanceOf(ReadableStream); + }); + + it("converts ArrayBuffer to ReadableStream", async () => { + if (!hasReadableStream) return; + const input = new Uint8Array([1, 2, 3]).buffer; + const stream = transformRawToType(input, "stream"); + expect(stream).toBeInstanceOf(ReadableStream); + }); + + it("throws on unsupported input", () => { + if (!hasReadableStream) return; + expect(() => transformRawToType(123, "stream")).toThrow(); + expect(() => transformRawToType({}, "stream")).toThrow(); + }); + }); + + describe("unknown type", () => { + it("throws on unknown type", () => { + expect(() => transformRawToType("hello", "unknown" as any)).toThrow(); + }); + }); +}); From ab88439c122b68ffcc5bb643e3951dae9e7a5072 Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Sun, 13 Apr 2025 13:53:38 +0200 Subject: [PATCH 4/8] remove buffer checks --- src/utils.ts | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 518190109..35c75ec2e 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -134,10 +134,7 @@ export function transformRawToType( ) { // Handle "bytes" if (type === "bytes") { - if ( - raw instanceof Uint8Array || - (typeof Buffer !== "undefined" && raw instanceof Buffer) - ) { + if (raw instanceof Uint8Array) { return raw; } if (typeof raw === "string") { @@ -155,10 +152,7 @@ export function transformRawToType( if (typeof Blob !== "undefined" && raw instanceof Blob) { return raw; } - if ( - raw instanceof Uint8Array || - (typeof Buffer !== "undefined" && raw instanceof Buffer) - ) { + if (raw instanceof Uint8Array) { return new Blob([raw]); } if (typeof raw === "string") { @@ -180,10 +174,7 @@ export function transformRawToType( } // Convert Uint8Array, Buffer, string, or ArrayBuffer to stream let uint8array: Uint8Array; - if ( - raw instanceof Uint8Array || - (typeof Buffer !== "undefined" && raw instanceof Buffer) - ) { + if (raw instanceof Uint8Array) { uint8array = raw; } else if (typeof raw === "string") { uint8array = new TextEncoder().encode(raw); From 5b020ecbfbd1010b62710553ae67bffabe720d11 Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Sun, 13 Apr 2025 14:11:10 +0200 Subject: [PATCH 5/8] simplify --- src/storage.ts | 40 ++++---- src/utils.ts | 95 +++++++------------ ...-raw-to-type.test.ts => to-binary.test.ts} | 70 +++++++------- 3 files changed, 89 insertions(+), 116 deletions(-) rename test/{transform-raw-to-type.test.ts => to-binary.test.ts} (66%) diff --git a/src/storage.ts b/src/storage.ts index eb232ba3e..76bc08545 100644 --- a/src/storage.ts +++ b/src/storage.ts @@ -16,7 +16,7 @@ import { joinKeys, filterKeyByDepth, filterKeyByBase, - transformRawToType, + toBinary, } from "./utils"; interface StorageCTX { @@ -185,34 +185,36 @@ export function createStorage( const type = opts.type; if (type === "bytes" || type === "stream" || type === "blob") { - if (driver.getItemRaw) { - const raw = await asyncCall(driver.getItemRaw, relativeKey, opts); - return transformRawToType(raw, type); + const raw = driver.getItemRaw + ? await asyncCall(driver.getItemRaw, relativeKey, opts) + : await asyncCall(driver.getItem, relativeKey, opts).then((val) => + deserializeRaw(val) + ); + if (raw === null || raw === undefined) { + return null; } - const rawValue = await asyncCall(driver.getItem, relativeKey, opts); - const raw = deserializeRaw(rawValue); - return transformRawToType(raw, type); + return toBinary(raw, type); } const value = await asyncCall(driver.getItem, relativeKey, opts); - if (value == null) { return null; } - if (type === "text") { - return typeof value === "string" ? value : String(value); - } - - if (type === "json") { - if (typeof value === "string") { - return JSON.parse(value); + switch (type) { + case "text": { + return typeof value === "string" ? value : String(value); + } + case "json": { + if (typeof value === "string") { + return JSON.parse(value); + } + return value; + } + default: { + return destr(value); } - return value; } - - // default behavior: try destr (safe JSON parse fallback to text) - return destr(value); }, getItems( items: (string | { key: string; options?: TransactionOptions })[], diff --git a/src/utils.ts b/src/utils.ts index 35c75ec2e..c04014baa 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -128,72 +128,43 @@ export function filterKeyByBase( /** * Transform raw data into the expected type. */ -export function transformRawToType( - raw: any, +export function toBinary( + value: unknown, type: "bytes" | "blob" | "stream" -) { - // Handle "bytes" - if (type === "bytes") { - if (raw instanceof Uint8Array) { - return raw; +): Uint8Array | Blob | ReadableStream { + switch (type) { + case "bytes": { + if (value instanceof Uint8Array) { + return value; + } + if (typeof value === "string") { + return new TextEncoder().encode(value); + } + if (value instanceof ArrayBuffer) { + return new Uint8Array(value); + } + throw new Error(`Cannot convert raw data to bytes: ${value}`); } - if (typeof raw === "string") { - return new TextEncoder().encode(raw); + case "blob": { + if (value instanceof Blob) { + return value; + } + return new Blob([toBinary(value, "bytes") as Uint8Array]); } - // Try to convert ArrayBuffer - if (raw instanceof ArrayBuffer) { - return new Uint8Array(raw); + case "stream": { + if (value instanceof ReadableStream) { + return value; + } + const bytes = toBinary(value, "bytes") as Uint8Array; + return new ReadableStream({ + start(controller) { + controller.enqueue([bytes]); + controller.close(); + }, + }); } - throw new Error("[unstorage] Cannot convert raw data to bytes"); - } - - // Handle "blob" - if (type === "blob") { - if (typeof Blob !== "undefined" && raw instanceof Blob) { - return raw; - } - if (raw instanceof Uint8Array) { - return new Blob([raw]); + default: { + throw new Error(`Unsupported type: ${type}`); } - if (typeof raw === "string") { - return new Blob([new TextEncoder().encode(raw)]); - } - if (raw instanceof ArrayBuffer) { - return new Blob([new Uint8Array(raw)]); - } - throw new Error("[unstorage] Cannot convert raw data to Blob"); } - - // Handle "stream" - if (type === "stream") { - if ( - typeof ReadableStream !== "undefined" && - raw instanceof ReadableStream - ) { - return raw; - } - // Convert Uint8Array, Buffer, string, or ArrayBuffer to stream - let uint8array: Uint8Array; - if (raw instanceof Uint8Array) { - uint8array = raw; - } else if (typeof raw === "string") { - uint8array = new TextEncoder().encode(raw); - } else if (raw instanceof ArrayBuffer) { - uint8array = new Uint8Array(raw); - } else { - throw new TypeError( - "[unstorage] Cannot convert raw data to ReadableStream" - ); - } - - // Create a ReadableStream from Uint8Array - return new ReadableStream({ - start(controller) { - controller.enqueue(uint8array); - controller.close(); - }, - }); - } - - throw new Error("[unstorage] Unknown type for transformRawToType"); } diff --git a/test/transform-raw-to-type.test.ts b/test/to-binary.test.ts similarity index 66% rename from test/transform-raw-to-type.test.ts rename to test/to-binary.test.ts index 00ce617e6..f8172c9f4 100644 --- a/test/transform-raw-to-type.test.ts +++ b/test/to-binary.test.ts @@ -1,28 +1,28 @@ import { describe, it, expect } from "vitest"; -import { transformRawToType } from "../src/utils"; +import { toBinary } from "../src/utils"; const isNode = typeof Buffer !== "undefined"; const hasBlob = typeof Blob !== "undefined"; const hasReadableStream = typeof ReadableStream !== "undefined"; -describe("transformRawToType", () => { +describe("toBinary", () => { describe('"bytes"', () => { it("returns Uint8Array as-is", () => { const input = new Uint8Array([1, 2, 3]); - const result = transformRawToType(input, "bytes"); + const result = toBinary(input, "bytes"); expect(result).toBe(input); }); it("returns Buffer as-is", () => { if (!isNode) return; const input = Buffer.from([1, 2, 3]); - const result = transformRawToType(input, "bytes"); + const result = toBinary(input, "bytes"); expect(result).toBe(input); }); it("converts string to Uint8Array", () => { const input = "hello"; - const result = transformRawToType(input, "bytes"); + const result = toBinary(input, "bytes"); expect( result instanceof Uint8Array || (isNode && result instanceof Buffer) ).toBe(true); @@ -36,7 +36,7 @@ describe("transformRawToType", () => { it("converts ArrayBuffer to Uint8Array", () => { const input = new Uint8Array([1, 2, 3]).buffer; - const result = transformRawToType(input, "bytes"); + const result = toBinary(input, "bytes"); expect( result instanceof Uint8Array || (isNode && result instanceof Buffer) ).toBe(true); @@ -49,15 +49,15 @@ describe("transformRawToType", () => { }); it('throws when "bytes" input is invalid', () => { - expect(() => transformRawToType(123, "bytes")).toThrow(); - expect(() => transformRawToType({}, "bytes")).toThrow(); - expect(() => transformRawToType(null, "bytes")).toThrow(); - expect(() => transformRawToType(undefined, "bytes")).toThrow(); + expect(() => toBinary(123, "bytes")).toThrow(); + expect(() => toBinary({}, "bytes")).toThrow(); + expect(() => toBinary(null, "bytes")).toThrow(); + expect(() => toBinary(undefined, "bytes")).toThrow(); }); it("throws on unsupported input", () => { - expect(() => transformRawToType(123, "bytes")).toThrow(); - expect(() => transformRawToType({}, "bytes")).toThrow(); + expect(() => toBinary(123, "bytes")).toThrow(); + expect(() => toBinary({}, "bytes")).toThrow(); }); }); @@ -65,28 +65,28 @@ describe("transformRawToType", () => { it("returns Blob as-is", () => { if (!hasBlob) return; const input = new Blob(["hello"]); - const result = transformRawToType(input, "blob"); + const result = toBinary(input, "blob"); expect(result).toBe(input); }); it("converts Uint8Array to Blob", () => { if (!hasBlob) return; const input = new Uint8Array([1, 2, 3]); - const result = transformRawToType(input, "blob"); + const result = toBinary(input, "blob"); expect(result).toBeInstanceOf(Blob); }); it("converts Buffer to Blob", () => { if (!hasBlob || !isNode) return; const input = Buffer.from([1, 2, 3]); - const result = transformRawToType(input, "blob"); + const result = toBinary(input, "blob"); expect(result).toBeInstanceOf(Blob); }); it("converts string to Blob", () => { if (!hasBlob) return; const input = "hello"; - const result = transformRawToType(input, "blob"); + const result = toBinary(input, "blob"); expect(result).toBeInstanceOf(Blob); }); @@ -95,10 +95,10 @@ describe("transformRawToType", () => { // Force Blob to be defined to enter error branches globalThis.Blob = class MockBlob {} as any; try { - expect(() => transformRawToType(123, "blob")).toThrow(); - expect(() => transformRawToType({}, "blob")).toThrow(); - expect(() => transformRawToType(null, "blob")).toThrow(); - expect(() => transformRawToType(undefined, "blob")).toThrow(); + expect(() => toBinary(123, "blob")).toThrow(); + expect(() => toBinary({}, "blob")).toThrow(); + expect(() => toBinary(null, "blob")).toThrow(); + expect(() => toBinary(undefined, "blob")).toThrow(); } finally { globalThis.Blob = originalBlob; } @@ -107,14 +107,14 @@ describe("transformRawToType", () => { it("converts ArrayBuffer to Blob", () => { if (!hasBlob) return; const input = new Uint8Array([1, 2, 3]).buffer; - const result = transformRawToType(input, "blob"); + const result = toBinary(input, "blob"); expect(result).toBeInstanceOf(Blob); }); it("throws on unsupported input", () => { if (!hasBlob) return; - expect(() => transformRawToType(123, "blob")).toThrow(); - expect(() => transformRawToType({}, "blob")).toThrow(); + expect(() => toBinary(123, "blob")).toThrow(); + expect(() => toBinary({}, "blob")).toThrow(); }); }); @@ -122,21 +122,21 @@ describe("transformRawToType", () => { it("returns ReadableStream as-is", () => { if (!hasReadableStream) return; const input = new ReadableStream({}); - const result = transformRawToType(input, "stream"); + const result = toBinary(input, "stream"); expect(result).toBe(input); }); it("converts Uint8Array to ReadableStream", async () => { if (!hasReadableStream) return; const input = new Uint8Array([1, 2, 3]); - const stream = transformRawToType(input, "stream"); + const stream = toBinary(input, "stream"); expect(stream).toBeInstanceOf(ReadableStream); }); it("converts Buffer to ReadableStream", async () => { if (!hasReadableStream || !isNode) return; const input = Buffer.from([1, 2, 3]); - const stream = transformRawToType(input, "stream"); + const stream = toBinary(input, "stream"); expect(stream).toBeInstanceOf(ReadableStream); }); @@ -145,10 +145,10 @@ describe("transformRawToType", () => { // Force ReadableStream to be defined to enter error branches globalThis.ReadableStream = class MockStream {} as any; try { - expect(() => transformRawToType(123, "stream")).toThrow(); - expect(() => transformRawToType({}, "stream")).toThrow(); - expect(() => transformRawToType(null, "stream")).toThrow(); - expect(() => transformRawToType(undefined, "stream")).toThrow(); + expect(() => toBinary(123, "stream")).toThrow(); + expect(() => toBinary({}, "stream")).toThrow(); + expect(() => toBinary(null, "stream")).toThrow(); + expect(() => toBinary(undefined, "stream")).toThrow(); } finally { globalThis.ReadableStream = originalStream; } @@ -157,27 +157,27 @@ describe("transformRawToType", () => { it("converts string to ReadableStream", async () => { if (!hasReadableStream) return; const input = "hello"; - const stream = transformRawToType(input, "stream"); + const stream = toBinary(input, "stream"); expect(stream).toBeInstanceOf(ReadableStream); }); it("converts ArrayBuffer to ReadableStream", async () => { if (!hasReadableStream) return; const input = new Uint8Array([1, 2, 3]).buffer; - const stream = transformRawToType(input, "stream"); + const stream = toBinary(input, "stream"); expect(stream).toBeInstanceOf(ReadableStream); }); it("throws on unsupported input", () => { if (!hasReadableStream) return; - expect(() => transformRawToType(123, "stream")).toThrow(); - expect(() => transformRawToType({}, "stream")).toThrow(); + expect(() => toBinary(123, "stream")).toThrow(); + expect(() => toBinary({}, "stream")).toThrow(); }); }); describe("unknown type", () => { it("throws on unknown type", () => { - expect(() => transformRawToType("hello", "unknown" as any)).toThrow(); + expect(() => toBinary("hello", "unknown" as any)).toThrow(); }); }); }); From 82f12279c67fcd64b968a52e0633c78b1fa19e3a Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Sun, 13 Apr 2025 14:17:52 +0200 Subject: [PATCH 6/8] improve types for json --- src/types.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/types.ts b/src/types.ts index 2fdaa8bed..29129a212 100644 --- a/src/types.ts +++ b/src/types.ts @@ -6,6 +6,12 @@ type MaybePromise = T | Promise; type MaybeDefined = T extends any ? T : any; +// prettier-ignore +export type JSONValue = string | number | boolean | null | JSONObject | JSONArray; +// prettier-ignore +export interface JSONObject { [key: string]: JSONValue; } +export type JSONArray = JSONValue[]; + export type Unwatch = () => MaybePromise; export interface StorageMeta { @@ -110,7 +116,7 @@ export interface Storage { getItem( key: string, opts: { type: "json" } & TransactionOptions - ): Promise; + ): Promise; getItem( key: string, From 4ba918ddb253208a8079809f31824c8b343c4d9e Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Sun, 13 Apr 2025 14:18:31 +0200 Subject: [PATCH 7/8] remove empty line --- src/types.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/types.ts b/src/types.ts index 29129a212..1885e90a8 100644 --- a/src/types.ts +++ b/src/types.ts @@ -222,7 +222,6 @@ export interface Storage { // Aliases keys: Storage["getKeys"]; get: Storage["getItem"]; - set: Storage["setItem"]; has: Storage["hasItem"]; del: Storage["removeItem"]; From fab5d9d6186da47a0e978fa66d4bee0168e8a1c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Lucas?= Date: Sun, 13 Apr 2025 12:39:32 -0300 Subject: [PATCH 8/8] chore: add undio and remove custom binary transformer --- package.json | 3 +- pnpm-lock.yaml | 8 ++ src/storage.ts | 27 +++++- src/utils.ts | 44 ---------- test/storage.test.ts | 29 +++++++ test/to-binary.test.ts | 183 ----------------------------------------- 6 files changed, 64 insertions(+), 230 deletions(-) delete mode 100644 test/to-binary.test.ts diff --git a/package.json b/package.json index 93db54c27..6a50f6a13 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,8 @@ "lru-cache": "^10.4.3", "node-fetch-native": "^1.6.6", "ofetch": "^1.4.1", - "ufo": "^1.6.1" + "ufo": "^1.6.1", + "undio": "^0.2.0" }, "devDependencies": { "@azure/app-configuration": "^1.9.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 51326451d..08069072c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -32,6 +32,9 @@ importers: ufo: specifier: ^1.6.1 version: 1.6.1 + undio: + specifier: ^0.2.0 + version: 0.2.0 devDependencies: '@azure/app-configuration': specifier: ^1.9.0 @@ -4458,6 +4461,9 @@ packages: resolution: {integrity: sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==} engines: {node: '>=14.0'} + undio@0.2.0: + resolution: {integrity: sha512-1LH824ipsUNqX1qsO6qpcusv0oGPlfFWVykwWq5jJB0Mq6x4kEHO/izSq2KLjGZvOosEd91+HXoxYUSoVI0zPg==} + unenv@2.0.0-rc.15: resolution: {integrity: sha512-J/rEIZU8w6FOfLNz/hNKsnY+fFHWnu9MH4yRbSZF3xbbGHovcetXPs7sD+9p8L6CeNC//I9bhRYAOsBt2u7/OA==} @@ -9384,6 +9390,8 @@ snapshots: dependencies: '@fastify/busboy': 2.1.1 + undio@0.2.0: {} + unenv@2.0.0-rc.15: dependencies: defu: 6.1.4 diff --git a/src/storage.ts b/src/storage.ts index 76bc08545..eb4986f57 100644 --- a/src/storage.ts +++ b/src/storage.ts @@ -16,8 +16,8 @@ import { joinKeys, filterKeyByDepth, filterKeyByBase, - toBinary, } from "./utils"; +import { toBlob, toReadableStream, toUint8Array } from "undio"; interface StorageCTX { mounts: Record; @@ -190,10 +190,33 @@ export function createStorage( : await asyncCall(driver.getItem, relativeKey, opts).then((val) => deserializeRaw(val) ); + if (raw === null || raw === undefined) { return null; } - return toBinary(raw, type); + + switch (type) { + case "bytes": { + return await toUint8Array(raw); + } + case "blob": { + if (typeof Blob === "undefined") { + throw new TypeError("Blob is not supported in this environment."); + } + return await toBlob(raw); + } + case "stream": { + if (typeof ReadableStream === "undefined") { + throw new TypeError( + "ReadableStream is not supported in this environment." + ); + } + return await toReadableStream(raw); + } + default: { + throw new Error(`Invalid binary type: ${type}`); + } + } } const value = await asyncCall(driver.getItem, relativeKey, opts); diff --git a/src/utils.ts b/src/utils.ts index c04014baa..98af01c0c 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -124,47 +124,3 @@ export function filterKeyByBase( return key[key.length - 1] !== "$"; } - -/** - * Transform raw data into the expected type. - */ -export function toBinary( - value: unknown, - type: "bytes" | "blob" | "stream" -): Uint8Array | Blob | ReadableStream { - switch (type) { - case "bytes": { - if (value instanceof Uint8Array) { - return value; - } - if (typeof value === "string") { - return new TextEncoder().encode(value); - } - if (value instanceof ArrayBuffer) { - return new Uint8Array(value); - } - throw new Error(`Cannot convert raw data to bytes: ${value}`); - } - case "blob": { - if (value instanceof Blob) { - return value; - } - return new Blob([toBinary(value, "bytes") as Uint8Array]); - } - case "stream": { - if (value instanceof ReadableStream) { - return value; - } - const bytes = toBinary(value, "bytes") as Uint8Array; - return new ReadableStream({ - start(controller) { - controller.enqueue([bytes]); - controller.close(); - }, - }); - } - default: { - throw new Error(`Unsupported type: ${type}`); - } - } -} diff --git a/test/storage.test.ts b/test/storage.test.ts index d9ac3d5ea..db54939b0 100644 --- a/test/storage.test.ts +++ b/test/storage.test.ts @@ -313,4 +313,33 @@ describe("get() with type option", () => { const len = result?.length || result?.byteLength; expect(len).toBe(4); }); + + it("should get bytes with type=stream", async () => { + const bytes = new Uint8Array([1, 2, 3, 4]); + await storage.setItemRaw("stream-key", bytes); + const result = await storage.get("stream-key", { type: "stream" }); + expect(result).toBeInstanceOf(ReadableStream); + const reader = result?.getReader(); + if (!reader) { + throw new Error("Reader is not defined"); + } + const { done, value } = await reader.read(); + expect(done).toBe(false); + expect(value).toEqual(bytes); + const len = value?.length || value?.byteLength; + expect(len).toBe(4); + + const { done: doneAfter, value: valueAfter } = await reader.read(); + expect(doneAfter).toBe(true); + expect(valueAfter).toBeUndefined(); + }); + + it("should get bytes with type=blob", async () => { + const bytes = new Uint8Array([1, 2, 3, 4]); + await storage.setItemRaw("blob-key", bytes); + const result = await storage.get("blob-key", { type: "blob" }); + expect(result).toBeInstanceOf(Blob); + const arrayBuffer = await result?.arrayBuffer(); + expect(new Uint8Array(arrayBuffer || [])).toEqual(bytes); + }); }); diff --git a/test/to-binary.test.ts b/test/to-binary.test.ts deleted file mode 100644 index f8172c9f4..000000000 --- a/test/to-binary.test.ts +++ /dev/null @@ -1,183 +0,0 @@ -import { describe, it, expect } from "vitest"; -import { toBinary } from "../src/utils"; - -const isNode = typeof Buffer !== "undefined"; -const hasBlob = typeof Blob !== "undefined"; -const hasReadableStream = typeof ReadableStream !== "undefined"; - -describe("toBinary", () => { - describe('"bytes"', () => { - it("returns Uint8Array as-is", () => { - const input = new Uint8Array([1, 2, 3]); - const result = toBinary(input, "bytes"); - expect(result).toBe(input); - }); - - it("returns Buffer as-is", () => { - if (!isNode) return; - const input = Buffer.from([1, 2, 3]); - const result = toBinary(input, "bytes"); - expect(result).toBe(input); - }); - - it("converts string to Uint8Array", () => { - const input = "hello"; - const result = toBinary(input, "bytes"); - expect( - result instanceof Uint8Array || (isNode && result instanceof Buffer) - ).toBe(true); - if ( - result instanceof Uint8Array || - (isNode && result instanceof Buffer) - ) { - expect(new TextDecoder().decode(result)).toBe("hello"); - } - }); - - it("converts ArrayBuffer to Uint8Array", () => { - const input = new Uint8Array([1, 2, 3]).buffer; - const result = toBinary(input, "bytes"); - expect( - result instanceof Uint8Array || (isNode && result instanceof Buffer) - ).toBe(true); - if ( - result instanceof Uint8Array || - (isNode && result instanceof Buffer) - ) { - expect([...result]).toEqual([1, 2, 3]); - } - }); - - it('throws when "bytes" input is invalid', () => { - expect(() => toBinary(123, "bytes")).toThrow(); - expect(() => toBinary({}, "bytes")).toThrow(); - expect(() => toBinary(null, "bytes")).toThrow(); - expect(() => toBinary(undefined, "bytes")).toThrow(); - }); - - it("throws on unsupported input", () => { - expect(() => toBinary(123, "bytes")).toThrow(); - expect(() => toBinary({}, "bytes")).toThrow(); - }); - }); - - describe('"blob"', () => { - it("returns Blob as-is", () => { - if (!hasBlob) return; - const input = new Blob(["hello"]); - const result = toBinary(input, "blob"); - expect(result).toBe(input); - }); - - it("converts Uint8Array to Blob", () => { - if (!hasBlob) return; - const input = new Uint8Array([1, 2, 3]); - const result = toBinary(input, "blob"); - expect(result).toBeInstanceOf(Blob); - }); - - it("converts Buffer to Blob", () => { - if (!hasBlob || !isNode) return; - const input = Buffer.from([1, 2, 3]); - const result = toBinary(input, "blob"); - expect(result).toBeInstanceOf(Blob); - }); - - it("converts string to Blob", () => { - if (!hasBlob) return; - const input = "hello"; - const result = toBinary(input, "blob"); - expect(result).toBeInstanceOf(Blob); - }); - - it('throws when "blob" input is invalid', () => { - const originalBlob = globalThis.Blob; - // Force Blob to be defined to enter error branches - globalThis.Blob = class MockBlob {} as any; - try { - expect(() => toBinary(123, "blob")).toThrow(); - expect(() => toBinary({}, "blob")).toThrow(); - expect(() => toBinary(null, "blob")).toThrow(); - expect(() => toBinary(undefined, "blob")).toThrow(); - } finally { - globalThis.Blob = originalBlob; - } - }); - - it("converts ArrayBuffer to Blob", () => { - if (!hasBlob) return; - const input = new Uint8Array([1, 2, 3]).buffer; - const result = toBinary(input, "blob"); - expect(result).toBeInstanceOf(Blob); - }); - - it("throws on unsupported input", () => { - if (!hasBlob) return; - expect(() => toBinary(123, "blob")).toThrow(); - expect(() => toBinary({}, "blob")).toThrow(); - }); - }); - - describe('"stream"', () => { - it("returns ReadableStream as-is", () => { - if (!hasReadableStream) return; - const input = new ReadableStream({}); - const result = toBinary(input, "stream"); - expect(result).toBe(input); - }); - - it("converts Uint8Array to ReadableStream", async () => { - if (!hasReadableStream) return; - const input = new Uint8Array([1, 2, 3]); - const stream = toBinary(input, "stream"); - expect(stream).toBeInstanceOf(ReadableStream); - }); - - it("converts Buffer to ReadableStream", async () => { - if (!hasReadableStream || !isNode) return; - const input = Buffer.from([1, 2, 3]); - const stream = toBinary(input, "stream"); - expect(stream).toBeInstanceOf(ReadableStream); - }); - - it('throws when "stream" input is invalid', () => { - const originalStream = globalThis.ReadableStream; - // Force ReadableStream to be defined to enter error branches - globalThis.ReadableStream = class MockStream {} as any; - try { - expect(() => toBinary(123, "stream")).toThrow(); - expect(() => toBinary({}, "stream")).toThrow(); - expect(() => toBinary(null, "stream")).toThrow(); - expect(() => toBinary(undefined, "stream")).toThrow(); - } finally { - globalThis.ReadableStream = originalStream; - } - }); - - it("converts string to ReadableStream", async () => { - if (!hasReadableStream) return; - const input = "hello"; - const stream = toBinary(input, "stream"); - expect(stream).toBeInstanceOf(ReadableStream); - }); - - it("converts ArrayBuffer to ReadableStream", async () => { - if (!hasReadableStream) return; - const input = new Uint8Array([1, 2, 3]).buffer; - const stream = toBinary(input, "stream"); - expect(stream).toBeInstanceOf(ReadableStream); - }); - - it("throws on unsupported input", () => { - if (!hasReadableStream) return; - expect(() => toBinary(123, "stream")).toThrow(); - expect(() => toBinary({}, "stream")).toThrow(); - }); - }); - - describe("unknown type", () => { - it("throws on unknown type", () => { - expect(() => toBinary("hello", "unknown" as any)).toThrow(); - }); - }); -});