From ce859ab474e81ecea63ab90533df308b0e4e2624 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 18 Jun 2025 08:09:47 +0000 Subject: [PATCH 01/10] chore(release): 13.6.6 [skip ci] ## [ipfs-unixfs-exporter-v13.6.6](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.5...ipfs-unixfs-exporter-13.6.6) (2025-06-18) ### Bug Fixes * constrain the unixfs type ([#435](https://github.com/ipfs/js-ipfs-unixfs/issues/435)) ([7663b87](https://github.com/ipfs/js-ipfs-unixfs/commit/7663b87ed2e3e8cd4da1484ca601638740ea0ae7)) --- packages/ipfs-unixfs-exporter/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-exporter/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index f11662da..77395b0d 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-exporter-v13.6.6](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.5...ipfs-unixfs-exporter-13.6.6) (2025-06-18) + +### Bug Fixes + +* constrain the unixfs type ([#435](https://github.com/ipfs/js-ipfs-unixfs/issues/435)) ([7663b87](https://github.com/ipfs/js-ipfs-unixfs/commit/7663b87ed2e3e8cd4da1484ca601638740ea0ae7)) + ## [ipfs-unixfs-exporter-v13.6.5](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.4...ipfs-unixfs-exporter-13.6.5) (2025-06-18) ### Documentation diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 2110db9e..7ec27009 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.6.5", + "version": "13.6.6", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme", From 93cb3d06713c876ad861fe61082e21af4f1905ac Mon Sep 17 00:00:00 2001 From: web3-bot <81333946+web3-bot@users.noreply.github.com> Date: Sat, 26 Jul 2025 08:29:51 +0100 Subject: [PATCH 02/10] chore: add or force update .github/workflows/stale.yml (#436) --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 1f9d6b8c..7c955c41 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -1,4 +1,4 @@ -name: Close and mark stale issue +name: Close Stale Issues on: schedule: From 332a794227f7792e1ddee1b1e47d01fd510d6cf4 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Wed, 30 Jul 2025 10:06:06 +0200 Subject: [PATCH 03/10] feat: add 'extended' option to exporter (#437) To list directories without resolving the root node of each directory entry, add an `extended` option to the exporter. Defaults to `true` to preserve backwards compatibility. --- packages/ipfs-unixfs-exporter/src/index.ts | 52 ++++++++++++++++++- .../resolvers/unixfs-v1/content/directory.ts | 25 +++++++-- .../content/hamt-sharded-directory.ts | 34 +++++++++--- .../src/utils/is-basic-exporter-options.ts | 5 ++ .../test/exporter-sharded.spec.ts | 52 +++++++++++++++++++ .../test/exporter.spec.ts | 50 ++++++++++++++++++ 6 files changed, 206 insertions(+), 12 deletions(-) create mode 100644 packages/ipfs-unixfs-exporter/src/utils/is-basic-exporter-options.ts diff --git a/packages/ipfs-unixfs-exporter/src/index.ts b/packages/ipfs-unixfs-exporter/src/index.ts index e8b06f59..ee6c2e9e 100644 --- a/packages/ipfs-unixfs-exporter/src/index.ts +++ b/packages/ipfs-unixfs-exporter/src/index.ts @@ -57,6 +57,7 @@ import type { PBNode } from '@ipld/dag-pb' import type { Bucket } from 'hamt-sharding' import type { Blockstore } from 'interface-blockstore' import type { UnixFS } from 'ipfs-unixfs' +import type { AbortOptions } from 'it-pushable' import type { ProgressOptions, ProgressEvent } from 'progress-events' export * from './errors.js' @@ -136,6 +137,21 @@ export interface ExporterOptions extends ProgressOptions blockReadConcurrency?: number } +export interface BasicExporterOptions extends ExporterOptions { + /** + * When directory contents are listed, by default the root node of each entry + * is fetched to decode the UnixFS metadata and know if the entry is a file or + * a directory. This can result in fetching extra data which may not be + * desirable, depending on your application. + * + * Pass false here to only return the CID and the name of the entry and not + * any extended metadata. + * + * @default true + */ + extended: false +} + export interface Exportable { /** * A disambiguator to allow TypeScript to work out the type of the entry. @@ -218,7 +234,7 @@ export interface Exportable { * // `entries` contains the first 5 files/directories in the directory * ``` */ - content(options?: ExporterOptions): AsyncGenerator + content(options?: ExporterOptions | BasicExporterOptions): AsyncGenerator } /** @@ -316,7 +332,39 @@ export interface Resolver { (cid: CID, name: string, path: string, toResolve: st export type UnixfsV1FileContent = AsyncIterable | Iterable export type UnixfsV1DirectoryContent = AsyncIterable | Iterable export type UnixfsV1Content = UnixfsV1FileContent | UnixfsV1DirectoryContent -export interface UnixfsV1Resolver { (cid: CID, node: PBNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage): (options: ExporterOptions) => UnixfsV1Content } + +export interface UnixfsV1BasicContent { + /** + * The name of the entry + */ + name: string + + /** + * The path of the entry within the DAG in which it was encountered + */ + path: string + + /** + * The CID of the entry + */ + cid: CID + + /** + * Resolve the root node of the entry to parse the UnixFS metadata contained + * there. The metadata will contain what kind of node it is (e.g. file, + * directory, etc), the file size, and more. + */ + resolve(options?: AbortOptions): Promise +} + +export interface UnixFsV1ContentResolver { + (options: ExporterOptions): UnixfsV1Content + (options: BasicExporterOptions): UnixfsV1BasicContent +} + +export interface UnixfsV1Resolver { + (cid: CID, node: PBNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage): (options: ExporterOptions) => UnixfsV1Content +} export interface ShardTraversalContext { hamtDepth: number diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts index afab2634..614b33ca 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts @@ -3,10 +3,11 @@ import map from 'it-map' import parallel from 'it-parallel' import { pipe } from 'it-pipe' import { CustomProgressEvent } from 'progress-events' -import type { ExporterOptions, ExportWalk, UnixfsV1DirectoryContent, UnixfsV1Resolver } from '../../../index.js' +import { isBasicExporterOptions } from '../../../utils/is-basic-exporter-options.ts' +import type { BasicExporterOptions, ExporterOptions, ExportWalk, UnixFSEntry, UnixfsV1BasicContent, UnixfsV1Resolver } from '../../../index.js' const directoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => { - async function * yieldDirectoryContent (options: ExporterOptions = {}): UnixfsV1DirectoryContent { + async function * yieldDirectoryContent (options: ExporterOptions | BasicExporterOptions = {}): any { const offset = options.offset ?? 0 const length = options.length ?? node.Links.length const links = node.Links.slice(offset, length) @@ -21,8 +22,24 @@ const directoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, de return async () => { const linkName = link.Name ?? '' const linkPath = `${path}/${linkName}` - const result = await resolve(link.Hash, linkName, linkPath, [], depth + 1, blockstore, options) - return result.entry + + const load = async (options = {}): Promise => { + const result = await resolve(link.Hash, linkName, linkPath, [], depth + 1, blockstore, options) + return result.entry + } + + if (isBasicExporterOptions(options)) { + const basic: UnixfsV1BasicContent = { + cid: link.Hash, + name: linkName, + path: linkPath, + resolve: load + } + + return basic + } + + return load(options) } }), source => parallel(source, { diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts index b08255a2..a3f56189 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts @@ -5,11 +5,12 @@ import parallel from 'it-parallel' import { pipe } from 'it-pipe' import { CustomProgressEvent } from 'progress-events' import { NotUnixFSError } from '../../../errors.js' -import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk } from '../../../index.js' +import { isBasicExporterOptions } from '../../../utils/is-basic-exporter-options.ts' +import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk, BasicExporterOptions, UnixFSEntry } from '../../../index.js' import type { PBNode } from '@ipld/dag-pb' const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => { - function yieldHamtDirectoryContent (options: ExporterOptions = {}): UnixfsV1DirectoryContent { + function yieldHamtDirectoryContent (options: ExporterOptions | BasicExporterOptions = {}): UnixfsV1DirectoryContent { options.onProgress?.(new CustomProgressEvent('unixfs:exporter:walk:hamt-sharded-directory', { cid })) @@ -20,7 +21,7 @@ const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, return yieldHamtDirectoryContent } -async function * listDirectory (node: PBNode, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions): UnixfsV1DirectoryContent { +async function * listDirectory (node: PBNode, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions | BasicExporterOptions): any { const links = node.Links if (node.Data == null) { @@ -47,9 +48,28 @@ async function * listDirectory (node: PBNode, path: string, resolve: Resolve, de const name = link.Name != null ? link.Name.substring(padLength) : null if (name != null && name !== '') { - const result = await resolve(link.Hash, name, `${path}/${name}`, [], depth + 1, blockstore, options) + const linkPath = `${path}/${name}` + const load = async (options = {}): Promise => { + const result = await resolve(link.Hash, name, linkPath, [], depth + 1, blockstore, options) + return result.entry + } - return { entries: result.entry == null ? [] : [result.entry] } + if (isBasicExporterOptions(options)) { + return { + entries: [{ + cid: link.Hash, + name, + path: linkPath, + resolve: load + }] + } + } + + return { + entries: [ + await load() + ].filter(Boolean) + } } else { // descend into subshard const block = await blockstore.get(link.Hash, options) @@ -59,7 +79,9 @@ async function * listDirectory (node: PBNode, path: string, resolve: Resolve, de cid: link.Hash })) - return { entries: listDirectory(node, path, resolve, depth, blockstore, options) } + return { + entries: listDirectory(node, path, resolve, depth, blockstore, options) + } } } }), diff --git a/packages/ipfs-unixfs-exporter/src/utils/is-basic-exporter-options.ts b/packages/ipfs-unixfs-exporter/src/utils/is-basic-exporter-options.ts new file mode 100644 index 00000000..95190ea5 --- /dev/null +++ b/packages/ipfs-unixfs-exporter/src/utils/is-basic-exporter-options.ts @@ -0,0 +1,5 @@ +import type { BasicExporterOptions } from '../index.js' + +export function isBasicExporterOptions (obj?: any): obj is BasicExporterOptions { + return obj?.extended === false +} diff --git a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts index dd2354c5..fe16201e 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts @@ -363,4 +363,56 @@ describe('exporter sharded', function () { content: file?.node }]).to.deep.equal(files) }) + + it('exports basic sharded directory', async () => { + const files: Record = {} + + // needs to result in a block that is larger than SHARD_SPLIT_THRESHOLD bytes + for (let i = 0; i < 100; i++) { + files[`file-${Math.random()}.txt`] = { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD, + rawLeaves: false + })) + + const dirCid = imported.pop()?.cid + + if (dirCid == null) { + throw new Error('No directory CID found') + } + + const exported = await exporter(dirCid, block) + const dirFiles = await all(exported.content()) + + // delete shard contents + for (const entry of dirFiles) { + await block.delete(entry.cid) + } + + // list the contents again, this time just the basic version + const basicDirFiles = await all(exported.content({ + extended: false + })) + expect(basicDirFiles.length).to.equal(dirFiles.length) + + for (let i = 0; i < basicDirFiles.length; i++) { + const dirFile = basicDirFiles[i] + + expect(dirFile).to.have.property('name') + expect(dirFile).to.have.property('path') + expect(dirFile).to.have.property('cid') + expect(dirFile).to.have.property('resolve') + + // should fail because we have deleted this block + await expect(dirFile.resolve()).to.eventually.be.rejected() + } + }) }) diff --git a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts index 67326ec4..9ce3056e 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts @@ -1605,4 +1605,54 @@ describe('exporter', () => { expect(actualInvocations).to.deep.equal(expectedInvocations) }) + + it('exports basic directory', async () => { + const files: Record = {} + + for (let i = 0; i < 10; i++) { + files[`file-${Math.random()}.txt`] = { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + rawLeaves: false + })) + + const dirCid = imported.pop()?.cid + + if (dirCid == null) { + throw new Error('No directory CID found') + } + + const exported = await exporter(dirCid, block) + const dirFiles = await all(exported.content()) + + // delete shard contents + for (const entry of dirFiles) { + await block.delete(entry.cid) + } + + // list the contents again, this time just the basic version + const basicDirFiles = await all(exported.content({ + extended: false + })) + expect(basicDirFiles.length).to.equal(dirFiles.length) + + for (let i = 0; i < basicDirFiles.length; i++) { + const dirFile = basicDirFiles[i] + + expect(dirFile).to.have.property('name') + expect(dirFile).to.have.property('path') + expect(dirFile).to.have.property('cid') + expect(dirFile).to.have.property('resolve') + + // should fail because we have deleted this block + await expect(dirFile.resolve()).to.eventually.be.rejected() + } + }) }) From 5a916ab1c291e0091d76fb1093916098fd1441c4 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 30 Jul 2025 08:12:34 +0000 Subject: [PATCH 04/10] chore(release): 13.7.0 [skip ci] ## [ipfs-unixfs-exporter-v13.7.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.6...ipfs-unixfs-exporter-13.7.0) (2025-07-30) ### Features * add 'extended' option to exporter ([#437](https://github.com/ipfs/js-ipfs-unixfs/issues/437)) ([332a794](https://github.com/ipfs/js-ipfs-unixfs/commit/332a794227f7792e1ddee1b1e47d01fd510d6cf4)) --- packages/ipfs-unixfs-exporter/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-exporter/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index 77395b0d..bc190919 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-exporter-v13.7.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.6...ipfs-unixfs-exporter-13.7.0) (2025-07-30) + +### Features + +* add 'extended' option to exporter ([#437](https://github.com/ipfs/js-ipfs-unixfs/issues/437)) ([332a794](https://github.com/ipfs/js-ipfs-unixfs/commit/332a794227f7792e1ddee1b1e47d01fd510d6cf4)) + ## [ipfs-unixfs-exporter-v13.6.6](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.5...ipfs-unixfs-exporter-13.6.6) (2025-06-18) ### Bug Fixes diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 7ec27009..29ddc64f 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.6.6", + "version": "13.7.0", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme", From c9a9bf45a5c8a779ed73cc2238a58c01e090edb7 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Thu, 31 Jul 2025 11:44:58 +0200 Subject: [PATCH 05/10] fix: add option to export non-extended unixfs (#438) Adds overrides for non-extended exports --- packages/ipfs-unixfs-exporter/src/index.ts | 52 ++++++++-------- .../resolvers/unixfs-v1/content/directory.ts | 15 ++--- .../content/hamt-sharded-directory.ts | 25 ++++---- .../src/resolvers/unixfs-v1/index.ts | 15 ++++- .../test/exporter-sharded.spec.ts | 3 +- .../test/exporter.spec.ts | 59 ++++++++++++++++++- 6 files changed, 114 insertions(+), 55 deletions(-) diff --git a/packages/ipfs-unixfs-exporter/src/index.ts b/packages/ipfs-unixfs-exporter/src/index.ts index ee6c2e9e..71019551 100644 --- a/packages/ipfs-unixfs-exporter/src/index.ts +++ b/packages/ipfs-unixfs-exporter/src/index.ts @@ -57,7 +57,6 @@ import type { PBNode } from '@ipld/dag-pb' import type { Bucket } from 'hamt-sharding' import type { Blockstore } from 'interface-blockstore' import type { UnixFS } from 'ipfs-unixfs' -import type { AbortOptions } from 'it-pushable' import type { ProgressOptions, ProgressEvent } from 'progress-events' export * from './errors.js' @@ -314,6 +313,23 @@ export interface IdentityNode extends Exportable { */ export type UnixFSEntry = UnixFSFile | UnixFSDirectory | ObjectNode | RawNode | IdentityNode +export interface UnixFSBasicEntry { + /** + * The name of the entry + */ + name: string + + /** + * The path of the entry within the DAG in which it was encountered + */ + path: string + + /** + * The CID of the entry + */ + cid: CID +} + export interface NextResult { cid: CID name: string @@ -327,39 +343,15 @@ export interface ResolveResult { } export interface Resolve { (cid: CID, name: string, path: string, toResolve: string[], depth: number, blockstore: ReadableStorage, options: ExporterOptions): Promise } -export interface Resolver { (cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions): Promise } +export interface Resolver { (cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions | BasicExporterOptions): Promise } export type UnixfsV1FileContent = AsyncIterable | Iterable export type UnixfsV1DirectoryContent = AsyncIterable | Iterable export type UnixfsV1Content = UnixfsV1FileContent | UnixfsV1DirectoryContent -export interface UnixfsV1BasicContent { - /** - * The name of the entry - */ - name: string - - /** - * The path of the entry within the DAG in which it was encountered - */ - path: string - - /** - * The CID of the entry - */ - cid: CID - - /** - * Resolve the root node of the entry to parse the UnixFS metadata contained - * there. The metadata will contain what kind of node it is (e.g. file, - * directory, etc), the file size, and more. - */ - resolve(options?: AbortOptions): Promise -} - export interface UnixFsV1ContentResolver { (options: ExporterOptions): UnixfsV1Content - (options: BasicExporterOptions): UnixfsV1BasicContent + (options: BasicExporterOptions): UnixFSBasicEntry } export interface UnixfsV1Resolver { @@ -435,6 +427,8 @@ const cidAndRest = (path: string | Uint8Array | CID): { cid: CID, toResolve: str * // entries contains 4x `entry` objects * ``` */ +export function walkPath (path: string | CID, blockstore: ReadableStorage, options?: ExporterOptions): AsyncGenerator +export function walkPath (path: string | CID, blockstore: ReadableStorage, options: BasicExporterOptions): AsyncGenerator export async function * walkPath (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): AsyncGenerator { let { cid, @@ -491,6 +485,8 @@ export async function * walkPath (path: string | CID, blockstore: ReadableStorag * } * ``` */ +export async function exporter (path: string | CID, blockstore: ReadableStorage, options?: ExporterOptions): Promise +export async function exporter (path: string | CID, blockstore: ReadableStorage, options: BasicExporterOptions): Promise export async function exporter (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): Promise { const result = await last(walkPath(path, blockstore, options)) @@ -519,6 +515,8 @@ export async function exporter (path: string | CID, blockstore: ReadableStorage, * // entries contains all children of the `Qmfoo/foo/bar` directory and it's children * ``` */ +export function recursive (path: string | CID, blockstore: ReadableStorage, options?: ExporterOptions): AsyncGenerator +export function recursive (path: string | CID, blockstore: ReadableStorage, options: BasicExporterOptions): AsyncGenerator export async function * recursive (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): AsyncGenerator { const node = await exporter(path, blockstore, options) diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts index 614b33ca..73dc5c06 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts @@ -4,7 +4,7 @@ import parallel from 'it-parallel' import { pipe } from 'it-pipe' import { CustomProgressEvent } from 'progress-events' import { isBasicExporterOptions } from '../../../utils/is-basic-exporter-options.ts' -import type { BasicExporterOptions, ExporterOptions, ExportWalk, UnixFSEntry, UnixfsV1BasicContent, UnixfsV1Resolver } from '../../../index.js' +import type { BasicExporterOptions, ExporterOptions, ExportWalk, UnixFSBasicEntry, UnixfsV1Resolver } from '../../../index.js' const directoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => { async function * yieldDirectoryContent (options: ExporterOptions | BasicExporterOptions = {}): any { @@ -23,23 +23,18 @@ const directoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, de const linkName = link.Name ?? '' const linkPath = `${path}/${linkName}` - const load = async (options = {}): Promise => { - const result = await resolve(link.Hash, linkName, linkPath, [], depth + 1, blockstore, options) - return result.entry - } - if (isBasicExporterOptions(options)) { - const basic: UnixfsV1BasicContent = { + const basic: UnixFSBasicEntry = { cid: link.Hash, name: linkName, - path: linkPath, - resolve: load + path: linkPath } return basic } - return load(options) + const result = await resolve(link.Hash, linkName, linkPath, [], depth + 1, blockstore, options) + return result.entry } }), source => parallel(source, { diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts index a3f56189..d191a688 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts @@ -6,7 +6,7 @@ import { pipe } from 'it-pipe' import { CustomProgressEvent } from 'progress-events' import { NotUnixFSError } from '../../../errors.js' import { isBasicExporterOptions } from '../../../utils/is-basic-exporter-options.ts' -import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk, BasicExporterOptions, UnixFSEntry } from '../../../index.js' +import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk, BasicExporterOptions, UnixFSBasicEntry } from '../../../index.js' import type { PBNode } from '@ipld/dag-pb' const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => { @@ -49,25 +49,26 @@ async function * listDirectory (node: PBNode, path: string, resolve: Resolve, de if (name != null && name !== '') { const linkPath = `${path}/${name}` - const load = async (options = {}): Promise => { - const result = await resolve(link.Hash, name, linkPath, [], depth + 1, blockstore, options) - return result.entry - } if (isBasicExporterOptions(options)) { + const basic: UnixFSBasicEntry = { + cid: link.Hash, + name, + path: linkPath + } + return { - entries: [{ - cid: link.Hash, - name, - path: linkPath, - resolve: load - }] + entries: [ + basic + ] } } + const result = await resolve(link.Hash, name, linkPath, [], depth + 1, blockstore, options) + return { entries: [ - await load() + result.entry ].filter(Boolean) } } else { diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts index 4c5983e4..c847ca4c 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts @@ -2,10 +2,11 @@ import { decode } from '@ipld/dag-pb' import { UnixFS } from 'ipfs-unixfs' import { NotFoundError, NotUnixFSError } from '../../errors.js' import findShardCid from '../../utils/find-cid-in-shard.js' +import { isBasicExporterOptions } from '../../utils/is-basic-exporter-options.ts' import contentDirectory from './content/directory.js' import contentFile from './content/file.js' import contentHamtShardedDirectory from './content/hamt-sharded-directory.js' -import type { Resolver, UnixfsV1Resolver } from '../../index.js' +import type { Resolver, UnixFSBasicEntry, UnixfsV1Resolver } from '../../index.js' import type { PBNode } from '@ipld/dag-pb' import type { CID } from 'multiformats/cid' @@ -30,6 +31,18 @@ const contentExporters: Record = { // @ts-expect-error types are wrong const unixFsResolver: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => { + if (isBasicExporterOptions(options)) { + const basic: UnixFSBasicEntry = { + cid, + name, + path + } + + return { + entry: basic + } + } + const block = await blockstore.get(cid, options) const node = decode(block) let unixfs diff --git a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts index fe16201e..29fe8fc0 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts @@ -409,10 +409,9 @@ describe('exporter sharded', function () { expect(dirFile).to.have.property('name') expect(dirFile).to.have.property('path') expect(dirFile).to.have.property('cid') - expect(dirFile).to.have.property('resolve') // should fail because we have deleted this block - await expect(dirFile.resolve()).to.eventually.be.rejected() + await expect(exporter(dirFile.cid, block)).to.eventually.be.rejected() } }) }) diff --git a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts index 9ce3056e..be9e58e7 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts @@ -1606,7 +1606,7 @@ describe('exporter', () => { expect(actualInvocations).to.deep.equal(expectedInvocations) }) - it('exports basic directory', async () => { + it('exports basic directory contents', async () => { const files: Record = {} for (let i = 0; i < 10; i++) { @@ -1649,10 +1649,63 @@ describe('exporter', () => { expect(dirFile).to.have.property('name') expect(dirFile).to.have.property('path') expect(dirFile).to.have.property('cid') - expect(dirFile).to.have.property('resolve') // should fail because we have deleted this block - await expect(dirFile.resolve()).to.eventually.be.rejected() + await expect(exporter(dirFile.cid, block)).to.eventually.be.rejected() + } + }) + + it('exports basic file', async () => { + const imported = await all(importer([{ + content: uint8ArrayFromString('hello') + }], block, { + rawLeaves: false + })) + + const regularFile = await exporter(imported[0].cid, block) + expect(regularFile).to.have.property('unixfs') + + const basicFile = await exporter(imported[0].cid, block, { + extended: false + }) + + expect(basicFile).to.have.property('name') + expect(basicFile).to.have.property('path') + expect(basicFile).to.have.property('cid') + expect(basicFile).to.not.have.property('unixfs') + }) + + it('exports basic directory', async () => { + const files: Record = {} + + for (let i = 0; i < 10; i++) { + files[`file-${Math.random()}.txt`] = { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + rawLeaves: false + })) + + const dirCid = imported.pop()?.cid + + if (dirCid == null) { + throw new Error('No directory CID found') } + + const basicDir = await exporter(dirCid, block, { + extended: false + }) + + expect(basicDir).to.have.property('name') + expect(basicDir).to.have.property('path') + expect(basicDir).to.have.property('cid') + expect(basicDir).to.not.have.property('unixfs') + expect(basicDir).to.not.have.property('content') }) }) From 9509fb7f341240417f839c0919bba90c4fe54093 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 31 Jul 2025 09:50:27 +0000 Subject: [PATCH 06/10] chore(release): 13.7.1 [skip ci] ## [ipfs-unixfs-exporter-v13.7.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.0...ipfs-unixfs-exporter-13.7.1) (2025-07-31) ### Bug Fixes * add option to export non-extended unixfs ([#438](https://github.com/ipfs/js-ipfs-unixfs/issues/438)) ([c9a9bf4](https://github.com/ipfs/js-ipfs-unixfs/commit/c9a9bf45a5c8a779ed73cc2238a58c01e090edb7)) --- packages/ipfs-unixfs-exporter/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-exporter/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index bc190919..e31ff5e4 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-exporter-v13.7.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.0...ipfs-unixfs-exporter-13.7.1) (2025-07-31) + +### Bug Fixes + +* add option to export non-extended unixfs ([#438](https://github.com/ipfs/js-ipfs-unixfs/issues/438)) ([c9a9bf4](https://github.com/ipfs/js-ipfs-unixfs/commit/c9a9bf45a5c8a779ed73cc2238a58c01e090edb7)) + ## [ipfs-unixfs-exporter-v13.7.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.6...ipfs-unixfs-exporter-13.7.0) (2025-07-30) ### Features diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 29ddc64f..5df17ca1 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.7.0", + "version": "13.7.1", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme", From 278aea4c2ed76a8d890a0d2d3a079b03a9c00334 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Thu, 31 Jul 2025 12:15:25 +0200 Subject: [PATCH 07/10] fix: add extended to default exporter options (#439) Otherwise we can't pass true or undefined. --- packages/ipfs-unixfs-exporter/src/index.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/ipfs-unixfs-exporter/src/index.ts b/packages/ipfs-unixfs-exporter/src/index.ts index 71019551..35e92277 100644 --- a/packages/ipfs-unixfs-exporter/src/index.ts +++ b/packages/ipfs-unixfs-exporter/src/index.ts @@ -134,6 +134,19 @@ export interface ExporterOptions extends ProgressOptions * (default: undefined) */ blockReadConcurrency?: number + + /** + * When directory contents are listed, by default the root node of each entry + * is fetched to decode the UnixFS metadata and know if the entry is a file or + * a directory. This can result in fetching extra data which may not be + * desirable, depending on your application. + * + * Pass false here to only return the CID and the name of the entry and not + * any extended metadata. + * + * @default true + */ + extended?: boolean } export interface BasicExporterOptions extends ExporterOptions { From e3fbc9672a6b6223fcac839fdf691f4c8bcbdca5 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 31 Jul 2025 10:21:14 +0000 Subject: [PATCH 08/10] chore(release): 13.7.2 [skip ci] ## [ipfs-unixfs-exporter-v13.7.2](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.1...ipfs-unixfs-exporter-13.7.2) (2025-07-31) ### Bug Fixes * add extended to default exporter options ([#439](https://github.com/ipfs/js-ipfs-unixfs/issues/439)) ([278aea4](https://github.com/ipfs/js-ipfs-unixfs/commit/278aea4c2ed76a8d890a0d2d3a079b03a9c00334)) --- packages/ipfs-unixfs-exporter/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-exporter/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index e31ff5e4..7fbdf258 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-exporter-v13.7.2](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.1...ipfs-unixfs-exporter-13.7.2) (2025-07-31) + +### Bug Fixes + +* add extended to default exporter options ([#439](https://github.com/ipfs/js-ipfs-unixfs/issues/439)) ([278aea4](https://github.com/ipfs/js-ipfs-unixfs/commit/278aea4c2ed76a8d890a0d2d3a079b03a9c00334)) + ## [ipfs-unixfs-exporter-v13.7.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.0...ipfs-unixfs-exporter-13.7.1) (2025-07-31) ### Bug Fixes diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 5df17ca1..5d4cae58 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.7.1", + "version": "13.7.2", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme", From b8d33deb0dfc76cc53eb82e31a67748a8da24eae Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Tue, 12 Aug 2025 07:56:24 +0100 Subject: [PATCH 09/10] fix: export basic file from dir or shard (#440) Fixes a bug whereby when exporting a deep path to a file within a directory or shard, the returned entry had the CID of the containing folder rather than the target file. --- .../src/resolvers/unixfs-v1/index.ts | 2 +- .../test/exporter-sharded.spec.ts | 33 +++++++++++++++++++ .../test/exporter.spec.ts | 29 ++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts index c847ca4c..4fd337f3 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts @@ -31,7 +31,7 @@ const contentExporters: Record = { // @ts-expect-error types are wrong const unixFsResolver: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => { - if (isBasicExporterOptions(options)) { + if (isBasicExporterOptions(options) && toResolve.length === 0) { const basic: UnixFSBasicEntry = { cid, name, diff --git a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts index 29fe8fc0..a6abbc85 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts @@ -414,4 +414,37 @@ describe('exporter sharded', function () { await expect(exporter(dirFile.cid, block)).to.eventually.be.rejected() } }) + + it('exports basic file from sharded directory', async () => { + const files: Record = {} + + // needs to result in a block that is larger than SHARD_SPLIT_THRESHOLD bytes + for (let i = 0; i < 100; i++) { + files[`file-${Math.random()}.txt`] = { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD, + rawLeaves: false + })) + + const file = imported[0] + const dir = imported[imported.length - 1] + + const basicfile = await exporter(`/ipfs/${dir.cid}/${file.path}`, block, { + extended: false + }) + + expect(basicfile).to.have.property('name', file.path) + expect(basicfile).to.have.property('path', `${dir.cid}/${file.path}`) + expect(basicfile).to.have.deep.property('cid', file.cid) + expect(basicfile).to.not.have.property('unixfs') + expect(basicfile).to.not.have.property('content') + }) }) diff --git a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts index be9e58e7..7d4de993 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts @@ -1708,4 +1708,33 @@ describe('exporter', () => { expect(basicDir).to.not.have.property('unixfs') expect(basicDir).to.not.have.property('content') }) + + it('exports basic file from directory', async () => { + const files: Record = { + 'file.txt': { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + rawLeaves: false + })) + + const file = imported[0] + const dir = imported[imported.length - 1] + + const basicfile = await exporter(`/ipfs/${dir.cid}/${file.path}`, block, { + extended: false + }) + + expect(basicfile).to.have.property('name', file.path) + expect(basicfile).to.have.property('path', `${dir.cid}/${file.path}`) + expect(basicfile).to.have.deep.property('cid', file.cid) + expect(basicfile).to.not.have.property('unixfs') + expect(basicfile).to.not.have.property('content') + }) }) From 7f15bafe6d9efc58a42eeb26ec5165c332493927 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 12 Aug 2025 07:01:59 +0000 Subject: [PATCH 10/10] chore(release): 13.7.3 [skip ci] ## [ipfs-unixfs-exporter-v13.7.3](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.2...ipfs-unixfs-exporter-13.7.3) (2025-08-12) ### Bug Fixes * export basic file from dir or shard ([#440](https://github.com/ipfs/js-ipfs-unixfs/issues/440)) ([b8d33de](https://github.com/ipfs/js-ipfs-unixfs/commit/b8d33deb0dfc76cc53eb82e31a67748a8da24eae)) --- packages/ipfs-unixfs-exporter/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-exporter/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index 7fbdf258..463832da 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-exporter-v13.7.3](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.2...ipfs-unixfs-exporter-13.7.3) (2025-08-12) + +### Bug Fixes + +* export basic file from dir or shard ([#440](https://github.com/ipfs/js-ipfs-unixfs/issues/440)) ([b8d33de](https://github.com/ipfs/js-ipfs-unixfs/commit/b8d33deb0dfc76cc53eb82e31a67748a8da24eae)) + ## [ipfs-unixfs-exporter-v13.7.2](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.1...ipfs-unixfs-exporter-13.7.2) (2025-07-31) ### Bug Fixes diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 5d4cae58..1f3a4177 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.7.2", + "version": "13.7.3", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme",