diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 1f9d6b8c..7c955c41 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -1,4 +1,4 @@ -name: Close and mark stale issue +name: Close Stale Issues on: schedule: diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index f11662da..463832da 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,33 @@ +## [ipfs-unixfs-exporter-v13.7.3](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.2...ipfs-unixfs-exporter-13.7.3) (2025-08-12) + +### Bug Fixes + +* export basic file from dir or shard ([#440](https://github.com/ipfs/js-ipfs-unixfs/issues/440)) ([b8d33de](https://github.com/ipfs/js-ipfs-unixfs/commit/b8d33deb0dfc76cc53eb82e31a67748a8da24eae)) + +## [ipfs-unixfs-exporter-v13.7.2](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.1...ipfs-unixfs-exporter-13.7.2) (2025-07-31) + +### Bug Fixes + +* add extended to default exporter options ([#439](https://github.com/ipfs/js-ipfs-unixfs/issues/439)) ([278aea4](https://github.com/ipfs/js-ipfs-unixfs/commit/278aea4c2ed76a8d890a0d2d3a079b03a9c00334)) + +## [ipfs-unixfs-exporter-v13.7.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.0...ipfs-unixfs-exporter-13.7.1) (2025-07-31) + +### Bug Fixes + +* add option to export non-extended unixfs ([#438](https://github.com/ipfs/js-ipfs-unixfs/issues/438)) ([c9a9bf4](https://github.com/ipfs/js-ipfs-unixfs/commit/c9a9bf45a5c8a779ed73cc2238a58c01e090edb7)) + +## [ipfs-unixfs-exporter-v13.7.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.6...ipfs-unixfs-exporter-13.7.0) (2025-07-30) + +### Features + +* add 'extended' option to exporter ([#437](https://github.com/ipfs/js-ipfs-unixfs/issues/437)) ([332a794](https://github.com/ipfs/js-ipfs-unixfs/commit/332a794227f7792e1ddee1b1e47d01fd510d6cf4)) + +## [ipfs-unixfs-exporter-v13.6.6](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.5...ipfs-unixfs-exporter-13.6.6) (2025-06-18) + +### Bug Fixes + +* constrain the unixfs type ([#435](https://github.com/ipfs/js-ipfs-unixfs/issues/435)) ([7663b87](https://github.com/ipfs/js-ipfs-unixfs/commit/7663b87ed2e3e8cd4da1484ca601638740ea0ae7)) + ## [ipfs-unixfs-exporter-v13.6.5](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.4...ipfs-unixfs-exporter-13.6.5) (2025-06-18) ### Documentation diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 2110db9e..1f3a4177 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.6.5", + "version": "13.7.3", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme", diff --git a/packages/ipfs-unixfs-exporter/src/index.ts b/packages/ipfs-unixfs-exporter/src/index.ts index e8b06f59..35e92277 100644 --- a/packages/ipfs-unixfs-exporter/src/index.ts +++ b/packages/ipfs-unixfs-exporter/src/index.ts @@ -134,6 +134,34 @@ export interface ExporterOptions extends ProgressOptions * (default: undefined) */ blockReadConcurrency?: number + + /** + * When directory contents are listed, by default the root node of each entry + * is fetched to decode the UnixFS metadata and know if the entry is a file or + * a directory. This can result in fetching extra data which may not be + * desirable, depending on your application. + * + * Pass false here to only return the CID and the name of the entry and not + * any extended metadata. + * + * @default true + */ + extended?: boolean +} + +export interface BasicExporterOptions extends ExporterOptions { + /** + * When directory contents are listed, by default the root node of each entry + * is fetched to decode the UnixFS metadata and know if the entry is a file or + * a directory. This can result in fetching extra data which may not be + * desirable, depending on your application. + * + * Pass false here to only return the CID and the name of the entry and not + * any extended metadata. + * + * @default true + */ + extended: false } export interface Exportable { @@ -218,7 +246,7 @@ export interface Exportable { * // `entries` contains the first 5 files/directories in the directory * ``` */ - content(options?: ExporterOptions): AsyncGenerator + content(options?: ExporterOptions | BasicExporterOptions): AsyncGenerator } /** @@ -298,6 +326,23 @@ export interface IdentityNode extends Exportable { */ export type UnixFSEntry = UnixFSFile | UnixFSDirectory | ObjectNode | RawNode | IdentityNode +export interface UnixFSBasicEntry { + /** + * The name of the entry + */ + name: string + + /** + * The path of the entry within the DAG in which it was encountered + */ + path: string + + /** + * The CID of the entry + */ + cid: CID +} + export interface NextResult { cid: CID name: string @@ -311,12 +356,20 @@ export interface ResolveResult { } export interface Resolve { (cid: CID, name: string, path: string, toResolve: string[], depth: number, blockstore: ReadableStorage, options: ExporterOptions): Promise } -export interface Resolver { (cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions): Promise } +export interface Resolver { (cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions | BasicExporterOptions): Promise } export type UnixfsV1FileContent = AsyncIterable | Iterable export type UnixfsV1DirectoryContent = AsyncIterable | Iterable export type UnixfsV1Content = UnixfsV1FileContent | UnixfsV1DirectoryContent -export interface UnixfsV1Resolver { (cid: CID, node: PBNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage): (options: ExporterOptions) => UnixfsV1Content } + +export interface UnixFsV1ContentResolver { + (options: ExporterOptions): UnixfsV1Content + (options: BasicExporterOptions): UnixFSBasicEntry +} + +export interface UnixfsV1Resolver { + (cid: CID, node: PBNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage): (options: ExporterOptions) => UnixfsV1Content +} export interface ShardTraversalContext { hamtDepth: number @@ -387,6 +440,8 @@ const cidAndRest = (path: string | Uint8Array | CID): { cid: CID, toResolve: str * // entries contains 4x `entry` objects * ``` */ +export function walkPath (path: string | CID, blockstore: ReadableStorage, options?: ExporterOptions): AsyncGenerator +export function walkPath (path: string | CID, blockstore: ReadableStorage, options: BasicExporterOptions): AsyncGenerator export async function * walkPath (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): AsyncGenerator { let { cid, @@ -443,6 +498,8 @@ export async function * walkPath (path: string | CID, blockstore: ReadableStorag * } * ``` */ +export async function exporter (path: string | CID, blockstore: ReadableStorage, options?: ExporterOptions): Promise +export async function exporter (path: string | CID, blockstore: ReadableStorage, options: BasicExporterOptions): Promise export async function exporter (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): Promise { const result = await last(walkPath(path, blockstore, options)) @@ -471,6 +528,8 @@ export async function exporter (path: string | CID, blockstore: ReadableStorage, * // entries contains all children of the `Qmfoo/foo/bar` directory and it's children * ``` */ +export function recursive (path: string | CID, blockstore: ReadableStorage, options?: ExporterOptions): AsyncGenerator +export function recursive (path: string | CID, blockstore: ReadableStorage, options: BasicExporterOptions): AsyncGenerator export async function * recursive (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): AsyncGenerator { const node = await exporter(path, blockstore, options) diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts index afab2634..73dc5c06 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts @@ -3,10 +3,11 @@ import map from 'it-map' import parallel from 'it-parallel' import { pipe } from 'it-pipe' import { CustomProgressEvent } from 'progress-events' -import type { ExporterOptions, ExportWalk, UnixfsV1DirectoryContent, UnixfsV1Resolver } from '../../../index.js' +import { isBasicExporterOptions } from '../../../utils/is-basic-exporter-options.ts' +import type { BasicExporterOptions, ExporterOptions, ExportWalk, UnixFSBasicEntry, UnixfsV1Resolver } from '../../../index.js' const directoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => { - async function * yieldDirectoryContent (options: ExporterOptions = {}): UnixfsV1DirectoryContent { + async function * yieldDirectoryContent (options: ExporterOptions | BasicExporterOptions = {}): any { const offset = options.offset ?? 0 const length = options.length ?? node.Links.length const links = node.Links.slice(offset, length) @@ -21,6 +22,17 @@ const directoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, de return async () => { const linkName = link.Name ?? '' const linkPath = `${path}/${linkName}` + + if (isBasicExporterOptions(options)) { + const basic: UnixFSBasicEntry = { + cid: link.Hash, + name: linkName, + path: linkPath + } + + return basic + } + const result = await resolve(link.Hash, linkName, linkPath, [], depth + 1, blockstore, options) return result.entry } diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts index b08255a2..d191a688 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts @@ -5,11 +5,12 @@ import parallel from 'it-parallel' import { pipe } from 'it-pipe' import { CustomProgressEvent } from 'progress-events' import { NotUnixFSError } from '../../../errors.js' -import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk } from '../../../index.js' +import { isBasicExporterOptions } from '../../../utils/is-basic-exporter-options.ts' +import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk, BasicExporterOptions, UnixFSBasicEntry } from '../../../index.js' import type { PBNode } from '@ipld/dag-pb' const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => { - function yieldHamtDirectoryContent (options: ExporterOptions = {}): UnixfsV1DirectoryContent { + function yieldHamtDirectoryContent (options: ExporterOptions | BasicExporterOptions = {}): UnixfsV1DirectoryContent { options.onProgress?.(new CustomProgressEvent('unixfs:exporter:walk:hamt-sharded-directory', { cid })) @@ -20,7 +21,7 @@ const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, return yieldHamtDirectoryContent } -async function * listDirectory (node: PBNode, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions): UnixfsV1DirectoryContent { +async function * listDirectory (node: PBNode, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions | BasicExporterOptions): any { const links = node.Links if (node.Data == null) { @@ -47,9 +48,29 @@ async function * listDirectory (node: PBNode, path: string, resolve: Resolve, de const name = link.Name != null ? link.Name.substring(padLength) : null if (name != null && name !== '') { - const result = await resolve(link.Hash, name, `${path}/${name}`, [], depth + 1, blockstore, options) + const linkPath = `${path}/${name}` - return { entries: result.entry == null ? [] : [result.entry] } + if (isBasicExporterOptions(options)) { + const basic: UnixFSBasicEntry = { + cid: link.Hash, + name, + path: linkPath + } + + return { + entries: [ + basic + ] + } + } + + const result = await resolve(link.Hash, name, linkPath, [], depth + 1, blockstore, options) + + return { + entries: [ + result.entry + ].filter(Boolean) + } } else { // descend into subshard const block = await blockstore.get(link.Hash, options) @@ -59,7 +80,9 @@ async function * listDirectory (node: PBNode, path: string, resolve: Resolve, de cid: link.Hash })) - return { entries: listDirectory(node, path, resolve, depth, blockstore, options) } + return { + entries: listDirectory(node, path, resolve, depth, blockstore, options) + } } } }), diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts index 4c5983e4..4fd337f3 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts @@ -2,10 +2,11 @@ import { decode } from '@ipld/dag-pb' import { UnixFS } from 'ipfs-unixfs' import { NotFoundError, NotUnixFSError } from '../../errors.js' import findShardCid from '../../utils/find-cid-in-shard.js' +import { isBasicExporterOptions } from '../../utils/is-basic-exporter-options.ts' import contentDirectory from './content/directory.js' import contentFile from './content/file.js' import contentHamtShardedDirectory from './content/hamt-sharded-directory.js' -import type { Resolver, UnixfsV1Resolver } from '../../index.js' +import type { Resolver, UnixFSBasicEntry, UnixfsV1Resolver } from '../../index.js' import type { PBNode } from '@ipld/dag-pb' import type { CID } from 'multiformats/cid' @@ -30,6 +31,18 @@ const contentExporters: Record = { // @ts-expect-error types are wrong const unixFsResolver: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => { + if (isBasicExporterOptions(options) && toResolve.length === 0) { + const basic: UnixFSBasicEntry = { + cid, + name, + path + } + + return { + entry: basic + } + } + const block = await blockstore.get(cid, options) const node = decode(block) let unixfs diff --git a/packages/ipfs-unixfs-exporter/src/utils/is-basic-exporter-options.ts b/packages/ipfs-unixfs-exporter/src/utils/is-basic-exporter-options.ts new file mode 100644 index 00000000..95190ea5 --- /dev/null +++ b/packages/ipfs-unixfs-exporter/src/utils/is-basic-exporter-options.ts @@ -0,0 +1,5 @@ +import type { BasicExporterOptions } from '../index.js' + +export function isBasicExporterOptions (obj?: any): obj is BasicExporterOptions { + return obj?.extended === false +} diff --git a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts index dd2354c5..a6abbc85 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts @@ -363,4 +363,88 @@ describe('exporter sharded', function () { content: file?.node }]).to.deep.equal(files) }) + + it('exports basic sharded directory', async () => { + const files: Record = {} + + // needs to result in a block that is larger than SHARD_SPLIT_THRESHOLD bytes + for (let i = 0; i < 100; i++) { + files[`file-${Math.random()}.txt`] = { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD, + rawLeaves: false + })) + + const dirCid = imported.pop()?.cid + + if (dirCid == null) { + throw new Error('No directory CID found') + } + + const exported = await exporter(dirCid, block) + const dirFiles = await all(exported.content()) + + // delete shard contents + for (const entry of dirFiles) { + await block.delete(entry.cid) + } + + // list the contents again, this time just the basic version + const basicDirFiles = await all(exported.content({ + extended: false + })) + expect(basicDirFiles.length).to.equal(dirFiles.length) + + for (let i = 0; i < basicDirFiles.length; i++) { + const dirFile = basicDirFiles[i] + + expect(dirFile).to.have.property('name') + expect(dirFile).to.have.property('path') + expect(dirFile).to.have.property('cid') + + // should fail because we have deleted this block + await expect(exporter(dirFile.cid, block)).to.eventually.be.rejected() + } + }) + + it('exports basic file from sharded directory', async () => { + const files: Record = {} + + // needs to result in a block that is larger than SHARD_SPLIT_THRESHOLD bytes + for (let i = 0; i < 100; i++) { + files[`file-${Math.random()}.txt`] = { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD, + rawLeaves: false + })) + + const file = imported[0] + const dir = imported[imported.length - 1] + + const basicfile = await exporter(`/ipfs/${dir.cid}/${file.path}`, block, { + extended: false + }) + + expect(basicfile).to.have.property('name', file.path) + expect(basicfile).to.have.property('path', `${dir.cid}/${file.path}`) + expect(basicfile).to.have.deep.property('cid', file.cid) + expect(basicfile).to.not.have.property('unixfs') + expect(basicfile).to.not.have.property('content') + }) }) diff --git a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts index 67326ec4..7d4de993 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts @@ -1605,4 +1605,136 @@ describe('exporter', () => { expect(actualInvocations).to.deep.equal(expectedInvocations) }) + + it('exports basic directory contents', async () => { + const files: Record = {} + + for (let i = 0; i < 10; i++) { + files[`file-${Math.random()}.txt`] = { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + rawLeaves: false + })) + + const dirCid = imported.pop()?.cid + + if (dirCid == null) { + throw new Error('No directory CID found') + } + + const exported = await exporter(dirCid, block) + const dirFiles = await all(exported.content()) + + // delete shard contents + for (const entry of dirFiles) { + await block.delete(entry.cid) + } + + // list the contents again, this time just the basic version + const basicDirFiles = await all(exported.content({ + extended: false + })) + expect(basicDirFiles.length).to.equal(dirFiles.length) + + for (let i = 0; i < basicDirFiles.length; i++) { + const dirFile = basicDirFiles[i] + + expect(dirFile).to.have.property('name') + expect(dirFile).to.have.property('path') + expect(dirFile).to.have.property('cid') + + // should fail because we have deleted this block + await expect(exporter(dirFile.cid, block)).to.eventually.be.rejected() + } + }) + + it('exports basic file', async () => { + const imported = await all(importer([{ + content: uint8ArrayFromString('hello') + }], block, { + rawLeaves: false + })) + + const regularFile = await exporter(imported[0].cid, block) + expect(regularFile).to.have.property('unixfs') + + const basicFile = await exporter(imported[0].cid, block, { + extended: false + }) + + expect(basicFile).to.have.property('name') + expect(basicFile).to.have.property('path') + expect(basicFile).to.have.property('cid') + expect(basicFile).to.not.have.property('unixfs') + }) + + it('exports basic directory', async () => { + const files: Record = {} + + for (let i = 0; i < 10; i++) { + files[`file-${Math.random()}.txt`] = { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + rawLeaves: false + })) + + const dirCid = imported.pop()?.cid + + if (dirCid == null) { + throw new Error('No directory CID found') + } + + const basicDir = await exporter(dirCid, block, { + extended: false + }) + + expect(basicDir).to.have.property('name') + expect(basicDir).to.have.property('path') + expect(basicDir).to.have.property('cid') + expect(basicDir).to.not.have.property('unixfs') + expect(basicDir).to.not.have.property('content') + }) + + it('exports basic file from directory', async () => { + const files: Record = { + 'file.txt': { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + rawLeaves: false + })) + + const file = imported[0] + const dir = imported[imported.length - 1] + + const basicfile = await exporter(`/ipfs/${dir.cid}/${file.path}`, block, { + extended: false + }) + + expect(basicfile).to.have.property('name', file.path) + expect(basicfile).to.have.property('path', `${dir.cid}/${file.path}`) + expect(basicfile).to.have.deep.property('cid', file.cid) + expect(basicfile).to.not.have.property('unixfs') + expect(basicfile).to.not.have.property('content') + }) })