diff --git a/.changeset/beige-ghosts-know.md b/.changeset/beige-ghosts-know.md new file mode 100644 index 00000000000..bf523447358 --- /dev/null +++ b/.changeset/beige-ghosts-know.md @@ -0,0 +1,5 @@ +--- +"@pnpm/constants": major +--- + +Changed the format of the side-effects cache key. diff --git a/.changeset/breezy-eggs-repair.md b/.changeset/breezy-eggs-repair.md new file mode 100644 index 00000000000..ebaee0f029d --- /dev/null +++ b/.changeset/breezy-eggs-repair.md @@ -0,0 +1,13 @@ +--- +"@pnpm/plugin-commands-installation": major +"@pnpm/core": major +"@pnpm/config": major +"pnpm": major +--- + +The `pnpm link` command adds overrides to the root `package.json`. In a workspace the override is added to the root of the workspace, so it links the dependency to all projects in a workspace. + +To link a package globally, just run `pnpm link` from the package's directory. Previously, the command `pnpm link -g` was required to link a package globally. + +Related PR: [#8653](https://github.com/pnpm/pnpm/pull/8653). + diff --git a/.changeset/brown-deers-fry.md b/.changeset/brown-deers-fry.md new file mode 100644 index 00000000000..fa13b45f271 --- /dev/null +++ b/.changeset/brown-deers-fry.md @@ -0,0 +1,6 @@ +--- +"@pnpm/dependency-path": major +"pnpm": major +--- + +Use sha256 for hashing long paths inside `node_modules/.pnpm`. diff --git a/.changeset/dull-feet-march.md b/.changeset/dull-feet-march.md new file mode 100644 index 00000000000..85097d348d8 --- /dev/null +++ b/.changeset/dull-feet-march.md @@ -0,0 +1,5 @@ +--- +"pnpm": major +--- + +Using SHA256 instead of md5 for hashing long peer dependency hashes in the lockfile. Should not affect a lot of users as the hashing is used for really long keys in the lockfile. diff --git a/.changeset/fair-news-beam.md b/.changeset/fair-news-beam.md new file mode 100644 index 00000000000..1539f065f3e --- /dev/null +++ b/.changeset/fair-news-beam.md @@ -0,0 +1,5 @@ +--- +"@pnpm/crypto.hash": major +--- + +Initial release. diff --git a/.changeset/fifty-forks-think.md b/.changeset/fifty-forks-think.md new file mode 100644 index 00000000000..74ef7316fd8 --- /dev/null +++ b/.changeset/fifty-forks-think.md @@ -0,0 +1,6 @@ +--- +"@pnpm/config": major +"pnpm": major +--- + +pnpm will now manage it's own versions according to the `packageManager` filed of `package.json`. To disable this, set `manage-package-manager-versions` to `false`. diff --git a/.changeset/friendly-carrots-poke.md b/.changeset/friendly-carrots-poke.md new file mode 100644 index 00000000000..69c1c6d8519 --- /dev/null +++ b/.changeset/friendly-carrots-poke.md @@ -0,0 +1,6 @@ +--- +"@pnpm/plugin-commands-script-runners": major +"pnpm": major +--- + +`pnpm test` should pass all the params after the `test` keyword to the underlying script. This is similar to how `pnpm run test` works [#8619](https://github.com/pnpm/pnpm/pull/8619). diff --git a/.changeset/funny-wolves-enjoy.md b/.changeset/funny-wolves-enjoy.md new file mode 100644 index 00000000000..806fbc4afda --- /dev/null +++ b/.changeset/funny-wolves-enjoy.md @@ -0,0 +1,6 @@ +--- +"pnpm": major +"@pnpm/core": major +--- + +Changed the hash stored in the `packageExtensionsChecksum` field of `pnpm-lock.yaml` to SHA256. diff --git a/.changeset/happy-cheetahs-grab.md b/.changeset/happy-cheetahs-grab.md new file mode 100644 index 00000000000..c11d7f820f2 --- /dev/null +++ b/.changeset/happy-cheetahs-grab.md @@ -0,0 +1,5 @@ +--- +"pnpm": major +--- + +Use an SHA256 hash for the side effects cache keys. diff --git a/.changeset/honest-fans-sip.md b/.changeset/honest-fans-sip.md new file mode 100644 index 00000000000..db6ab29364c --- /dev/null +++ b/.changeset/honest-fans-sip.md @@ -0,0 +1,6 @@ +--- +"@pnpm/config": major +"pnpm": major +--- + +Do not hoist to the root of `node_modules` packages that contain the word `eslint` or `prettier` in their name. Changed the default value of the `public-hoist-pattern` setting [#8378](https://github.com/pnpm/pnpm/issues/8378). diff --git a/.changeset/itchy-horses-wink.md b/.changeset/itchy-horses-wink.md new file mode 100644 index 00000000000..a95ea01fa26 --- /dev/null +++ b/.changeset/itchy-horses-wink.md @@ -0,0 +1,5 @@ +--- +"@pnpm/get-context": major +--- + +Don't validate (and possibly purge) modules directory as a side effect of `getContext` and `getContextForSingleImporter` [#8657](https://github.com/pnpm/pnpm/pull/8657). diff --git a/.changeset/long-peaches-applaud.md b/.changeset/long-peaches-applaud.md new file mode 100644 index 00000000000..371109f539b --- /dev/null +++ b/.changeset/long-peaches-applaud.md @@ -0,0 +1,5 @@ +--- +"pnpm": patch +--- + +Don't validate (and possibly purge) `node_modules` in commands which should not modify it (e.g. `pnpm install --lockfile-only`) [#8657](https://github.com/pnpm/pnpm/pull/8657). diff --git a/.changeset/loud-schools-rhyme.md b/.changeset/loud-schools-rhyme.md new file mode 100644 index 00000000000..d9b58da4f23 --- /dev/null +++ b/.changeset/loud-schools-rhyme.md @@ -0,0 +1,6 @@ +--- +"pnpm": major +"@pnpm/hooks.read-package-hook": major +--- + +Update the compatibility database (`@yarnpkg/extensions` to v2.0.3). This might change your lockfile. diff --git a/.changeset/many-cooks-develop.md b/.changeset/many-cooks-develop.md new file mode 100644 index 00000000000..486d70d539c --- /dev/null +++ b/.changeset/many-cooks-develop.md @@ -0,0 +1,5 @@ +--- +"@pnpm/get-context": major +--- + +`PnpmContext.hoistPattern` and `PnpmContext.publicHoistPattern` are no longer affected by modules directory state [#8657](https://github.com/pnpm/pnpm/pull/8657). Prior behavior can be recreated with the new properties `PnpmContext.currentHoistPattern` (`_.currentHoistPattern ?? _.hoistPattern`) and `PnpmContext.currentPublicHoistPattern` (`_.currentPublicHoistPattern ?? _.publicHoistPattern`). diff --git a/.changeset/nine-cups-smash.md b/.changeset/nine-cups-smash.md new file mode 100644 index 00000000000..25c0a70f1f6 --- /dev/null +++ b/.changeset/nine-cups-smash.md @@ -0,0 +1,6 @@ +--- +"@pnpm/pnpmfile": major +"pnpm": major +--- + +Use SHA256 for storing the pnpmfile checksum in the lockfile [#8530](https://github.com/pnpm/pnpm/pull/8530). diff --git a/.changeset/perfect-spoons-call.md b/.changeset/perfect-spoons-call.md new file mode 100644 index 00000000000..1157d324851 --- /dev/null +++ b/.changeset/perfect-spoons-call.md @@ -0,0 +1,5 @@ +--- +"@pnpm/get-context": major +--- + +`PnpmSingleContext.hoistPattern` and `PnpmSingleContext.publicHoistPattern` are no longer affected by modules directory state [#8657](https://github.com/pnpm/pnpm/pull/8657). diff --git a/.changeset/pretty-houses-refuse.md b/.changeset/pretty-houses-refuse.md new file mode 100644 index 00000000000..ac38a45f673 --- /dev/null +++ b/.changeset/pretty-houses-refuse.md @@ -0,0 +1,5 @@ +--- +"@pnpm/core": patch +--- + +Don't validate (and possibly purge) modules directory in operations that do not mutate the structure (e.g. `mutateModules({ ... }, { ..., lockfileOnly: true })`) [#8657](https://github.com/pnpm/pnpm/pull/8657). diff --git a/.changeset/red-mice-compare.md b/.changeset/red-mice-compare.md new file mode 100644 index 00000000000..4da073b9eed --- /dev/null +++ b/.changeset/red-mice-compare.md @@ -0,0 +1,25 @@ +--- +"@pnpm/plugin-commands-store-inspecting": minor +"@pnpm/package-requester": major +"@pnpm/plugin-commands-rebuild": major +"@pnpm/plugin-commands-store": major +"@pnpm/license-scanner": major +"@pnpm/assert-project": major +"@pnpm/assert-store": major +"@pnpm/mount-modules": minor +"@pnpm/headless": major +"@pnpm/package-store": major +"@pnpm/core": major +"@pnpm/store.cafs": major +"pnpm": major +--- + +Some registries allow identical content to be published under different package names or versions. To accommodate this, index files in the store are now stored using both the content hash and package identifier. + +This approach ensures that we can: +1. Validate that the integrity in the lockfile corresponds to the correct package, + which might not be the case after a poorly resolved Git conflict. +2. Allow the same content to be referenced by different packages or different versions of the same package. + +Related PR: [#8510](https://github.com/pnpm/pnpm/pull/8510) +Related issue: [#8204](https://github.com/pnpm/pnpm/issues/8204) diff --git a/.changeset/short-ravens-clap.md b/.changeset/short-ravens-clap.md new file mode 100644 index 00000000000..d39cc2c75b5 --- /dev/null +++ b/.changeset/short-ravens-clap.md @@ -0,0 +1,5 @@ +--- +"pnpm": major +--- + +Allow passing CLI flags and options to `pnpm test` without `--` [#4821](https://github.com/pnpm/pnpm/issues/4821). diff --git a/.changeset/smart-flowers-cheat.md b/.changeset/smart-flowers-cheat.md new file mode 100644 index 00000000000..14090b68d9d --- /dev/null +++ b/.changeset/smart-flowers-cheat.md @@ -0,0 +1,11 @@ +--- +"@pnpm/plugin-commands-store-inspecting": major +"@pnpm/headless": major +"@pnpm/core": major +"@pnpm/cafs-types": major +"@pnpm/store.cafs": major +"@pnpm/worker": major +"pnpm": major +--- + +Changed the structure of the index files in the store to store side effects cache information more efficiently. In the new version, side effects do not list all the files of the package but just the differences [#8636](https://github.com/pnpm/pnpm/pull/8636). diff --git a/.changeset/soft-wasps-drum.md b/.changeset/soft-wasps-drum.md new file mode 100644 index 00000000000..a29d0ae2f65 --- /dev/null +++ b/.changeset/soft-wasps-drum.md @@ -0,0 +1,5 @@ +--- +"@pnpm/npm-resolver": major +--- + +Use SHA256 to encode the package name of a package that has upper case letters in its name. diff --git a/.changeset/spicy-apricots-beam.md b/.changeset/spicy-apricots-beam.md new file mode 100644 index 00000000000..5a439ccab02 --- /dev/null +++ b/.changeset/spicy-apricots-beam.md @@ -0,0 +1,5 @@ +--- +"@pnpm/get-context": major +--- + +`UnexpectedStoreError` and `UnexpectedVirtualStoreDirError` are no longer exported [#8657](https://github.com/pnpm/pnpm/pull/8657). They can be imported from `@pnpm/core` instead. diff --git a/.changeset/spotty-goats-trade.md b/.changeset/spotty-goats-trade.md new file mode 100644 index 00000000000..80d9eab3edd --- /dev/null +++ b/.changeset/spotty-goats-trade.md @@ -0,0 +1,5 @@ +--- +"@pnpm/plugin-commands-script-runners": major +--- + +Update dlx cache key to use SHA256. diff --git a/.changeset/strange-students-whisper.md b/.changeset/strange-students-whisper.md new file mode 100644 index 00000000000..a1c66c7eeb7 --- /dev/null +++ b/.changeset/strange-students-whisper.md @@ -0,0 +1,5 @@ +--- +"@pnpm/crypto.object-hasher": major +--- + +Use SHA256 encoded in base64 to hash objects. diff --git a/.changeset/tiny-terms-help.md b/.changeset/tiny-terms-help.md new file mode 100644 index 00000000000..7f1200a12bf --- /dev/null +++ b/.changeset/tiny-terms-help.md @@ -0,0 +1,5 @@ +--- +"@pnpm/get-context": major +--- + +Argument `alreadyPurged` removed from `getContextForSingleImporter` [#8657](https://github.com/pnpm/pnpm/pull/8657). diff --git a/.changeset/two-peaches-dress.md b/.changeset/two-peaches-dress.md new file mode 100644 index 00000000000..5c8883df172 --- /dev/null +++ b/.changeset/two-peaches-dress.md @@ -0,0 +1,6 @@ +--- +"@pnpm/config": major +"pnpm": major +--- + +The default value of `virtual-store-dir-max-length` on Windows reduced to 60 characters. diff --git a/.changeset/violet-rockets-pull.md b/.changeset/violet-rockets-pull.md new file mode 100644 index 00000000000..700ad044ced --- /dev/null +++ b/.changeset/violet-rockets-pull.md @@ -0,0 +1,7 @@ +--- +"@pnpm/dependency-path": major +"@pnpm/core": major +pnpm: major +--- + +Escape the `#` character in directory names within the virtual store (`node_modules/.pnpm`) [#8557](https://github.com/pnpm/pnpm/pull/8557). diff --git a/.changeset/wise-buses-roll.md b/.changeset/wise-buses-roll.md new file mode 100644 index 00000000000..8a0ec6ff71e --- /dev/null +++ b/.changeset/wise-buses-roll.md @@ -0,0 +1,6 @@ +--- +"@pnpm/constants": major +"pnpm": major +--- + +Store version bumped to v10. The new store layout has a different directory called "index" for storing the package content mappings. Previously these files were stored in the same directory where the package contents are (in "files"). The new store has also a new format for storing the mappings for side-effects cache. diff --git a/.meta-updater/src/index.ts b/.meta-updater/src/index.ts index b25343ee49c..8163ef305e4 100644 --- a/.meta-updater/src/index.ts +++ b/.meta-updater/src/index.ts @@ -8,7 +8,7 @@ import loadJsonFile from 'load-json-file' import normalizePath from 'normalize-path' import writeJsonFile from 'write-json-file' -const NEXT_TAG = 'next-9' +const NEXT_TAG = 'next-10' const CLI_PKG_NAME = 'pnpm' export default async (workspaceDir: string) => { // eslint-disable-line diff --git a/__utils__/assert-project/src/index.ts b/__utils__/assert-project/src/index.ts index 645fdd8974a..28abd3aff20 100644 --- a/__utils__/assert-project/src/index.ts +++ b/__utils__/assert-project/src/index.ts @@ -20,9 +20,9 @@ export interface Project { hasNot: (pkgName: string, modulesDir?: string) => void getStorePath: () => string resolve: (pkgName: string, version?: string, relativePath?: string) => string - getPkgIndexFilePath: (pkgName: string, version?: string) => string - cafsHas: (pkgName: string, version?: string) => void - cafsHasNot: (pkgName: string, version?: string) => void + getPkgIndexFilePath: (pkgName: string, version: string) => string + cafsHas: (pkgName: string, version: string) => void + cafsHasNot: (pkgName: string, version: string) => void storeHas: (pkgName: string, version?: string) => string storeHasNot: (pkgName: string, version?: string) => void isExecutable: (pathToExe: string) => void @@ -48,9 +48,9 @@ export function assertProject (projectPath: string, encodedRegistryName?: string interface StoreInstance { storePath: string - getPkgIndexFilePath: (pkgName: string, version?: string) => string - cafsHas: (pkgName: string, version?: string) => void - cafsHasNot: (pkgName: string, version?: string) => void + getPkgIndexFilePath: (pkgName: string, version: string) => string + cafsHas: (pkgName: string, version: string) => void + cafsHasNot: (pkgName: string, version: string) => void storeHas: (pkgName: string, version?: string) => void storeHasNot: (pkgName: string, version?: string) => void resolve: (pkgName: string, version?: string, relativePath?: string) => string @@ -107,15 +107,15 @@ export function assertProject (projectPath: string, encodedRegistryName?: string const store = getStoreInstance() return store.resolve(pkgName, version, relativePath) }, - getPkgIndexFilePath (pkgName: string, version?: string): string { + getPkgIndexFilePath (pkgName: string, version: string): string { const store = getStoreInstance() return store.getPkgIndexFilePath(pkgName, version) }, - cafsHas (pkgName: string, version?: string) { + cafsHas (pkgName: string, version: string) { const store = getStoreInstance() store.cafsHas(pkgName, version) }, - cafsHasNot (pkgName: string, version?: string) { + cafsHasNot (pkgName: string, version: string) { const store = getStoreInstance() store.cafsHasNot(pkgName, version) }, diff --git a/__utils__/assert-store/package.json b/__utils__/assert-store/package.json index 2abfa7520eb..e7bb6e896ee 100644 --- a/__utils__/assert-store/package.json +++ b/__utils__/assert-store/package.json @@ -2,18 +2,6 @@ "name": "@pnpm/assert-store", "description": "Utils for testing pnpm store", "version": "1.0.92", - "author": { - "name": "Zoltan Kochan", - "email": "z@kochan.io", - "url": "https://www.kochan.io/" - }, - "contributors": [ - { - "name": "Tejasvi Nareddy", - "email": "tejunareddy@gmail.com", - "url": "https://www.tejunareddy.com" - } - ], "bugs": { "url": "https://github.com/pnpm/pnpm/issues" }, @@ -44,6 +32,7 @@ "@pnpm/store.cafs": "workspace:*" }, "devDependencies": { - "@pnpm/assert-store": "workspace:*" + "@pnpm/assert-store": "workspace:*", + "@pnpm/constants": "workspace:*" } } diff --git a/__utils__/assert-store/src/index.ts b/__utils__/assert-store/src/index.ts index 14efc0f84ea..b8b1b49b427 100644 --- a/__utils__/assert-store/src/index.ts +++ b/__utils__/assert-store/src/index.ts @@ -4,9 +4,9 @@ import { getIndexFilePathInCafs } from '@pnpm/store.cafs' import { getIntegrity, REGISTRY_MOCK_PORT } from '@pnpm/registry-mock' export interface StoreAssertions { - getPkgIndexFilePath: (pkgName: string, version?: string) => string - cafsHas: (pkgName: string, version?: string) => void - cafsHasNot: (pkgName: string, version?: string) => void + getPkgIndexFilePath: (pkgName: string, version: string) => string + cafsHas: (pkgName: string, version: string) => void + cafsHasNot: (pkgName: string, version: string) => void storeHas: (pkgName: string, version?: string) => void storeHasNot: (pkgName: string, version?: string) => void resolve: (pkgName: string, version?: string, relativePath?: string) => string @@ -22,16 +22,15 @@ export function assertStore ( const notOk = (value: any) => expect(value).toBeFalsy() const ern = encodedRegistryName ?? `localhost+${REGISTRY_MOCK_PORT}` const store = { - getPkgIndexFilePath (pkgName: string, version?: string): string { - const cafsDir = path.join(storePath, 'files') - const integrity = version ? getIntegrity(pkgName, version) : pkgName - return getIndexFilePathInCafs(cafsDir, integrity) + getPkgIndexFilePath (pkgName: string, version: string): string { + const integrity = getIntegrity(pkgName, version) + return getIndexFilePathInCafs(storePath, integrity, `${pkgName}@${version}`) }, - cafsHas (pkgName: string, version?: string): void { + cafsHas (pkgName: string, version: string): void { const pathToCheck = store.getPkgIndexFilePath(pkgName, version) ok(fs.existsSync(pathToCheck)) }, - cafsHasNot (pkgName: string, version?: string): void { + cafsHasNot (pkgName: string, version: string): void { const pathToCheck = store.getPkgIndexFilePath(pkgName, version) notOk(fs.existsSync(pathToCheck)) }, diff --git a/__utils__/assert-store/test/index.ts b/__utils__/assert-store/test/index.ts index 72f58707a28..9dbc614b455 100644 --- a/__utils__/assert-store/test/index.ts +++ b/__utils__/assert-store/test/index.ts @@ -1,9 +1,10 @@ /// import path from 'path' import { assertStore } from '@pnpm/assert-store' +import { STORE_VERSION } from '@pnpm/constants' test('assertStore() store assertions', async () => { - const storePath = path.join(__dirname, 'fixture/store/v3/') + const storePath = path.join(__dirname, `fixture/store/${STORE_VERSION}/`) const encodedRegistryName = 'registry.npmjs.org' const store = assertStore(storePath, encodedRegistryName) @@ -13,7 +14,7 @@ test('assertStore() store assertions', async () => { }) test('assertStore() resolve', async () => { - const storePath = path.join(__dirname, 'fixture/store/v3/') + const storePath = path.join(__dirname, `fixture/store/${STORE_VERSION}/`) const encodedRegistryName = 'registry.npmjs.org' const store = assertStore(storePath, encodedRegistryName) diff --git a/__utils__/assert-store/tsconfig.json b/__utils__/assert-store/tsconfig.json index 1987344781b..aa2e467e0d2 100644 --- a/__utils__/assert-store/tsconfig.json +++ b/__utils__/assert-store/tsconfig.json @@ -9,6 +9,9 @@ "../../__typings__/**/*.d.ts" ], "references": [ + { + "path": "../../packages/constants" + }, { "path": "../../store/cafs" } diff --git a/cache/api/src/cacheView.ts b/cache/api/src/cacheView.ts index 4ba10806213..c4b09c54fa0 100644 --- a/cache/api/src/cacheView.ts +++ b/cache/api/src/cacheView.ts @@ -17,7 +17,6 @@ export async function cacheView (opts: { cacheDir: string, storeDir: string, reg const metaFilePaths = (await fastGlob(`${prefix}/${packageName}.json`, { cwd: opts.cacheDir, })).sort() - const cafsDir = path.join(opts.storeDir, 'files') const metaFilesByPath: Record = {} for (const filePath of metaFilePaths) { const metaObject = JSON.parse(fs.readFileSync(path.join(opts.cacheDir, filePath), 'utf8')) as PackageMeta @@ -25,7 +24,7 @@ export async function cacheView (opts: { cacheDir: string, storeDir: string, reg const nonCachedVersions: string[] = [] for (const [version, manifest] of Object.entries(metaObject.versions)) { if (!manifest.dist.integrity) continue - const indexFilePath = getIndexFilePathInCafs(cafsDir, manifest.dist.integrity) + const indexFilePath = getIndexFilePathInCafs(opts.storeDir, manifest.dist.integrity, `${manifest.name}@${manifest.version}`) if (fs.existsSync(indexFilePath)) { cachedVersions.push(version) } else { diff --git a/config/config/src/Config.ts b/config/config/src/Config.ts index 262f2b0bec3..46b47877455 100644 --- a/config/config/src/Config.ts +++ b/config/config/src/Config.ts @@ -197,6 +197,7 @@ export interface Config { extendNodePath?: boolean gitBranchLockfile?: boolean globalDir?: string + globalPkgDir: string lockfile?: boolean dedupeInjectedDeps?: boolean nodeOptions?: string diff --git a/config/config/src/index.ts b/config/config/src/index.ts index 08362da9b09..1929f8df12b 100644 --- a/config/config/src/index.ts +++ b/config/config/src/index.ts @@ -148,7 +148,7 @@ export async function getConfig (opts: { 'ignore-workspace-root-check': false, 'link-workspace-packages': false, 'lockfile-include-tarball-url': false, - 'manage-package-manager-versions': false, + 'manage-package-manager-versions': true, 'modules-cache-max-age': 7 * 24 * 60, // 7 days 'dlx-cache-max-age': 24 * 60, // 1 day 'node-linker': 'isolated', @@ -157,10 +157,7 @@ export async function getConfig (opts: { 'package-manager-strict': process.env.COREPACK_ENABLE_STRICT !== '0', 'package-manager-strict-version': false, 'prefer-workspace-packages': false, - 'public-hoist-pattern': [ - '*eslint*', - '*prettier*', - ], + 'public-hoist-pattern': [], 'recursive-install': true, registry: npmDefaults.registry, 'resolution-mode': 'highest', @@ -185,7 +182,7 @@ export async function getConfig (opts: { 'workspace-prefix': opts.workspaceDir, 'embed-readme': false, 'registry-supports-time-field': false, - 'virtual-store-dir-max-length': 120, + 'virtual-store-dir-max-length': isWindows() ? 60 : 120, 'peers-suffix-max-length': 1000, } @@ -263,16 +260,16 @@ export async function getConfig (opts: { return undefined })() pnpmConfig.pnpmHomeDir = getDataDir(process) + let globalDirRoot + if (pnpmConfig.globalDir) { + globalDirRoot = pnpmConfig.globalDir + } else { + globalDirRoot = path.join(pnpmConfig.pnpmHomeDir, 'global') + } + pnpmConfig.globalPkgDir = path.join(globalDirRoot, LAYOUT_VERSION.toString()) if (cliOptions['global']) { - let globalDirRoot - if (pnpmConfig.globalDir) { - globalDirRoot = pnpmConfig.globalDir - } else { - globalDirRoot = path.join(pnpmConfig.pnpmHomeDir, 'global') - } - pnpmConfig.dir = path.join(globalDirRoot, LAYOUT_VERSION.toString()) - + pnpmConfig.dir = pnpmConfig.globalPkgDir pnpmConfig.bin = npmConfig.get('global-bin-dir') ?? env.PNPM_HOME if (pnpmConfig.bin) { fs.mkdirSync(pnpmConfig.bin, { recursive: true }) diff --git a/crypto/hash/README.md b/crypto/hash/README.md new file mode 100644 index 00000000000..5bc7a8fd3f1 --- /dev/null +++ b/crypto/hash/README.md @@ -0,0 +1,13 @@ +# @pnpm/crypto.hash + +> Generate hashes + +## Installation + +```sh +pnpm add @pnpm/crypto.hash +``` + +## License + +MIT diff --git a/packages/crypto.base32-hash/package.json b/crypto/hash/package.json similarity index 63% rename from packages/crypto.base32-hash/package.json rename to crypto/hash/package.json index 6eb15fc0cfe..5ed959b65b2 100644 --- a/packages/crypto.base32-hash/package.json +++ b/crypto/hash/package.json @@ -1,7 +1,7 @@ { - "name": "@pnpm/crypto.base32-hash", - "version": "3.0.1", - "description": "Create a base32 hash", + "name": "@pnpm/crypto.hash", + "version": "0.0.0", + "description": "Generate hashes", "main": "lib/index.js", "types": "lib/index.d.ts", "files": [ @@ -15,12 +15,11 @@ "prepublishOnly": "pnpm run compile", "compile": "tsc --build && pnpm run lint --fix" }, - "repository": "https://github.com/pnpm/pnpm/blob/main/packages/crypto.base32-hash", + "repository": "https://github.com/pnpm/pnpm/blob/main/crypto/hash", "keywords": [ "pnpm9", "hash", - "crypto", - "base32" + "crypto" ], "engines": { "node": ">=18.12" @@ -29,13 +28,12 @@ "bugs": { "url": "https://github.com/pnpm/pnpm/issues" }, - "homepage": "https://github.com/pnpm/pnpm/blob/main/packages/crypto.base32-hash#readme", + "homepage": "https://github.com/pnpm/pnpm/blob/main/crypto/hash#readme", "dependencies": { - "@pnpm/crypto.polyfill": "workspace:*", - "rfc4648": "catalog:" + "@pnpm/crypto.polyfill": "workspace:*" }, "devDependencies": { - "@pnpm/crypto.base32-hash": "workspace:*", + "@pnpm/crypto.hash": "workspace:*", "@pnpm/prepare": "workspace:*" }, "funding": "https://opencollective.com/pnpm", diff --git a/crypto/hash/src/index.ts b/crypto/hash/src/index.ts new file mode 100644 index 00000000000..865c1537614 --- /dev/null +++ b/crypto/hash/src/index.ts @@ -0,0 +1,27 @@ +import * as crypto from '@pnpm/crypto.polyfill' +import fs from 'fs' + +export function createShortHash (input: string): string { + return createHexHash(input).substring(0, 32) +} + +export function createHexHash (input: string): string { + return crypto.hash('sha256', input, 'hex') +} + +export function createHash (input: string): string { + return `sha256-${crypto.hash('sha256', input, 'base64')}` +} + +export async function createHashFromFile (file: string): Promise { + return createHash(await readNormalizedFile(file)) +} + +export async function createHexHashFromFile (file: string): Promise { + return createHexHash(await readNormalizedFile(file)) +} + +async function readNormalizedFile (file: string): Promise { + const content = await fs.promises.readFile(file, 'utf8') + return content.split('\r\n').join('\n') +} diff --git a/crypto/hash/test/index.ts b/crypto/hash/test/index.ts new file mode 100644 index 00000000000..9ccadb664f3 --- /dev/null +++ b/crypto/hash/test/index.ts @@ -0,0 +1,15 @@ +/// +import fs from 'fs' +import { createShortHash, createHashFromFile } from '@pnpm/crypto.hash' +import { tempDir } from '@pnpm/prepare' + +test('createShortHash()', () => { + expect(createShortHash('AAA')).toEqual('cb1ad2119d8fafb69566510ee712661f') +}) + +test('createHashFromFile normalizes line endings before calculating the hash', async () => { + tempDir() + fs.writeFileSync('win-eol.txt', 'a\r\nb\r\nc') + fs.writeFileSync('posix-eol.txt', 'a\nb\r\nc') + expect(await createHashFromFile('win-eol.txt')).toEqual(await createHashFromFile('posix-eol.txt')) +}) diff --git a/packages/crypto.base32-hash/test/tsconfig.json b/crypto/hash/test/tsconfig.json similarity index 100% rename from packages/crypto.base32-hash/test/tsconfig.json rename to crypto/hash/test/tsconfig.json diff --git a/packages/crypto.base32-hash/tsconfig.json b/crypto/hash/tsconfig.json similarity index 87% rename from packages/crypto.base32-hash/tsconfig.json rename to crypto/hash/tsconfig.json index 257b59d3f2f..e7cb0ace06b 100644 --- a/packages/crypto.base32-hash/tsconfig.json +++ b/crypto/hash/tsconfig.json @@ -13,7 +13,7 @@ "path": "../../__utils__/prepare" }, { - "path": "../../crypto/polyfill" + "path": "../polyfill" } ] } diff --git a/packages/crypto.base32-hash/tsconfig.lint.json b/crypto/hash/tsconfig.lint.json similarity index 100% rename from packages/crypto.base32-hash/tsconfig.lint.json rename to crypto/hash/tsconfig.lint.json diff --git a/crypto/object-hasher/package.json b/crypto/object-hasher/package.json index ad86f75e96c..e3630f9629f 100644 --- a/crypto/object-hasher/package.json +++ b/crypto/object-hasher/package.json @@ -30,11 +30,13 @@ }, "homepage": "https://github.com/pnpm/pnpm/blob/main/crypto/object-hasher#readme", "dependencies": { - "object-hash": "catalog:" + "object-hash": "catalog:", + "ramda": "catalog:" }, "devDependencies": { "@pnpm/crypto.object-hasher": "workspace:*", - "@types/object-hash": "catalog:" + "@types/object-hash": "catalog:", + "@types/ramda": "catalog:" }, "funding": "https://opencollective.com/pnpm", "exports": { diff --git a/crypto/object-hasher/src/index.ts b/crypto/object-hasher/src/index.ts index d2f9945e683..91ba1f214b7 100644 --- a/crypto/object-hasher/src/index.ts +++ b/crypto/object-hasher/src/index.ts @@ -1,21 +1,24 @@ +import isEmpty from 'ramda/src/isEmpty' + // We use object-hash even though node-object-hash is faster. // Unlike node-object-hash, object-hash is streaming the hash updates, // avoiding "Invalid string length" errors. import hash from 'object-hash' -const defaultOptions: hash.BaseOptions = { +const defaultOptions: hash.NormalOption = { respectType: false, - algorithm: 'sha1', + algorithm: 'sha256', + encoding: 'base64', } -const withoutSortingOptions: hash.BaseOptions = { +const withoutSortingOptions: hash.NormalOption = { ...defaultOptions, unorderedArrays: false, unorderedObjects: false, unorderedSets: false, } -const withSortingOptions: hash.BaseOptions = { +const withSortingOptions: hash.NormalOption = { ...defaultOptions, unorderedArrays: true, unorderedObjects: true, @@ -24,11 +27,18 @@ const withSortingOptions: hash.BaseOptions = { function hashUnknown (object: unknown, options: hash.BaseOptions): string { if (object === undefined) { - // '0'.repeat(40) to match the length of other returned sha1 hashes. - return '0000000000000000000000000000000000000000' + // '0'.repeat(44) to match the length of other returned sha1 hashes. + return '00000000000000000000000000000000000000000000' } return hash(object, options) } export const hashObjectWithoutSorting = (object: unknown): string => hashUnknown(object, withoutSortingOptions) export const hashObject = (object: unknown): string => hashUnknown(object, withSortingOptions) + +export type PrefixedHash = `sha256-${string}` +export function hashObjectNullableWithPrefix (object: Record | undefined): PrefixedHash | undefined { + if (!object || isEmpty(object)) return undefined + const packageExtensionsChecksum = hash(object, withSortingOptions) + return `sha256-${packageExtensionsChecksum}` +} diff --git a/crypto/object-hasher/test/index.ts b/crypto/object-hasher/test/index.ts index 40dd99bb41e..0a30602aa52 100644 --- a/crypto/object-hasher/test/index.ts +++ b/crypto/object-hasher/test/index.ts @@ -1,10 +1,10 @@ -import { hashObject, hashObjectWithoutSorting } from '@pnpm/crypto.object-hasher' +import { hashObject, hashObjectWithoutSorting, hashObjectNullableWithPrefix } from '@pnpm/crypto.object-hasher' describe('hashObject', () => { const hash = hashObject it('creates a hash', () => { - expect(hash({ b: 1, a: 2 })).toEqual('e3d3f89836fac144779e57d0e831efd06336036b') - expect(hash(undefined)).toEqual('0000000000000000000000000000000000000000') + expect(hash({ b: 1, a: 2 })).toEqual('48AVoXIXcTKcnHt8qVKp5vNw4gyOB5VfztHwtYBRcAQ=') + expect(hash(undefined)).toEqual('00000000000000000000000000000000000000000000') }) it('sorts', () => { expect(hash({ b: 1, a: 2 })).toEqual(hash({ a: 2, b: 1 })) @@ -15,11 +15,24 @@ describe('hashObject', () => { describe('hashObjectWithoutSorting', () => { const hash = hashObjectWithoutSorting it('creates a hash', () => { - expect(hash({ b: 1, a: 2 })).toEqual('dd34c1644a1d52da41808e5c1e6849829ef77999') - expect(hash(undefined)).toEqual('0000000000000000000000000000000000000000') + expect(hash({ b: 1, a: 2 })).toEqual('mh+rYklpd1DBj/dg6dnG+yd8BQhU2UiUoRMSXjPV1JA=') + expect(hash(undefined)).toEqual('00000000000000000000000000000000000000000000') }) it('does not sort', () => { expect(hash({ b: 1, a: 2 })).not.toEqual(hash({ a: 2, b: 1 })) expect(hash({ b: new Set([1, 2, 3]), a: [1, 2, 3] })).not.toEqual(hash({ a: [2, 3, 1], b: new Set([3, 2, 1]) })) }) }) + +describe('hashObjectNullableWithPrefix', () => { + const hash = hashObjectNullableWithPrefix + it('creates a hash', () => { + expect(hash({ b: 1, a: 2 })).toStrictEqual('sha256-48AVoXIXcTKcnHt8qVKp5vNw4gyOB5VfztHwtYBRcAQ=') + expect(hash({})).toStrictEqual(undefined) + expect(hash(undefined)).toStrictEqual(undefined) + }) + it('sorts', () => { + expect(hash({ b: 1, a: 2 })).toStrictEqual(hash({ a: 2, b: 1 })) + expect(hash({ b: new Set([1, 2, 3]), a: [1, 2, 3] })).toStrictEqual(hash({ a: [2, 3, 1], b: new Set([3, 2, 1]) })) + }) +}) diff --git a/env/node.fetcher/src/index.ts b/env/node.fetcher/src/index.ts index e131d40fcae..f7b23fb48a1 100644 --- a/env/node.fetcher/src/index.ts +++ b/env/node.fetcher/src/index.ts @@ -15,7 +15,7 @@ import { isNonGlibcLinux } from 'detect-libc' import { getNodeTarball } from './getNodeTarball' export interface FetchNodeOptions { - cafsDir: string + storeDir: string fetchTimeout?: number nodeMirrorBaseUrl?: string retry?: RetryTimeoutOptions @@ -39,10 +39,10 @@ export async function fetchNode (fetch: FetchFromRegistry, version: string, targ rawConfig: {}, unsafePerm: false, }) - const cafs = createCafsStore(opts.cafsDir) + const cafs = createCafsStore(opts.storeDir) const fetchTarball = pickFetcher(fetchers, { tarball }) const { filesIndex } = await fetchTarball(cafs, { tarball } as any, { // eslint-disable-line @typescript-eslint/no-explicit-any - filesIndexFile: path.join(opts.cafsDir, encodeURIComponent(tarball)), // TODO: change the name or don't save an index file for node.js tarballs + filesIndexFile: path.join(opts.storeDir, encodeURIComponent(tarball)), // TODO: change the name or don't save an index file for node.js tarballs lockfileDir: process.cwd(), pkg: {}, }) diff --git a/env/node.fetcher/test/node.test.ts b/env/node.fetcher/test/node.test.ts index 1b4bcee20c8..a0366a3a994 100644 --- a/env/node.fetcher/test/node.test.ts +++ b/env/node.fetcher/test/node.test.ts @@ -35,7 +35,7 @@ test.skip('install Node using a custom node mirror', async () => { const nodeMirrorBaseUrl = 'https://pnpm-node-mirror-test.localhost/download/release/' const opts: FetchNodeOptions = { nodeMirrorBaseUrl, - cafsDir: path.resolve('files'), + storeDir: path.resolve('store'), } await fetchNode(fetchMock, '16.4.0', path.resolve('node'), opts) @@ -49,7 +49,7 @@ test.skip('install Node using the default node mirror', async () => { tempDir() const opts: FetchNodeOptions = { - cafsDir: path.resolve('files'), + storeDir: path.resolve('store'), } await fetchNode(fetchMock, '16.4.0', path.resolve('node'), opts) @@ -64,7 +64,7 @@ test('install Node using a custom node mirror', async () => { tempDir() const opts: FetchNodeOptions = { - cafsDir: path.resolve('files'), + storeDir: path.resolve('store'), } await expect( diff --git a/env/plugin-commands-env/src/node.ts b/env/plugin-commands-env/src/node.ts index 899f78e2ff2..06893981988 100644 --- a/env/plugin-commands-env/src/node.ts +++ b/env/plugin-commands-env/src/node.ts @@ -97,11 +97,10 @@ export async function getNodeDir (fetch: FetchFromRegistry, opts: NvmNodeCommand storePath: opts.storeDir, pnpmHomeDir: opts.pnpmHomeDir, }) - const cafsDir = path.join(storeDir, 'files') globalInfo(`Fetching Node.js ${opts.useNodeVersion} ...`) await fetchNode(fetch, opts.useNodeVersion, versionDir, { ...opts, - cafsDir, + storeDir, retry: { maxTimeout: opts.fetchRetryMaxtimeout, minTimeout: opts.fetchRetryMintimeout, diff --git a/exec/plugin-commands-rebuild/src/implementation/index.ts b/exec/plugin-commands-rebuild/src/implementation/index.ts index 306a39e8a8d..fd6cf60a751 100644 --- a/exec/plugin-commands-rebuild/src/implementation/index.ts +++ b/exec/plugin-commands-rebuild/src/implementation/index.ts @@ -249,7 +249,6 @@ async function _rebuild ( ): Promise> { const depGraph = lockfileToDepGraph(ctx.currentLockfile) const depsStateCache: DepsStateCache = {} - const cafsDir = path.join(opts.storeDir, 'files') const pkgsThatWereRebuilt = new Set() const graph = new Map() const pkgSnapshots: PackageSnapshots = ctx.currentLockfile.packages ?? {} @@ -315,8 +314,9 @@ async function _rebuild ( } const resolution = (pkgSnapshot.resolution as TarballResolution) let sideEffectsCacheKey: string | undefined + const pkgId = `${pkgInfo.name}@${pkgInfo.version}` if (opts.skipIfHasSideEffectsCache && resolution.integrity) { - const filesIndexFile = getIndexFilePathInCafs(cafsDir, resolution.integrity!.toString()) + const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, resolution.integrity!.toString(), pkgId) const pkgFilesIndex = await loadJsonFile(filesIndexFile) sideEffectsCacheKey = calcDepState(depGraph, depsStateCache, depPath, { isBuilt: true, @@ -341,7 +341,7 @@ async function _rebuild ( }) if (hasSideEffects && (opts.sideEffectsCacheWrite ?? true) && resolution.integrity) { builtDepPaths.add(depPath) - const filesIndexFile = getIndexFilePathInCafs(cafsDir, resolution.integrity!.toString()) + const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, resolution.integrity!.toString(), pkgId) try { if (!sideEffectsCacheKey) { sideEffectsCacheKey = calcDepState(depGraph, depsStateCache, depPath, { diff --git a/exec/plugin-commands-rebuild/test/index.ts b/exec/plugin-commands-rebuild/test/index.ts index 95b87cc44f9..127c2f9bf61 100644 --- a/exec/plugin-commands-rebuild/test/index.ts +++ b/exec/plugin-commands-rebuild/test/index.ts @@ -2,7 +2,7 @@ import fs from 'fs' import path from 'path' import { getIndexFilePathInCafs } from '@pnpm/store.cafs' -import { ENGINE_NAME, WANTED_LOCKFILE } from '@pnpm/constants' +import { ENGINE_NAME, STORE_VERSION, WANTED_LOCKFILE } from '@pnpm/constants' import { hashObject } from '@pnpm/crypto.object-hasher' import { rebuild } from '@pnpm/plugin-commands-rebuild' import { prepare } from '@pnpm/prepare' @@ -74,13 +74,12 @@ test('rebuilds dependencies', async () => { ]) } - const cafsDir = path.join(storeDir, 'v3/files') - const cacheIntegrityPath = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')) + const cacheIntegrityPath = getIndexFilePathInCafs(path.join(storeDir, STORE_VERSION), getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') const cacheIntegrity = loadJsonFile.sync(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any expect(cacheIntegrity!.sideEffects).toBeTruthy() - const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}` - expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'generated-by-postinstall.js']) - delete cacheIntegrity!.sideEffects[sideEffectsKey]['generated-by-postinstall.js'] + const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}` + expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'added', 'generated-by-postinstall.js']) + delete cacheIntegrity!.sideEffects[sideEffectsKey].added['generated-by-postinstall.js'] }) test('skipIfHasSideEffectsCache', async () => { @@ -99,12 +98,11 @@ test('skipIfHasSideEffectsCache', async () => { `--cache-dir=${cacheDir}`, ]) - const cafsDir = path.join(storeDir, 'v3/files') - const cacheIntegrityPath = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')) + const cacheIntegrityPath = getIndexFilePathInCafs(path.join(storeDir, STORE_VERSION), getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') let cacheIntegrity = loadJsonFile.sync(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any - const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}` + const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}` cacheIntegrity.sideEffects = { - [sideEffectsKey]: { foo: 'bar' }, + [sideEffectsKey]: { added: { foo: 'bar' } }, } fs.writeFileSync(cacheIntegrityPath, JSON.stringify(cacheIntegrity, null, 2), 'utf8') @@ -130,7 +128,7 @@ test('skipIfHasSideEffectsCache', async () => { cacheIntegrity = loadJsonFile.sync(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any expect(cacheIntegrity!.sideEffects).toBeTruthy() - expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'foo']) + expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'added', 'foo']) }) test('rebuild does not fail when a linked package is present', async () => { diff --git a/exec/plugin-commands-rebuild/test/utils/index.ts b/exec/plugin-commands-rebuild/test/utils/index.ts index 57c43cb49d8..9a6d5d68138 100644 --- a/exec/plugin-commands-rebuild/test/utils/index.ts +++ b/exec/plugin-commands-rebuild/test/utils/index.ts @@ -52,5 +52,5 @@ export const DEFAULT_OPTS = { cpu: ['current'], libc: ['current'], }, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, } diff --git a/exec/plugin-commands-script-runners/package.json b/exec/plugin-commands-script-runners/package.json index 16e4108cff4..aec0fc07c21 100644 --- a/exec/plugin-commands-script-runners/package.json +++ b/exec/plugin-commands-script-runners/package.json @@ -50,7 +50,7 @@ "@pnpm/common-cli-options-help": "workspace:*", "@pnpm/config": "workspace:*", "@pnpm/core-loggers": "workspace:*", - "@pnpm/crypto.base32-hash": "workspace:*", + "@pnpm/crypto.hash": "workspace:*", "@pnpm/env.path": "workspace:*", "@pnpm/error": "workspace:*", "@pnpm/lifecycle": "workspace:*", diff --git a/exec/plugin-commands-script-runners/src/dlx.ts b/exec/plugin-commands-script-runners/src/dlx.ts index d8bcb0a70a5..4274ef7e080 100644 --- a/exec/plugin-commands-script-runners/src/dlx.ts +++ b/exec/plugin-commands-script-runners/src/dlx.ts @@ -4,7 +4,7 @@ import util from 'util' import { docsUrl } from '@pnpm/cli-utils' import { OUTPUT_OPTIONS } from '@pnpm/common-cli-options-help' import { type Config, types } from '@pnpm/config' -import { createBase32Hash } from '@pnpm/crypto.base32-hash' +import { createHexHash } from '@pnpm/crypto.hash' import { PnpmError } from '@pnpm/error' import { add } from '@pnpm/plugin-commands-installation' import { readPackageJsonFromDir } from '@pnpm/read-package-json' @@ -200,7 +200,7 @@ export function createCacheKey (pkgs: string[], registries: Record a.localeCompare(b)) const sortedRegistries = Object.entries(registries).sort(([k1], [k2]) => k1.localeCompare(k2)) const hashStr = JSON.stringify([sortedPkgs, sortedRegistries]) - return createBase32Hash(hashStr) + return createHexHash(hashStr) } function getValidCacheDir (cacheLink: string, dlxCacheMaxAge: number): string | undefined { diff --git a/exec/plugin-commands-script-runners/src/index.ts b/exec/plugin-commands-script-runners/src/index.ts index 2c7207aed7f..b09a1e1b139 100644 --- a/exec/plugin-commands-script-runners/src/index.ts +++ b/exec/plugin-commands-script-runners/src/index.ts @@ -3,11 +3,5 @@ import * as dlx from './dlx' import * as exec from './exec' import * as restart from './restart' import * as run from './run' -import * as _test from './test' -const test = { - ...run, - ..._test, -} - -export { create, dlx, exec, restart, run, test } +export { create, dlx, exec, restart, run } diff --git a/exec/plugin-commands-script-runners/src/run.ts b/exec/plugin-commands-script-runners/src/run.ts index 8c7d9572bb7..b72366e633b 100644 --- a/exec/plugin-commands-script-runners/src/run.ts +++ b/exec/plugin-commands-script-runners/src/run.ts @@ -187,7 +187,7 @@ export async function handler ( params: string[] ): Promise { let dir: string - const [scriptName, ...passedThruArgs] = params + let [scriptName, ...passedThruArgs] = params if (opts.recursive) { if (scriptName || Object.keys(opts.selectedProjectsGraph).length > 1) { return runRecursive(params, opts) as Promise @@ -203,6 +203,9 @@ export async function handler ( : undefined return printProjectCommands(manifest, rootManifest ?? undefined) } + if (opts.fallbackCommandUsed && (scriptName === 't' || scriptName === 'tst')) { + scriptName = 'test' + } const specifiedScripts = getSpecifiedScripts(manifest.scripts ?? {}, scriptName) diff --git a/exec/plugin-commands-script-runners/src/test.ts b/exec/plugin-commands-script-runners/src/test.ts deleted file mode 100644 index 5d1cf00c3a4..00000000000 --- a/exec/plugin-commands-script-runners/src/test.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { docsUrl } from '@pnpm/cli-utils' -import { FILTERING } from '@pnpm/common-cli-options-help' -import renderHelp from 'render-help' -import * as run from './run' - -export const commandNames = ['test', 't', 'tst'] - -export function help (): string { - return renderHelp({ - aliases: ['t', 'tst'], - description: 'Runs a package\'s "test" script, if one was provided.', - descriptionLists: [ - { - title: 'Options', - - list: [ - { - description: '\ -Run the tests in every package found in subdirectories \ -or every workspace package, when executed inside a workspace. \ -For options that may be used with `-r`, see "pnpm help recursive"', - name: '--recursive', - shortAlias: '-r', - }, - ], - }, - FILTERING, - ], - url: docsUrl('test'), - usages: ['pnpm test [-- ...]'], - }) -} - -export async function handler ( - opts: run.RunOpts, - params: string[] = [] -): Promise { - return run.handler(opts, ['test', ...params]) -} diff --git a/exec/plugin-commands-script-runners/test/dlx.createCacheKey.test.ts b/exec/plugin-commands-script-runners/test/dlx.createCacheKey.test.ts index 86076375dbc..f8d57c93d09 100644 --- a/exec/plugin-commands-script-runners/test/dlx.createCacheKey.test.ts +++ b/exec/plugin-commands-script-runners/test/dlx.createCacheKey.test.ts @@ -1,4 +1,4 @@ -import { createBase32Hash } from '@pnpm/crypto.base32-hash' +import { createHexHash } from '@pnpm/crypto.hash' import { createCacheKey } from '../src/dlx' test('creates a hash', () => { @@ -6,7 +6,7 @@ test('creates a hash', () => { default: 'https://registry.npmjs.com/', '@foo': 'https://example.com/npm-registry/foo/', }) - const expected = createBase32Hash(JSON.stringify([['@foo/bar', 'shx'], [ + const expected = createHexHash(JSON.stringify([['@foo/bar', 'shx'], [ ['@foo', 'https://example.com/npm-registry/foo/'], ['default', 'https://registry.npmjs.com/'], ]])) diff --git a/exec/plugin-commands-script-runners/test/index.ts b/exec/plugin-commands-script-runners/test/index.ts index 539549a2517..228a5229764 100644 --- a/exec/plugin-commands-script-runners/test/index.ts +++ b/exec/plugin-commands-script-runners/test/index.ts @@ -6,7 +6,6 @@ import { filterPackagesFromDir } from '@pnpm/workspace.filter-packages-from-dir' import { restart, run, - test as testCommand, } from '@pnpm/plugin-commands-script-runners' import { prepare, preparePackages } from '@pnpm/prepare' import { createTestIpcServer } from '@pnpm/test-ipc-server' @@ -137,14 +136,14 @@ test('test: pass the args to the command that is specified in the build script o fs.writeFileSync('args.json', '[]', 'utf8') fs.writeFileSync('recordArgs.js', RECORD_ARGS_FILE, 'utf8') - await testCommand.handler({ + await run.handler({ bin: 'node_modules/.bin', dir: process.cwd(), extraBinPaths: [], extraEnv: {}, pnpmHomeDir: '', rawConfig: {}, - }, ['arg', '--flag=true', '--help', '-h']) + }, ['test', 'arg', '--flag=true', '--help', '-h']) const { default: args } = await import(path.resolve('args.json')) expect(args).toStrictEqual([['arg', '--flag=true', '--help', '-h']]) diff --git a/exec/plugin-commands-script-runners/test/testRecursive.ts b/exec/plugin-commands-script-runners/test/testRecursive.ts index ea0bae1b927..2a834093422 100644 --- a/exec/plugin-commands-script-runners/test/testRecursive.ts +++ b/exec/plugin-commands-script-runners/test/testRecursive.ts @@ -1,7 +1,7 @@ import path from 'path' import { filterPkgsBySelectorObjects } from '@pnpm/filter-workspace-packages' import { filterPackagesFromDir } from '@pnpm/workspace.filter-packages-from-dir' -import { test as testCommand } from '@pnpm/plugin-commands-script-runners' +import { run } from '@pnpm/plugin-commands-script-runners' import { preparePackages } from '@pnpm/prepare' import { createTestIpcServer } from '@pnpm/test-ipc-server' import execa from 'execa' @@ -62,14 +62,14 @@ test('pnpm recursive test', async () => { '--store-dir', path.resolve(DEFAULT_OPTS.storeDir), ]) - await testCommand.handler({ + await run.handler({ ...DEFAULT_OPTS, allProjects, dir: process.cwd(), recursive: true, selectedProjectsGraph, workspaceDir: process.cwd(), - }) + }, ['test']) expect(server1.getLines()).toStrictEqual(['project-1', 'project-2']) expect(server2.getLines()).toStrictEqual(['project-1', 'project-3']) @@ -116,14 +116,14 @@ test('`pnpm recursive test` does not fail if none of the packages has a test com path.resolve(DEFAULT_OPTS.storeDir), ]) - await testCommand.handler({ + await run.handler({ ...DEFAULT_OPTS, allProjects, dir: process.cwd(), recursive: true, selectedProjectsGraph, workspaceDir: process.cwd(), - }) + }, ['test']) }) test('pnpm recursive test with filtering', async () => { @@ -166,14 +166,14 @@ test('pnpm recursive test with filtering', async () => { '--store-dir', path.resolve(DEFAULT_OPTS.storeDir), ]) - await testCommand.handler({ + await run.handler({ ...DEFAULT_OPTS, allProjects, dir: process.cwd(), recursive: true, selectedProjectsGraph, workspaceDir: process.cwd(), - }) + }, ['test']) expect(server.getLines()).toStrictEqual(['project-1']) }) diff --git a/exec/plugin-commands-script-runners/test/utils/index.ts b/exec/plugin-commands-script-runners/test/utils/index.ts index 2fa17dbbf46..d9795c936ca 100644 --- a/exec/plugin-commands-script-runners/test/utils/index.ts +++ b/exec/plugin-commands-script-runners/test/utils/index.ts @@ -56,7 +56,7 @@ export const DEFAULT_OPTS = { cpu: ['current'], libc: ['current'], }, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, } export const DLX_DEFAULT_OPTS = { @@ -94,5 +94,5 @@ export const DLX_DEFAULT_OPTS = { cpu: ['current'], libc: ['current'], }, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, } diff --git a/exec/plugin-commands-script-runners/tsconfig.json b/exec/plugin-commands-script-runners/tsconfig.json index 586e7f83689..dc54de5e782 100644 --- a/exec/plugin-commands-script-runners/tsconfig.json +++ b/exec/plugin-commands-script-runners/tsconfig.json @@ -27,6 +27,9 @@ { "path": "../../config/config" }, + { + "path": "../../crypto/hash" + }, { "path": "../../env/path" }, @@ -36,9 +39,6 @@ { "path": "../../packages/core-loggers" }, - { - "path": "../../packages/crypto.base32-hash" - }, { "path": "../../packages/error" }, diff --git a/fetching/git-fetcher/src/index.ts b/fetching/git-fetcher/src/index.ts index ff47c214d62..97b20366419 100644 --- a/fetching/git-fetcher/src/index.ts +++ b/fetching/git-fetcher/src/index.ts @@ -55,7 +55,7 @@ export function createGitFetcher (createOpts: CreateGitFetcherOptions): { git: G // Even though we have the index of the package, // the linking of files to the store is in progress. return addFilesFromDir({ - cafsDir: cafs.cafsDir, + storeDir: cafs.storeDir, dir: pkgDir, files, filesIndexFile: opts.filesIndexFile, diff --git a/fetching/git-fetcher/test/index.ts b/fetching/git-fetcher/test/index.ts index a93b506a058..63d43b8ccf2 100644 --- a/fetching/git-fetcher/test/index.ts +++ b/fetching/git-fetcher/test/index.ts @@ -29,10 +29,10 @@ beforeEach(() => { }) test('fetch', async () => { - const cafsDir = tempy.directory() + const storeDir = tempy.directory() const fetch = createGitFetcher({ rawConfig: {} }).git const { filesIndex, manifest } = await fetch( - createCafsStore(cafsDir), + createCafsStore(storeDir), { commit: 'c9b30e71d704cd30fa71f2edd1ecc7dcc4985493', repo: 'https://github.com/kevva/is-positive.git', @@ -40,7 +40,7 @@ test('fetch', async () => { }, { readManifest: true, - filesIndexFile: path.join(cafsDir, 'index.json'), + filesIndexFile: path.join(storeDir, 'index.json'), } ) expect(filesIndex['package.json']).toBeTruthy() @@ -48,10 +48,10 @@ test('fetch', async () => { }) test('fetch a package from Git sub folder', async () => { - const cafsDir = tempy.directory() + const storeDir = tempy.directory() const fetch = createGitFetcher({ rawConfig: {} }).git const { filesIndex } = await fetch( - createCafsStore(cafsDir), + createCafsStore(storeDir), { commit: '2b42a57a945f19f8ffab8ecbd2021fdc2c58ee22', repo: 'https://github.com/RexSkz/test-git-subfolder-fetch.git', @@ -59,20 +59,20 @@ test('fetch a package from Git sub folder', async () => { type: 'git', }, { - filesIndexFile: path.join(cafsDir, 'index.json'), + filesIndexFile: path.join(storeDir, 'index.json'), } ) expect(filesIndex['public/index.html']).toBeTruthy() }) test('prevent directory traversal attack when using Git sub folder', async () => { - const cafsDir = tempy.directory() + const storeDir = tempy.directory() const fetch = createGitFetcher({ rawConfig: {} }).git const repo = 'https://github.com/RexSkz/test-git-subfolder-fetch.git' const pkgDir = '../../etc' await expect( fetch( - createCafsStore(cafsDir), + createCafsStore(storeDir), { commit: '2b42a57a945f19f8ffab8ecbd2021fdc2c58ee22', repo, @@ -80,20 +80,20 @@ test('prevent directory traversal attack when using Git sub folder', async () => type: 'git', }, { - filesIndexFile: path.join(cafsDir, 'index.json'), + filesIndexFile: path.join(storeDir, 'index.json'), } ) ).rejects.toThrow(`Failed to prepare git-hosted package fetched from "${repo}": Path "${pkgDir}" should be a sub directory`) }) test('prevent directory traversal attack when using Git sub folder', async () => { - const cafsDir = tempy.directory() + const storeDir = tempy.directory() const fetch = createGitFetcher({ rawConfig: {} }).git const repo = 'https://github.com/RexSkz/test-git-subfolder-fetch.git' const pkgDir = 'not/exists' await expect( fetch( - createCafsStore(cafsDir), + createCafsStore(storeDir), { commit: '2b42a57a945f19f8ffab8ecbd2021fdc2c58ee22', repo, @@ -101,24 +101,24 @@ test('prevent directory traversal attack when using Git sub folder', async () => type: 'git', }, { - filesIndexFile: path.join(cafsDir, 'index.json'), + filesIndexFile: path.join(storeDir, 'index.json'), } ) ).rejects.toThrow(`Failed to prepare git-hosted package fetched from "${repo}": Path "${pkgDir}" is not a directory`) }) test('fetch a package from Git that has a prepare script', async () => { - const cafsDir = tempy.directory() + const storeDir = tempy.directory() const fetch = createGitFetcher({ rawConfig: {} }).git const { filesIndex } = await fetch( - createCafsStore(cafsDir), + createCafsStore(storeDir), { commit: '8b333f12d5357f4f25a654c305c826294cb073bf', repo: 'https://github.com/pnpm/test-git-fetch.git', type: 'git', }, { - filesIndexFile: path.join(cafsDir, 'index.json'), + filesIndexFile: path.join(storeDir, 'index.json'), } ) expect(filesIndex['dist/index.js']).toBeTruthy() @@ -126,10 +126,10 @@ test('fetch a package from Git that has a prepare script', async () => { // Test case for https://github.com/pnpm/pnpm/issues/1866 test('fetch a package without a package.json', async () => { - const cafsDir = tempy.directory() + const storeDir = tempy.directory() const fetch = createGitFetcher({ rawConfig: {} }).git const { filesIndex } = await fetch( - createCafsStore(cafsDir), + createCafsStore(storeDir), { // a small Deno library with a 'denolib.json' instead of a 'package.json' commit: 'aeb6b15f9c9957c8fa56f9731e914c4d8a6d2f2b', @@ -137,7 +137,7 @@ test('fetch a package without a package.json', async () => { type: 'git', }, { - filesIndexFile: path.join(cafsDir, 'index.json'), + filesIndexFile: path.join(storeDir, 'index.json'), } ) expect(filesIndex['denolib.json']).toBeTruthy() @@ -145,30 +145,30 @@ test('fetch a package without a package.json', async () => { // Covers the regression reported in https://github.com/pnpm/pnpm/issues/4064 test('fetch a big repository', async () => { - const cafsDir = tempy.directory() + const storeDir = tempy.directory() const fetch = createGitFetcher({ rawConfig: {} }).git - const { filesIndex } = await fetch(createCafsStore(cafsDir), + const { filesIndex } = await fetch(createCafsStore(storeDir), { commit: 'a65fbf5a90f53c9d72fed4daaca59da50f074355', repo: 'https://github.com/sveltejs/action-deploy-docs.git', type: 'git', }, { - filesIndexFile: path.join(cafsDir, 'index.json'), + filesIndexFile: path.join(storeDir, 'index.json'), }) expect(filesIndex).toBeTruthy() }) test('still able to shallow fetch for allowed hosts', async () => { - const cafsDir = tempy.directory() + const storeDir = tempy.directory() const fetch = createGitFetcher({ gitShallowHosts: ['github.com'], rawConfig: {} }).git const resolution = { commit: 'c9b30e71d704cd30fa71f2edd1ecc7dcc4985493', repo: 'https://github.com/kevva/is-positive.git', type: 'git' as const, } - const { filesIndex, manifest } = await fetch(createCafsStore(cafsDir), resolution, { + const { filesIndex, manifest } = await fetch(createCafsStore(storeDir), resolution, { readManifest: true, - filesIndexFile: path.join(cafsDir, 'index.json'), + filesIndexFile: path.join(storeDir, 'index.json'), }) const calls = (execa as jest.Mock).mock.calls const expectedCalls = [ @@ -188,30 +188,30 @@ test('still able to shallow fetch for allowed hosts', async () => { }) test('fail when preparing a git-hosted package', async () => { - const cafsDir = tempy.directory() + const storeDir = tempy.directory() const fetch = createGitFetcher({ rawConfig: {} }).git await expect( - fetch(createCafsStore(cafsDir), + fetch(createCafsStore(storeDir), { commit: 'ba58874aae1210a777eb309dd01a9fdacc7e54e7', repo: 'https://github.com/pnpm-e2e/prepare-script-fails.git', type: 'git', }, { - filesIndexFile: path.join(cafsDir, 'index.json'), + filesIndexFile: path.join(storeDir, 'index.json'), }) ).rejects.toThrow('Failed to prepare git-hosted package fetched from "https://github.com/pnpm-e2e/prepare-script-fails.git": @pnpm.e2e/prepare-script-fails@1.0.0 npm-install: `npm install`') }) test('do not build the package when scripts are ignored', async () => { - const cafsDir = tempy.directory() + const storeDir = tempy.directory() const fetch = createGitFetcher({ ignoreScripts: true, rawConfig: {} }).git - const { filesIndex } = await fetch(createCafsStore(cafsDir), + const { filesIndex } = await fetch(createCafsStore(storeDir), { commit: '55416a9c468806a935636c0ad0371a14a64df8c9', repo: 'https://github.com/pnpm-e2e/prepare-script-works.git', type: 'git', }, { - filesIndexFile: path.join(cafsDir, 'index.json'), + filesIndexFile: path.join(storeDir, 'index.json'), }) expect(filesIndex['package.json']).toBeTruthy() expect(filesIndex['prepare.txt']).toBeFalsy() @@ -223,17 +223,17 @@ function prefixGitArgs (): string[] { } test('fetch only the included files', async () => { - const cafsDir = tempy.directory() + const storeDir = tempy.directory() const fetch = createGitFetcher({ rawConfig: {} }).git const { filesIndex } = await fetch( - createCafsStore(cafsDir), + createCafsStore(storeDir), { commit: '958d6d487217512bb154d02836e9b5b922a600d8', repo: 'https://github.com/pnpm-e2e/pkg-with-ignored-files', type: 'git', }, { - filesIndexFile: path.join(cafsDir, 'index.json'), + filesIndexFile: path.join(storeDir, 'index.json'), } ) expect(Object.keys(filesIndex).sort()).toStrictEqual([ diff --git a/fetching/tarball-fetcher/src/gitHostedTarballFetcher.ts b/fetching/tarball-fetcher/src/gitHostedTarballFetcher.ts index 95654299040..9c31c2c1b8c 100644 --- a/fetching/tarball-fetcher/src/gitHostedTarballFetcher.ts +++ b/fetching/tarball-fetcher/src/gitHostedTarballFetcher.ts @@ -103,7 +103,7 @@ async function prepareGitHostedPkg ( // the linking of files to the store is in progress. return { ...await addFilesFromDir({ - cafsDir: cafs.cafsDir, + storeDir: cafs.storeDir, dir: pkgDir, files, filesIndexFile, diff --git a/fetching/tarball-fetcher/src/localTarballFetcher.ts b/fetching/tarball-fetcher/src/localTarballFetcher.ts index b33a0aa2453..5d245c23124 100644 --- a/fetching/tarball-fetcher/src/localTarballFetcher.ts +++ b/fetching/tarball-fetcher/src/localTarballFetcher.ts @@ -17,7 +17,7 @@ export function createLocalTarballFetcher (): FetchFunction { const tarball = resolvePath(opts.lockfileDir, resolution.tarball.slice(5)) const buffer = gfs.readFileSync(tarball) return addFilesFromTarball({ - cafsDir: cafs.cafsDir, + storeDir: cafs.storeDir, buffer, filesIndexFile: opts.filesIndexFile, integrity: resolution.integrity, diff --git a/fetching/tarball-fetcher/src/remoteTarballFetcher.ts b/fetching/tarball-fetcher/src/remoteTarballFetcher.ts index 9125584511b..b5107390327 100644 --- a/fetching/tarball-fetcher/src/remoteTarballFetcher.ts +++ b/fetching/tarball-fetcher/src/remoteTarballFetcher.ts @@ -161,7 +161,7 @@ export function createDownloader ( } return addFilesFromTarball({ buffer: data, - cafsDir: opts.cafs.cafsDir, + storeDir: opts.cafs.storeDir, readManifest: opts.readManifest, integrity: opts.integrity, filesIndexFile: opts.filesIndexFile, diff --git a/fetching/tarball-fetcher/test/fetch.ts b/fetching/tarball-fetcher/test/fetch.ts index 3ac7eaf6b4f..e33c52d0324 100644 --- a/fetching/tarball-fetcher/test/fetch.ts +++ b/fetching/tarball-fetcher/test/fetch.ts @@ -27,9 +27,9 @@ beforeEach(() => { ;(globalWarn as jest.Mock).mockClear() }) -const cafsDir = tempy.directory() -const filesIndexFile = path.join(cafsDir, 'index.json') -const cafs = createCafsStore(cafsDir) +const storeDir = tempy.directory() +const filesIndexFile = path.join(storeDir, 'index.json') +const cafs = createCafsStore(storeDir) const f = fixtures(__dirname) const tarballPath = f.find('babel-helper-hoist-variables-6.24.1.tgz') diff --git a/hooks/pnpmfile/package.json b/hooks/pnpmfile/package.json index 4e4a23f0a2b..e599965c739 100644 --- a/hooks/pnpmfile/package.json +++ b/hooks/pnpmfile/package.json @@ -35,7 +35,7 @@ }, "dependencies": { "@pnpm/core-loggers": "workspace:*", - "@pnpm/crypto.base32-hash": "workspace:*", + "@pnpm/crypto.hash": "workspace:*", "@pnpm/error": "workspace:*", "@pnpm/hooks.types": "workspace:*", "@pnpm/lockfile.types": "workspace:*", diff --git a/hooks/pnpmfile/src/requireHooks.ts b/hooks/pnpmfile/src/requireHooks.ts index fa11d2e3fc7..e7c117109aa 100644 --- a/hooks/pnpmfile/src/requireHooks.ts +++ b/hooks/pnpmfile/src/requireHooks.ts @@ -1,6 +1,6 @@ import type { PreResolutionHookContext, PreResolutionHookLogger } from '@pnpm/hooks.types' import { hookLogger } from '@pnpm/core-loggers' -import { createBase32HashFromFile } from '@pnpm/crypto.base32-hash' +import { createHashFromFile } from '@pnpm/crypto.hash' import pathAbsolute from 'path-absolute' import type { CustomFetchers } from '@pnpm/fetcher-base' import { type ImportIndexedPackageAsync } from '@pnpm/store-controller-types' @@ -40,7 +40,7 @@ export function requireHooks ( let hooks: Hooks | undefined = pnpmFile?.hooks if (!globalHooks && !hooks) return { afterAllResolved: [], filterLog: [], readPackage: [] } - const calculatePnpmfileChecksum = hooks ? () => createBase32HashFromFile(pnpmfilePath) : undefined + const calculatePnpmfileChecksum = hooks ? () => createHashFromFile(pnpmfilePath) : undefined globalHooks = globalHooks ?? {} hooks = hooks ?? {} const cookedHooks: CookedHooks & Required> = { diff --git a/hooks/pnpmfile/tsconfig.json b/hooks/pnpmfile/tsconfig.json index 0e2d9e2a638..abeca860531 100644 --- a/hooks/pnpmfile/tsconfig.json +++ b/hooks/pnpmfile/tsconfig.json @@ -9,6 +9,9 @@ "../../__typings__/**/*.d.ts" ], "references": [ + { + "path": "../../crypto/hash" + }, { "path": "../../fetching/fetcher-base" }, @@ -18,9 +21,6 @@ { "path": "../../packages/core-loggers" }, - { - "path": "../../packages/crypto.base32-hash" - }, { "path": "../../packages/error" }, diff --git a/lockfile/plugin-commands-audit/test/fix.ts b/lockfile/plugin-commands-audit/test/fix.ts index 05ae9f9a802..98d116c1f82 100644 --- a/lockfile/plugin-commands-audit/test/fix.ts +++ b/lockfile/plugin-commands-audit/test/fix.ts @@ -29,7 +29,7 @@ test('overrides are added for vulnerable dependencies', async () => { userConfig: {}, rawConfig, registries, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }) expect(exitCode).toBe(0) @@ -54,7 +54,7 @@ test('no overrides are added if no vulnerabilities are found', async () => { userConfig: {}, rawConfig, registries, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }) expect(exitCode).toBe(0) @@ -90,7 +90,7 @@ test('CVEs found in the allow list are not added as overrides', async () => { userConfig: {}, rawConfig, registries, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }) expect(exitCode).toBe(0) expect(output).toMatch(/Run "pnpm install"/) diff --git a/lockfile/plugin-commands-audit/test/index.ts b/lockfile/plugin-commands-audit/test/index.ts index 6a57d798e14..bacd79c86fe 100644 --- a/lockfile/plugin-commands-audit/test/index.ts +++ b/lockfile/plugin-commands-audit/test/index.ts @@ -60,7 +60,7 @@ export const DEFAULT_OPTS = { useRunningStoreServer: false, useStoreServer: false, workspaceConcurrency: 4, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, peersSuffixMaxLength: 1000, } @@ -82,7 +82,7 @@ describe('plugin-commands-audit', () => { userConfig: {}, rawConfig, registries, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }) expect(exitCode).toBe(1) expect(stripAnsi(output)).toMatchSnapshot() @@ -100,7 +100,7 @@ describe('plugin-commands-audit', () => { userConfig: {}, rawConfig, registries, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }) expect(exitCode).toBe(1) @@ -118,7 +118,7 @@ describe('plugin-commands-audit', () => { userConfig: {}, rawConfig, registries, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }) expect(exitCode).toBe(1) @@ -135,7 +135,7 @@ describe('plugin-commands-audit', () => { userConfig: {}, rawConfig, registries, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }) expect(stripAnsi(output)).toBe('No known vulnerabilities found\n') @@ -153,7 +153,7 @@ describe('plugin-commands-audit', () => { userConfig: {}, rawConfig, registries, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }) const json = JSON.parse(output) @@ -173,7 +173,7 @@ describe('plugin-commands-audit', () => { rawConfig, dev: true, registries, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }) expect(exitCode).toBe(0) @@ -194,7 +194,7 @@ describe('plugin-commands-audit', () => { userConfig: {}, rawConfig, registries, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }) expect(exitCode).toBe(0) @@ -216,7 +216,7 @@ describe('plugin-commands-audit', () => { [`${registries.default.replace(/^https?:/, '')}:_authToken`]: '123', }, registries, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }) expect(stripAnsi(output)).toBe('No known vulnerabilities found\n') @@ -237,7 +237,7 @@ describe('plugin-commands-audit', () => { userConfig: {}, rawConfig, registries, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, })).rejects.toThrow(AuditEndpointNotExistsError) }) @@ -266,7 +266,7 @@ describe('plugin-commands-audit', () => { }, }, }, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }) expect(exitCode).toBe(1) @@ -298,7 +298,7 @@ describe('plugin-commands-audit', () => { }, }, }, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }) expect(exitCode).toBe(1) @@ -331,7 +331,7 @@ describe('plugin-commands-audit', () => { }, }, }, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }) expect(exitCode).toBe(1) diff --git a/lockfile/settings-checker/package.json b/lockfile/settings-checker/package.json index a8418a37143..8832f34d5af 100644 --- a/lockfile/settings-checker/package.json +++ b/lockfile/settings-checker/package.json @@ -30,7 +30,7 @@ }, "homepage": "https://github.com/pnpm/pnpm/blob/main/lockfile/settings-checker#readme", "dependencies": { - "@pnpm/crypto.base32-hash": "workspace:*", + "@pnpm/crypto.hash": "workspace:*", "@pnpm/lockfile.types": "workspace:*", "@pnpm/parse-overrides": "workspace:*", "p-map-values": "catalog:", diff --git a/lockfile/settings-checker/src/calcPatchHashes.ts b/lockfile/settings-checker/src/calcPatchHashes.ts index 07157482938..d93bea57935 100644 --- a/lockfile/settings-checker/src/calcPatchHashes.ts +++ b/lockfile/settings-checker/src/calcPatchHashes.ts @@ -1,12 +1,12 @@ import path from 'path' import pMapValues from 'p-map-values' -import { createBase32HashFromFile } from '@pnpm/crypto.base32-hash' +import { createHexHashFromFile } from '@pnpm/crypto.hash' import { type PatchFile } from '@pnpm/lockfile.types' export async function calcPatchHashes (patches: Record, lockfileDir: string): Promise> { return pMapValues(async (patchFilePath) => { return { - hash: await createBase32HashFromFile(patchFilePath), + hash: await createHexHashFromFile(patchFilePath), path: path.relative(lockfileDir, patchFilePath).replaceAll('\\', '/'), } }, patches) diff --git a/lockfile/settings-checker/tsconfig.json b/lockfile/settings-checker/tsconfig.json index 2c88a5b13bb..4df354e5701 100644 --- a/lockfile/settings-checker/tsconfig.json +++ b/lockfile/settings-checker/tsconfig.json @@ -16,7 +16,7 @@ "path": "../../config/parse-overrides" }, { - "path": "../../packages/crypto.base32-hash" + "path": "../../crypto/hash" }, { "path": "../types" diff --git a/modules-mounter/daemon/package.json b/modules-mounter/daemon/package.json index fcdd530c07a..c9919668862 100644 --- a/modules-mounter/daemon/package.json +++ b/modules-mounter/daemon/package.json @@ -37,6 +37,7 @@ "@pnpm/logger": "^5.1.0" }, "devDependencies": { + "@pnpm/constants": "workspace:*", "@pnpm/logger": "workspace:*", "@pnpm/mount-modules": "workspace:*", "@types/normalize-path": "catalog:", diff --git a/modules-mounter/daemon/src/cli.ts b/modules-mounter/daemon/src/cli.ts index 4874999fd55..7f5034d16de 100644 --- a/modules-mounter/daemon/src/cli.ts +++ b/modules-mounter/daemon/src/cli.ts @@ -12,12 +12,12 @@ import { createFuseHandlers } from './createFuseHandlers' cliOptions: {}, packageManager: { name: '', version: '' }, }) - const cafsDir = path.join(await getStorePath({ + const storeDir = await getStorePath({ pkgRoot: process.cwd(), storePath: config.storeDir, pnpmHomeDir: config.pnpmHomeDir, - }), 'files') - const fuse = new Fuse(mnt, await createFuseHandlers(process.cwd(), cafsDir), { debug: true }) + }) + const fuse = new Fuse(mnt, await createFuseHandlers(process.cwd(), storeDir), { debug: true }) fuse.mount(function (err?: Error) { if (err != null) console.error(err) }) diff --git a/modules-mounter/daemon/src/createFuseHandlers.ts b/modules-mounter/daemon/src/createFuseHandlers.ts index 0f0a07595f1..b0a86cfd2cf 100644 --- a/modules-mounter/daemon/src/createFuseHandlers.ts +++ b/modules-mounter/daemon/src/createFuseHandlers.ts @@ -32,13 +32,13 @@ export interface FuseHandlers { readdir: (p: string, cb: (returnCode: number, files?: string[]) => void) => void } -export async function createFuseHandlers (lockfileDir: string, cafsDir: string): Promise { +export async function createFuseHandlers (lockfileDir: string, storeDir: string): Promise { const lockfile = await readWantedLockfile(lockfileDir, { ignoreIncompatible: true }) if (lockfile == null) throw new Error('Cannot generate a .pnp.cjs without a lockfile') - return createFuseHandlersFromLockfile(lockfile, cafsDir) + return createFuseHandlersFromLockfile(lockfile, storeDir) } -export function createFuseHandlersFromLockfile (lockfile: Lockfile, cafsDir: string): FuseHandlers { +export function createFuseHandlersFromLockfile (lockfile: Lockfile, storeDir: string): FuseHandlers { const pkgSnapshotCache = new Map() const virtualNodeModules = makeVirtualNodeModules(lockfile) return { @@ -53,7 +53,7 @@ export function createFuseHandlersFromLockfile (lockfile: Lockfile, cafsDir: str cb(-1) return } - const filePathInStore = getFilePathByModeInCafs(cafsDir, fileInfo.integrity, fileInfo.mode) + const filePathInStore = getFilePathByModeInCafs(storeDir, fileInfo.integrity, fileInfo.mode) fs.open(filePathInStore, flags, (err, fd) => { if (err != null) { cb(-1) @@ -164,7 +164,7 @@ export function createFuseHandlersFromLockfile (lockfile: Lockfile, cafsDir: str currentDirEntry = currentDirEntry.entries[parts.shift()!] } if (currentDirEntry?.entryType === 'index') { - const pkg = getPkgInfo(currentDirEntry.depPath, cafsDir) + const pkg = getPkgInfo(currentDirEntry.depPath, storeDir) if (pkg == null) { return null } @@ -176,13 +176,14 @@ export function createFuseHandlersFromLockfile (lockfile: Lockfile, cafsDir: str } return currentDirEntry } - function getPkgInfo (depPath: string, cafsDir: string) { + function getPkgInfo (depPath: string, storeDir: string) { if (!pkgSnapshotCache.has(depPath)) { const pkgSnapshot = lockfile.packages?.[depPath as DepPath] if (pkgSnapshot == null) return undefined - const indexPath = getIndexFilePathInCafs(cafsDir, (pkgSnapshot.resolution as TarballResolution).integrity!) + const nameVer = nameVerFromPkgSnapshot(depPath, pkgSnapshot) + const indexPath = getIndexFilePathInCafs(storeDir, (pkgSnapshot.resolution as TarballResolution).integrity!, `${nameVer.name}@${nameVer.version}`) pkgSnapshotCache.set(depPath, { - ...nameVerFromPkgSnapshot(depPath, pkgSnapshot), + ...nameVer, pkgSnapshot, index: loadJsonFile.sync(indexPath), // TODO: maybe make it async? }) diff --git a/modules-mounter/daemon/test/createFuseHandlers.test.ts b/modules-mounter/daemon/test/createFuseHandlers.test.ts index 680f2a865b8..9860af343f4 100644 --- a/modules-mounter/daemon/test/createFuseHandlers.test.ts +++ b/modules-mounter/daemon/test/createFuseHandlers.test.ts @@ -1,3 +1,4 @@ +import { STORE_VERSION } from '@pnpm/constants' import path from 'path' jest.mock('fuse-native', () => ({ ENOENT: -2 })) @@ -11,7 +12,7 @@ describe('FUSE handlers', () => { let handlers: FuseHandlers beforeAll(async () => { const fixture = path.join(__dirname, '__fixtures__/simple') - handlers = await createFuseHandlers(fixture, path.join(fixture, 'store/v3/files')) + handlers = await createFuseHandlers(fixture, path.join(fixture, 'store', STORE_VERSION)) }) it('readdir', () => { diff --git a/modules-mounter/daemon/tsconfig.json b/modules-mounter/daemon/tsconfig.json index 87c07e6b2a5..9585337e8f5 100644 --- a/modules-mounter/daemon/tsconfig.json +++ b/modules-mounter/daemon/tsconfig.json @@ -18,6 +18,9 @@ { "path": "../../lockfile/utils" }, + { + "path": "../../packages/constants" + }, { "path": "../../packages/dependency-path" }, diff --git a/package.json b/package.json index df2233e2072..2172404f065 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "lint:meta": "pnpm run meta-updater --test", "copy-artifacts": "ts-node __utils__/scripts/src/copy-artifacts.ts", "make-release-description": "pnpm --filter=@pnpm/get-release-text run write-release-text", - "release": "pnpm --filter=@pnpm/exe publish --tag=next-9 --access=public && pnpm publish --filter=!pnpm --filter=!@pnpm/exe --access=public && pnpm publish --filter=pnpm --tag=next-9 --access=public", + "release": "pnpm --filter=@pnpm/exe publish --tag=next-10 --access=public && pnpm publish --filter=!pnpm --filter=!@pnpm/exe --access=public && pnpm publish --filter=pnpm --tag=next-10 --access=public", "dev-setup": "pnpm -C=./pnpm/dev link -g" }, "devDependencies": { diff --git a/packages/calc-dep-state/src/index.ts b/packages/calc-dep-state/src/index.ts index 485357b5f31..169cd6e8a0c 100644 --- a/packages/calc-dep-state/src/index.ts +++ b/packages/calc-dep-state/src/index.ts @@ -32,10 +32,10 @@ export function calcDepState ( let result = ENGINE_NAME if (opts.isBuilt) { const depStateObj = calcDepStateObj(depPath, depsGraph, cache, new Set()) - result += `-${hashObjectWithoutSorting(depStateObj)}` + result += `;deps=${hashObjectWithoutSorting(depStateObj)}` } if (opts.patchFileHash) { - result += `-${opts.patchFileHash}` + result += `;patch=${opts.patchFileHash}` } return result } diff --git a/packages/calc-dep-state/test/index.ts b/packages/calc-dep-state/test/index.ts index 74c09e131a2..a78135d5e61 100644 --- a/packages/calc-dep-state/test/index.ts +++ b/packages/calc-dep-state/test/index.ts @@ -21,7 +21,7 @@ const depsGraph = { test('calcDepState()', () => { expect(calcDepState(depsGraph, {}, 'registry/foo@1.0.0', { isBuilt: true, - })).toBe(`${ENGINE_NAME}-${hashObject({ + })).toBe(`${ENGINE_NAME};deps=${hashObject({ 'bar@1.0.0': { 'foo@1.0.0': {} }, })}`) }) diff --git a/packages/constants/src/index.ts b/packages/constants/src/index.ts index a3a37ef3275..b3d139b9de5 100644 --- a/packages/constants/src/index.ts +++ b/packages/constants/src/index.ts @@ -3,8 +3,9 @@ export const LOCKFILE_MAJOR_VERSION = '9' export const LOCKFILE_VERSION = `${LOCKFILE_MAJOR_VERSION}.0` export const LOCKFILE_VERSION_V6 = '6.0' -export const ENGINE_NAME = `${process.platform}-${process.arch}-node-${process.version.split('.')[0]}` +export const ENGINE_NAME = `${process.platform};${process.arch};node${process.version.split('.')[0].substring(1)}` export const LAYOUT_VERSION = 5 +export const STORE_VERSION = 'v10' export const WORKSPACE_MANIFEST_FILENAME = 'pnpm-workspace.yaml' diff --git a/packages/crypto.base32-hash/CHANGELOG.md b/packages/crypto.base32-hash/CHANGELOG.md deleted file mode 100644 index 7c01c98da37..00000000000 --- a/packages/crypto.base32-hash/CHANGELOG.md +++ /dev/null @@ -1,33 +0,0 @@ -# @pnpm/crypto.base32-hash - -## 3.0.1 - -### Patch Changes - -- Updated dependencies [222d10a] -- Updated dependencies [222d10a] - - @pnpm/crypto.polyfill@1.0.0 - -## 3.0.0 - -### Major Changes - -- 43cdd87: Node.js v16 support dropped. Use at least Node.js v18.12. - -## 2.0.0 - -### Major Changes - -- eceaa8b8b: Node.js 14 support dropped. - -## 1.0.1 - -### Patch Changes - -- 2bca856e0: The hash of the patch file should be the same on both Windows and POSIX [#4961](https://github.com/pnpm/pnpm/issues/4961). - -## 1.0.0 - -### Major Changes - -- 725636a90: Initial release. diff --git a/packages/crypto.base32-hash/README.md b/packages/crypto.base32-hash/README.md deleted file mode 100644 index 94edddce48b..00000000000 --- a/packages/crypto.base32-hash/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# @pnpm/crypto.base32-hash - -> Create a base32 hash - -## Installation - -```sh -pnpm add @pnpm/crypto.base32-hash -``` - -## License - -MIT diff --git a/packages/crypto.base32-hash/src/index.ts b/packages/crypto.base32-hash/src/index.ts deleted file mode 100644 index 9329d4b90ce..00000000000 --- a/packages/crypto.base32-hash/src/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -import * as crypto from '@pnpm/crypto.polyfill' -import fs from 'fs' -import { base32 } from 'rfc4648' - -export function createBase32Hash (str: string): string { - return base32.stringify(crypto.hash('md5', str, 'buffer')).replace(/(=+)$/, '').toLowerCase() -} - -export async function createBase32HashFromFile (file: string): Promise { - const content = await fs.promises.readFile(file, 'utf8') - return createBase32Hash(content.split('\r\n').join('\n')) -} diff --git a/packages/crypto.base32-hash/test/index.ts b/packages/crypto.base32-hash/test/index.ts deleted file mode 100644 index 0dfd3c8cf6d..00000000000 --- a/packages/crypto.base32-hash/test/index.ts +++ /dev/null @@ -1,15 +0,0 @@ -/// -import fs from 'fs' -import { createBase32Hash, createBase32HashFromFile } from '@pnpm/crypto.base32-hash' -import { tempDir } from '@pnpm/prepare' - -test('createBase32Hash()', () => { - expect(createBase32Hash('AAA')).toEqual('4h5p7m7gcttmf65hikljmi4gw4') -}) - -test('createBase32HashFromFile normalizes line endings before calculating the hash', async () => { - tempDir() - fs.writeFileSync('win-eol.txt', 'a\r\nb\r\nc') - fs.writeFileSync('posix-eol.txt', 'a\nb\r\nc') - expect(await createBase32HashFromFile('win-eol.txt')).toEqual(await createBase32HashFromFile('posix-eol.txt')) -}) diff --git a/packages/dependency-path/package.json b/packages/dependency-path/package.json index 20be190e669..86395cfca85 100644 --- a/packages/dependency-path/package.json +++ b/packages/dependency-path/package.json @@ -31,7 +31,7 @@ }, "homepage": "https://github.com/pnpm/pnpm/blob/main/packages/dependency-path#readme", "dependencies": { - "@pnpm/crypto.base32-hash": "workspace:*", + "@pnpm/crypto.hash": "workspace:*", "@pnpm/types": "workspace:*", "semver": "catalog:" }, diff --git a/packages/dependency-path/src/index.ts b/packages/dependency-path/src/index.ts index d3b93e5bfe3..42800d91075 100644 --- a/packages/dependency-path/src/index.ts +++ b/packages/dependency-path/src/index.ts @@ -1,4 +1,4 @@ -import { createBase32Hash } from '@pnpm/crypto.base32-hash' +import { createShortHash } from '@pnpm/crypto.hash' import { type DepPath, type PkgResolutionId, type Registries, type PkgId, type PkgIdWithPatchHash } from '@pnpm/types' import semver from 'semver' @@ -165,14 +165,14 @@ export function parse (dependencyPath: string): DependencyPath { } export function depPathToFilename (depPath: string, maxLengthWithoutHash: number): string { - let filename = depPathToFilenameUnescaped(depPath).replace(/[\\/:*?"<>|]/g, '+') + let filename = depPathToFilenameUnescaped(depPath).replace(/[\\/:*?"<>|#]/g, '+') if (filename.includes('(')) { filename = filename .replace(/\)$/, '') .replace(/(\)\()|\(|\)/g, '_') } if (filename.length > maxLengthWithoutHash || filename !== filename.toLowerCase() && !filename.startsWith('file+')) { - return `${filename.substring(0, maxLengthWithoutHash - 27)}_${createBase32Hash(filename)}` + return `${filename.substring(0, maxLengthWithoutHash - 33)}_${createShortHash(filename)}` } return filename } @@ -204,7 +204,7 @@ export function createPeersDirSuffix (peerIds: PeerId[], maxLength: number = 100 } ).sort().join(')(') if (dirName.length > maxLength) { - dirName = createBase32Hash(dirName) + dirName = createShortHash(dirName) } return `(${dirName})` } diff --git a/packages/dependency-path/test/index.ts b/packages/dependency-path/test/index.ts index 991bfd1a9dd..efade1888f5 100644 --- a/packages/dependency-path/test/index.ts +++ b/packages/dependency-path/test/index.ts @@ -96,8 +96,11 @@ test('depPathToFilename()', () => { expect(filename).toBe('file+test+foo-1.0.0.tgz_foo@2.0.0') expect(filename).not.toContain(':') - expect(depPathToFilename('abcd/'.repeat(200), 120)).toBe('abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+abc_jvx2blbax4cyhfgrgozfgpdv24') // cspell:disable-line - expect(depPathToFilename('/JSONSteam@1.0.0', 120)).toBe('JSONSteam@1.0.0_jmswpk4sf667aelr6wp2xd3p54') // cspell:disable-line + expect(depPathToFilename('abcd/'.repeat(200), 120)).toBe('abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+abcd+ab_e7c10c3598ebbc0ca640b6524c68e602') // cspell:disable-line + expect(depPathToFilename('/JSONSteam@1.0.0', 120)).toBe('JSONSteam@1.0.0_533d3b11e9111b7a24f914844c021ddf') // cspell:disable-line + + expect(depPathToFilename('foo@git+https://github.com/something/foo#1234', 120)).toBe('foo@git+https+++github.com+something+foo+1234') + expect(depPathToFilename('foo@https://codeload.github.com/something/foo/tar.gz/1234#path:packages/foo', 120)).toBe('foo@https+++codeload.github.com+something+foo+tar.gz+1234+path+packages+foo') }) test('tryGetPackageId', () => { diff --git a/packages/dependency-path/tsconfig.json b/packages/dependency-path/tsconfig.json index 6e13e37d33c..91751ad9c72 100644 --- a/packages/dependency-path/tsconfig.json +++ b/packages/dependency-path/tsconfig.json @@ -10,7 +10,7 @@ ], "references": [ { - "path": "../crypto.base32-hash" + "path": "../../crypto/hash" }, { "path": "../types" diff --git a/patching/plugin-commands-patching/test/patch.test.ts b/patching/plugin-commands-patching/test/patch.test.ts index 8d62da8da35..54259ef8612 100644 --- a/patching/plugin-commands-patching/test/patch.test.ts +++ b/patching/plugin-commands-patching/test/patch.test.ts @@ -27,7 +27,7 @@ const basePatchOption = { registries: { default: `http://localhost:${REGISTRY_MOCK_PORT}/` }, userConfig: {}, virtualStoreDir: 'node_modules/.pnpm', - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, } describe('patch and commit', () => { diff --git a/patching/plugin-commands-patching/test/utils/index.ts b/patching/plugin-commands-patching/test/utils/index.ts index dba77c22130..a445990d768 100644 --- a/patching/plugin-commands-patching/test/utils/index.ts +++ b/patching/plugin-commands-patching/test/utils/index.ts @@ -53,6 +53,6 @@ export const DEFAULT_OPTS = { cpu: ['current'], libc: ['current'], }, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, peersSuffixMaxLength: 1000, } diff --git a/pkg-manager/core/package.json b/pkg-manager/core/package.json index 16e16268f15..fe421ff7426 100644 --- a/pkg-manager/core/package.json +++ b/pkg-manager/core/package.json @@ -23,8 +23,8 @@ "@pnpm/catalogs.types": "workspace:*", "@pnpm/constants": "workspace:*", "@pnpm/core-loggers": "workspace:*", - "@pnpm/crypto.base32-hash": "workspace:*", - "@pnpm/crypto.polyfill": "workspace:*", + "@pnpm/crypto.hash": "workspace:*", + "@pnpm/crypto.object-hasher": "workspace:*", "@pnpm/dependency-path": "workspace:*", "@pnpm/deps.graph-sequencer": "workspace:*", "@pnpm/error": "workspace:*", @@ -64,6 +64,8 @@ "@pnpm/types": "workspace:*", "@pnpm/which-version-is-pinned": "workspace:*", "@zkochan/rimraf": "catalog:", + "ci-info": "catalog:", + "enquirer": "catalog:", "is-inner-link": "catalog:", "is-subdir": "catalog:", "load-json-file": "catalog:", @@ -73,15 +75,13 @@ "path-exists": "catalog:", "ramda": "catalog:", "run-groups": "catalog:", - "semver": "catalog:", - "sort-keys": "catalog:" + "semver": "catalog:" }, "devDependencies": { "@pnpm/assert-project": "workspace:*", "@pnpm/assert-store": "workspace:*", "@pnpm/client": "workspace:*", "@pnpm/core": "workspace:*", - "@pnpm/crypto.object-hasher": "workspace:*", "@pnpm/git-utils": "workspace:*", "@pnpm/lockfile.types": "workspace:*", "@pnpm/logger": "workspace:*", diff --git a/pkg-manager/core/src/api.ts b/pkg-manager/core/src/api.ts index b5818e15640..361db9736bd 100644 --- a/pkg-manager/core/src/api.ts +++ b/pkg-manager/core/src/api.ts @@ -1,4 +1,3 @@ export * from './install' export { PeerDependencyIssuesError } from './install/reportPeerDependencyIssues' -export * from './link' export * from './getPeerDependencyIssues' diff --git a/pkg-manager/core/src/index.ts b/pkg-manager/core/src/index.ts index 1dd92c3b902..9ced5912876 100644 --- a/pkg-manager/core/src/index.ts +++ b/pkg-manager/core/src/index.ts @@ -9,7 +9,9 @@ export type { export type { HoistingLimits } from '@pnpm/headless' export * from './api' -export { type ProjectOptions, UnexpectedStoreError, UnexpectedVirtualStoreDirError } from '@pnpm/get-context' +export { type ProjectOptions } from '@pnpm/get-context' +export { UnexpectedStoreError } from './install/checkCompatibility/UnexpectedStoreError' +export { UnexpectedVirtualStoreDirError } from './install/checkCompatibility/UnexpectedVirtualStoreDirError' export type { InstallOptions } from './install/extendInstallOptions' export type { WorkspacePackages } from '@pnpm/resolver-base' diff --git a/pkg-manager/get-context/src/checkCompatibility/BreakingChangeError.ts b/pkg-manager/core/src/install/checkCompatibility/BreakingChangeError.ts similarity index 100% rename from pkg-manager/get-context/src/checkCompatibility/BreakingChangeError.ts rename to pkg-manager/core/src/install/checkCompatibility/BreakingChangeError.ts diff --git a/pkg-manager/get-context/src/checkCompatibility/ErrorRelatedSources.ts b/pkg-manager/core/src/install/checkCompatibility/ErrorRelatedSources.ts similarity index 100% rename from pkg-manager/get-context/src/checkCompatibility/ErrorRelatedSources.ts rename to pkg-manager/core/src/install/checkCompatibility/ErrorRelatedSources.ts diff --git a/pkg-manager/get-context/src/checkCompatibility/ModulesBreakingChangeError.ts b/pkg-manager/core/src/install/checkCompatibility/ModulesBreakingChangeError.ts similarity index 100% rename from pkg-manager/get-context/src/checkCompatibility/ModulesBreakingChangeError.ts rename to pkg-manager/core/src/install/checkCompatibility/ModulesBreakingChangeError.ts diff --git a/pkg-manager/get-context/src/checkCompatibility/UnexpectedStoreError.ts b/pkg-manager/core/src/install/checkCompatibility/UnexpectedStoreError.ts similarity index 100% rename from pkg-manager/get-context/src/checkCompatibility/UnexpectedStoreError.ts rename to pkg-manager/core/src/install/checkCompatibility/UnexpectedStoreError.ts diff --git a/pkg-manager/get-context/src/checkCompatibility/UnexpectedVirtualStoreDirError.ts b/pkg-manager/core/src/install/checkCompatibility/UnexpectedVirtualStoreDirError.ts similarity index 100% rename from pkg-manager/get-context/src/checkCompatibility/UnexpectedVirtualStoreDirError.ts rename to pkg-manager/core/src/install/checkCompatibility/UnexpectedVirtualStoreDirError.ts diff --git a/pkg-manager/get-context/src/checkCompatibility/index.ts b/pkg-manager/core/src/install/checkCompatibility/index.ts similarity index 100% rename from pkg-manager/get-context/src/checkCompatibility/index.ts rename to pkg-manager/core/src/install/checkCompatibility/index.ts diff --git a/pkg-manager/core/src/install/index.test.ts b/pkg-manager/core/src/install/index.test.ts deleted file mode 100644 index 16b7f7afd59..00000000000 --- a/pkg-manager/core/src/install/index.test.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { createObjectChecksum } from './index' - -function assets () { - const sorted = { - abc: { - a: 0, - b: [0, 1, 2], - c: null, - }, - def: { - foo: 'bar', - hello: 'world', - }, - } as const - - const unsorted1 = { - abc: { - b: [0, 1, 2], - a: 0, - c: null, - }, - def: { - hello: 'world', - foo: 'bar', - }, - } as const - - const unsorted2 = { - def: { - foo: 'bar', - hello: 'world', - }, - abc: { - a: 0, - b: [0, 1, 2], - c: null, - }, - } as const - - const unsorted3 = { - def: { - hello: 'world', - foo: 'bar', - }, - abc: { - b: [0, 1, 2], - a: 0, - c: null, - }, - } as const - - return { sorted, unsorted1, unsorted2, unsorted3 } as const -} - -test('createObjectChecksum', () => { - const { sorted, unsorted1, unsorted2, unsorted3 } = assets() - expect(createObjectChecksum(unsorted1)).toBe(createObjectChecksum(sorted)) - expect(createObjectChecksum(unsorted2)).toBe(createObjectChecksum(sorted)) - expect(createObjectChecksum(unsorted3)).toBe(createObjectChecksum(sorted)) -}) diff --git a/pkg-manager/core/src/install/index.ts b/pkg-manager/core/src/install/index.ts index 435c1ae5a95..eaf39944b4d 100644 --- a/pkg-manager/core/src/install/index.ts +++ b/pkg-manager/core/src/install/index.ts @@ -13,7 +13,7 @@ import { stageLogger, summaryLogger, } from '@pnpm/core-loggers' -import * as crypto from '@pnpm/crypto.polyfill' +import { hashObjectNullableWithPrefix } from '@pnpm/crypto.object-hasher' import { calcPatchHashes, createOverridesMapFromParsed, @@ -45,9 +45,7 @@ import { getPreferredVersionsFromLockfileAndManifests } from '@pnpm/lockfile.pre import { logger, globalInfo, streamParser } from '@pnpm/logger' import { getAllDependenciesFromManifest, getAllUniqueSpecs } from '@pnpm/manifest-utils' import { writeModulesManifest } from '@pnpm/modules-yaml' -import { readModulesDir } from '@pnpm/read-modules-dir' import { safeReadProjectManifestOnly } from '@pnpm/read-project-manifest' -import { removeBin } from '@pnpm/remove-bins' import { getWantedDependencies, type DependenciesGraph, @@ -70,17 +68,13 @@ import { type ReadPackageHook, type ProjectRootDir, } from '@pnpm/types' -import rimraf from '@zkochan/rimraf' -import isInnerLink from 'is-inner-link' import isSubdir from 'is-subdir' -import pFilter from 'p-filter' import pLimit from 'p-limit' import mapValues from 'ramda/src/map' import clone from 'ramda/src/clone' import isEmpty from 'ramda/src/isEmpty' import pipeWith from 'ramda/src/pipeWith' import props from 'ramda/src/props' -import sortKeys from 'sort-keys' import { parseWantedDependencies } from '../parseWantedDependencies' import { removeDeps } from '../uninstall/removeDeps' import { @@ -90,6 +84,8 @@ import { } from './extendInstallOptions' import { linkPackages } from './link' import { reportPeerDependencyIssues } from './reportPeerDependencyIssues' +import { validateModules } from './validateModules' +import { isCI } from 'ci-info' class LockfileConfigMismatchError extends PnpmError { constructor (outdatedLockfileSettingName: string) { @@ -134,20 +130,12 @@ export interface UninstallSomeDepsMutation { targetDependenciesField?: DependenciesField } -export interface UnlinkDepsMutation { - mutation: 'unlink' -} - -export interface UnlinkSomeDepsMutation { - mutation: 'unlinkSome' - dependencyNames: string[] -} - -export type DependenciesMutation = InstallDepsMutation | InstallSomeDepsMutation | UninstallSomeDepsMutation | UnlinkDepsMutation | UnlinkSomeDepsMutation +export type DependenciesMutation = InstallDepsMutation | InstallSomeDepsMutation | UninstallSomeDepsMutation type Opts = Omit & { preferredVersions?: PreferredVersions pruneDirectDependencies?: boolean + binsDir?: string } & InstallMutationOptions export async function install ( @@ -172,6 +160,7 @@ export async function install ( buildIndex: 0, manifest, rootDir, + binsDir: opts.binsDir, }], } ) @@ -247,14 +236,38 @@ export async function mutateModules ( const installsOnly = allMutationsAreInstalls(projects) if (!installsOnly) opts.strictPeerDependencies = false - // @ts-expect-error - opts['forceNewModules'] = installsOnly const rootProjectManifest = opts.allProjects.find(({ rootDir }) => rootDir === opts.lockfileDir)?.manifest ?? // When running install/update on a subset of projects, the root project might not be included, // so reading its manifest explicitly here. await safeReadProjectManifestOnly(opts.lockfileDir) - const ctx = await getContext(opts) + let ctx = await getContext(opts) + + if (!opts.lockfileOnly && ctx.modulesFile != null) { + const { purged } = await validateModules(ctx.modulesFile, Object.values(ctx.projects), { + forceNewModules: installsOnly, + include: opts.include, + lockfileDir: opts.lockfileDir, + modulesDir: opts.modulesDir ?? 'node_modules', + registries: opts.registries, + storeDir: opts.storeDir, + virtualStoreDir: ctx.virtualStoreDir, + virtualStoreDirMaxLength: opts.virtualStoreDirMaxLength, + confirmModulesPurge: opts.confirmModulesPurge && !isCI, + + forceHoistPattern: opts.forceHoistPattern, + hoistPattern: opts.hoistPattern, + currentHoistPattern: ctx.currentHoistPattern, + + forcePublicHoistPattern: opts.forcePublicHoistPattern, + publicHoistPattern: opts.publicHoistPattern, + currentPublicHoistPattern: ctx.currentPublicHoistPattern, + global: opts.global, + }) + if (purged) { + ctx = await getContext(opts) + } + } if (opts.hooks.preResolution) { await opts.hooks.preResolution({ @@ -330,7 +343,7 @@ export async function mutateModules ( } ) } - const packageExtensionsChecksum = isEmpty(opts.packageExtensions ?? {}) ? undefined : createObjectChecksum(opts.packageExtensions!) + const packageExtensionsChecksum = hashObjectNullableWithPrefix(opts.packageExtensions) const pnpmfileChecksum = await opts.hooks.calculatePnpmfileChecksum?.() const patchedDependencies = opts.ignorePackageManifest ? ctx.wantedLockfile.patchedDependencies @@ -569,70 +582,6 @@ Note that in CI environments, this setting is enabled by default.`, }) break } - case 'unlink': { - const packageDirs = await readModulesDir(projectOpts.modulesDir) - const externalPackages = await pFilter( - packageDirs!, - async (packageDir: string) => isExternalLink(ctx.storeDir, projectOpts.modulesDir, packageDir) - ) - const allDeps = getAllDependenciesFromManifest(projectOpts.manifest) - const packagesToInstall: string[] = [] - for (const pkgName of externalPackages) { - await rimraf(path.join(projectOpts.modulesDir, pkgName)) - if (allDeps[pkgName]) { - packagesToInstall.push(pkgName) - } - } - if (packagesToInstall.length === 0) { - return { - updatedProjects: projects.map((mutatedProject) => ctx.projects[mutatedProject.rootDir]), - } - } - - // TODO: install only those that were unlinked - // but don't update their version specs in package.json - await installCase({ ...projectOpts, mutation: 'install' }) - break - } - case 'unlinkSome': { - if (projectOpts.manifest?.name && opts.globalBin) { - await removeBin(path.join(opts.globalBin, projectOpts.manifest?.name)) - } - const packagesToInstall: string[] = [] - const allDeps = getAllDependenciesFromManifest(projectOpts.manifest) - for (const depName of project.dependencyNames) { - try { - if (!await isExternalLink(ctx.storeDir, projectOpts.modulesDir, depName)) { - logger.warn({ - message: `${depName} is not an external link`, - prefix: project.rootDir, - }) - continue - } - } catch (err: any) { // eslint-disable-line - if (err['code'] !== 'ENOENT') throw err - } - await rimraf(path.join(projectOpts.modulesDir, depName)) - if (allDeps[depName]) { - packagesToInstall.push(depName) - } - } - if (packagesToInstall.length === 0) { - return { - updatedProjects: projects.map((mutatedProject) => ctx.projects[mutatedProject.rootDir]), - } - } - - // TODO: install only those that were unlinked - // but don't update their version specs in package.json - await installSome({ - ...projectOpts, - dependencySelectors: packagesToInstall, - mutation: 'installSome', - updatePackageManifest: false, - }) - break - } } } /* eslint-enable no-await-in-loop */ @@ -744,21 +693,10 @@ Note that in CI environments, this setting is enabled by default.`, } } -export function createObjectChecksum (obj: Record): string { - const s = JSON.stringify(sortKeys(obj, { deep: true })) - return crypto.hash('md5', s, 'hex') -} - function cacheExpired (prunedAt: string, maxAgeInMinutes: number): boolean { return ((Date.now() - new Date(prunedAt).valueOf()) / (1000 * 60)) > maxAgeInMinutes } -async function isExternalLink (storeDir: string, modules: string, pkgName: string): Promise { - const link = await isInnerLink(modules, pkgName) - - return !link.isInner -} - function pkgHasDependencies (manifest: ProjectManifest): boolean { return Boolean( (Object.keys(manifest.dependencies ?? {}).length > 0) || diff --git a/pkg-manager/core/src/install/validateModules.ts b/pkg-manager/core/src/install/validateModules.ts new file mode 100644 index 00000000000..8fd641918d2 --- /dev/null +++ b/pkg-manager/core/src/install/validateModules.ts @@ -0,0 +1,202 @@ +import { promises as fs } from 'fs' +import path from 'path' +import { PnpmError } from '@pnpm/error' +import { logger } from '@pnpm/logger' +import { + type IncludedDependencies, + type Modules, +} from '@pnpm/modules-yaml' +import { + DEPENDENCIES_FIELDS, + type Registries, + type ProjectRootDir, +} from '@pnpm/types' +import rimraf from '@zkochan/rimraf' +import enquirer from 'enquirer' +import equals from 'ramda/src/equals' +import { checkCompatibility } from './checkCompatibility' + +interface ImporterToPurge { + modulesDir: string + rootDir: ProjectRootDir +} + +export async function validateModules ( + modules: Modules, + projects: Array<{ + modulesDir: string + id: string + rootDir: ProjectRootDir + }>, + opts: { + currentHoistPattern?: string[] + currentPublicHoistPattern?: string[] + forceNewModules: boolean + include?: IncludedDependencies + lockfileDir: string + modulesDir: string + registries: Registries + storeDir: string + virtualStoreDir: string + virtualStoreDirMaxLength: number + confirmModulesPurge?: boolean + + hoistPattern?: string[] | undefined + forceHoistPattern?: boolean + + publicHoistPattern?: string[] | undefined + forcePublicHoistPattern?: boolean + global?: boolean + } +): Promise<{ purged: boolean }> { + const rootProject = projects.find(({ id }) => id === '.') + if (opts.virtualStoreDirMaxLength !== modules.virtualStoreDirMaxLength) { + if (opts.forceNewModules && (rootProject != null)) { + await purgeModulesDirsOfImporter(opts, rootProject) + return { purged: true } + } + throw new PnpmError( + 'VIRTUAL_STORE_DIR_MAX_LENGTH_DIFF', + 'This modules directory was created using a different virtual-store-dir-max-length value.' + + ' Run "pnpm install" to recreate the modules directory.' + ) + } + if ( + opts.forcePublicHoistPattern && + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + !equals(modules.publicHoistPattern, opts.publicHoistPattern || undefined) + ) { + if (opts.forceNewModules && (rootProject != null)) { + await purgeModulesDirsOfImporter(opts, rootProject) + return { purged: true } + } + throw new PnpmError( + 'PUBLIC_HOIST_PATTERN_DIFF', + 'This modules directory was created using a different public-hoist-pattern value.' + + ' Run "pnpm install" to recreate the modules directory.' + ) + } + + const importersToPurge: ImporterToPurge[] = [] + + if (opts.forceHoistPattern && (rootProject != null)) { + try { + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + if (!equals(opts.currentHoistPattern, opts.hoistPattern || undefined)) { + throw new PnpmError( + 'HOIST_PATTERN_DIFF', + 'This modules directory was created using a different hoist-pattern value.' + + ' Run "pnpm install" to recreate the modules directory.' + ) + } + } catch (err: any) { // eslint-disable-line + if (!opts.forceNewModules) throw err + importersToPurge.push(rootProject) + } + } + for (const project of projects) { + try { + checkCompatibility(modules, { + modulesDir: project.modulesDir, + storeDir: opts.storeDir, + virtualStoreDir: opts.virtualStoreDir, + }) + if (opts.lockfileDir !== project.rootDir && (opts.include != null) && modules.included) { + for (const depsField of DEPENDENCIES_FIELDS) { + if (opts.include[depsField] !== modules.included[depsField]) { + throw new PnpmError('INCLUDED_DEPS_CONFLICT', + `modules directory (at "${opts.lockfileDir}") was installed with ${stringifyIncludedDeps(modules.included)}. ` + + `Current install wants ${stringifyIncludedDeps(opts.include)}.` + ) + } + } + } + } catch (err: any) { // eslint-disable-line + if (!opts.forceNewModules) throw err + importersToPurge.push(project) + } + } + if (importersToPurge.length > 0 && (rootProject == null)) { + importersToPurge.push({ + modulesDir: path.join(opts.lockfileDir, opts.modulesDir), + rootDir: opts.lockfileDir as ProjectRootDir, + }) + } + + const purged = importersToPurge.length > 0 + if (purged) { + await purgeModulesDirsOfImporters(opts, importersToPurge) + } + + return { purged } +} + +async function purgeModulesDirsOfImporter ( + opts: { + confirmModulesPurge?: boolean + virtualStoreDir: string + }, + importer: ImporterToPurge +): Promise { + return purgeModulesDirsOfImporters(opts, [importer]) +} + +async function purgeModulesDirsOfImporters ( + opts: { + confirmModulesPurge?: boolean + virtualStoreDir: string + }, + importers: ImporterToPurge[] +): Promise { + if (opts.confirmModulesPurge ?? true) { + const confirmed = await enquirer.prompt<{ question: boolean }>({ + type: 'confirm', + name: 'question', + message: importers.length === 1 + ? `The modules directory at "${importers[0].modulesDir}" will be removed and reinstalled from scratch. Proceed?` + : 'The modules directories will be removed and reinstalled from scratch. Proceed?', + initial: true, + }) + if (!confirmed.question) { + throw new PnpmError('ABORTED_REMOVE_MODULES_DIR', 'Aborted removal of modules directory') + } + } + await Promise.all(importers.map(async (importer) => { + logger.info({ + message: `Recreating ${importer.modulesDir}`, + prefix: importer.rootDir, + }) + try { + // We don't remove the actual modules directory, just the contents of it. + // 1. we will need the directory anyway. + // 2. in some setups, pnpm won't even have permission to remove the modules directory. + await removeContentsOfDir(importer.modulesDir, opts.virtualStoreDir) + } catch (err: any) { // eslint-disable-line + if (err.code !== 'ENOENT') throw err + } + })) +} + +async function removeContentsOfDir (dir: string, virtualStoreDir: string): Promise { + const items = await fs.readdir(dir) + await Promise.all(items.map(async (item) => { + // The non-pnpm related hidden files are kept + if ( + item.startsWith('.') && + item !== '.bin' && + item !== '.modules.yaml' && + !dirsAreEqual(path.join(dir, item), virtualStoreDir) + ) { + return + } + await rimraf(path.join(dir, item)) + })) +} + +function dirsAreEqual (dir1: string, dir2: string): boolean { + return path.relative(dir1, dir2) === '' +} + +function stringifyIncludedDeps (included: IncludedDependencies): string { + return DEPENDENCIES_FIELDS.filter((depsField) => included[depsField]).join(', ') +} diff --git a/pkg-manager/core/src/link/index.ts b/pkg-manager/core/src/link/index.ts deleted file mode 100644 index d96508dee3c..00000000000 --- a/pkg-manager/core/src/link/index.ts +++ /dev/null @@ -1,191 +0,0 @@ -import path from 'path' -import { - summaryLogger, -} from '@pnpm/core-loggers' -import { PnpmError } from '@pnpm/error' -import { getContextForSingleImporter } from '@pnpm/get-context' -import { linkBinsOfPackages } from '@pnpm/link-bins' -import { - getLockfileImporterId, - type ProjectSnapshot, - writeCurrentLockfile, - writeLockfiles, -} from '@pnpm/lockfile.fs' -import { logger, streamParser } from '@pnpm/logger' -import { - getPref, - getSpecFromPackageManifest, - getDependencyTypeFromManifest, - guessDependencyType, - type PackageSpecObject, - updateProjectManifestObject, -} from '@pnpm/manifest-utils' -import { pruneSharedLockfile } from '@pnpm/lockfile.pruner' -import { readProjectManifest } from '@pnpm/read-project-manifest' -import { symlinkDirectRootDependency } from '@pnpm/symlink-dependency' -import { - type DependenciesField, - DEPENDENCIES_FIELDS, - type DependencyManifest, - type ProjectManifest, -} from '@pnpm/types' -import normalize from 'normalize-path' -import { - extendOptions, - type LinkOptions, -} from './options' - -type LinkFunctionOptions = LinkOptions & { - linkToBin?: string - dir: string -} - -export type { LinkFunctionOptions } - -export async function link ( - linkFromPkgs: Array<{ alias: string, path: string } | string>, - destModules: string, - maybeOpts: LinkFunctionOptions -): Promise { - const reporter = maybeOpts?.reporter - if ((reporter != null) && typeof reporter === 'function') { - streamParser.on('data', reporter) - } - const opts = await extendOptions(maybeOpts) - const ctx = await getContextForSingleImporter(opts.manifest, { - ...opts, - extraBinPaths: [], // ctx.extraBinPaths is not needed, so this is fine - }, true) - - const importerId = getLockfileImporterId(ctx.lockfileDir, opts.dir) - const specsToUpsert = [] as PackageSpecObject[] - - const linkedPkgs = await Promise.all( - linkFromPkgs.map(async (linkFrom) => { - let linkFromPath: string - let linkFromAlias: string | undefined - if (typeof linkFrom === 'string') { - linkFromPath = linkFrom - } else { - linkFromPath = linkFrom.path - linkFromAlias = linkFrom.alias - } - const { manifest } = await readProjectManifest(linkFromPath) as { manifest: DependencyManifest } - if (typeof linkFrom === 'string' && manifest.name === undefined) { - throw new PnpmError('INVALID_PACKAGE_NAME', `Package in ${linkFromPath} must have a name field to be linked`) - } - - const targetDependencyType = getDependencyTypeFromManifest(opts.manifest, manifest.name) ?? opts.targetDependenciesField - - specsToUpsert.push({ - alias: manifest.name, - pref: getPref(manifest.name, manifest.name, manifest.version, { - pinnedVersion: opts.pinnedVersion, - }), - saveType: (targetDependencyType ?? (ctx.manifest && guessDependencyType(manifest.name, ctx.manifest))) as DependenciesField, - }) - - const packagePath = normalize(path.relative(opts.dir, linkFromPath)) - const addLinkOpts = { - linkedPkgName: linkFromAlias ?? manifest.name, - manifest: ctx.manifest, - packagePath, - } - addLinkToLockfile(ctx.currentLockfile.importers[importerId], addLinkOpts) - addLinkToLockfile(ctx.wantedLockfile.importers[importerId], addLinkOpts) - - return { - alias: linkFromAlias ?? manifest.name, - manifest, - path: linkFromPath, - } - }) - ) - - const updatedCurrentLockfile = pruneSharedLockfile(ctx.currentLockfile) - - const warn = (message: string) => { - logger.warn({ message, prefix: opts.dir }) - } - const updatedWantedLockfile = pruneSharedLockfile(ctx.wantedLockfile, { warn }) - - // Linking should happen after removing orphans - // Otherwise would've been removed - await Promise.all(linkedPkgs.map(async ({ alias, manifest, path }) => { - // TODO: cover with test that linking reports with correct dependency types - const stu = specsToUpsert.find((s) => s.alias === manifest.name) - const targetDependencyType = getDependencyTypeFromManifest(opts.manifest, manifest.name) ?? opts.targetDependenciesField - await symlinkDirectRootDependency(path, destModules, alias, { - fromDependenciesField: stu?.saveType ?? (targetDependencyType as DependenciesField), - linkedPackage: manifest, - prefix: opts.dir, - }) - })) - - const linkToBin = maybeOpts?.linkToBin ?? path.join(destModules, '.bin') - await linkBinsOfPackages(linkedPkgs.map((p) => ({ manifest: p.manifest, location: p.path })), linkToBin, { - extraNodePaths: ctx.extraNodePaths, - preferSymlinkedExecutables: opts.preferSymlinkedExecutables, - }) - - let newPkg!: ProjectManifest - if (opts.targetDependenciesField) { - newPkg = await updateProjectManifestObject(opts.dir, opts.manifest, specsToUpsert) - for (const { alias } of specsToUpsert) { - updatedWantedLockfile.importers[importerId].specifiers[alias] = getSpecFromPackageManifest(newPkg, alias) - } - } else { - newPkg = opts.manifest - } - const lockfileOpts = { useGitBranchLockfile: opts.useGitBranchLockfile, mergeGitBranchLockfiles: opts.mergeGitBranchLockfiles } - if (opts.useLockfile) { - await writeLockfiles({ - currentLockfile: updatedCurrentLockfile, - currentLockfileDir: ctx.virtualStoreDir, - wantedLockfile: updatedWantedLockfile, - wantedLockfileDir: ctx.lockfileDir, - ...lockfileOpts, - }) - } else { - await writeCurrentLockfile(ctx.virtualStoreDir, updatedCurrentLockfile) - } - - summaryLogger.debug({ prefix: opts.dir }) - - if ((reporter != null) && typeof reporter === 'function') { - streamParser.removeListener('data', reporter) - } - - return newPkg -} - -function addLinkToLockfile ( - projectSnapshot: ProjectSnapshot, - opts: { - linkedPkgName: string - packagePath: string - manifest?: ProjectManifest - } -) { - const id = `link:${opts.packagePath}` - let addedTo: DependenciesField | undefined - for (const depType of DEPENDENCIES_FIELDS) { - if (!addedTo && opts.manifest?.[depType]?.[opts.linkedPkgName]) { - addedTo = depType - projectSnapshot[depType] = projectSnapshot[depType] ?? {} - projectSnapshot[depType]![opts.linkedPkgName] = id - } else if (projectSnapshot[depType] != null) { - delete projectSnapshot[depType]![opts.linkedPkgName] - } - } - - // package.json might not be available when linking to global - if (opts.manifest == null) return - - const availableSpec = getSpecFromPackageManifest(opts.manifest, opts.linkedPkgName) - if (availableSpec) { - projectSnapshot.specifiers[opts.linkedPkgName] = availableSpec - } else { - delete projectSnapshot.specifiers[opts.linkedPkgName] - } -} diff --git a/pkg-manager/core/src/link/options.ts b/pkg-manager/core/src/link/options.ts deleted file mode 100644 index c6e81fbf340..00000000000 --- a/pkg-manager/core/src/link/options.ts +++ /dev/null @@ -1,74 +0,0 @@ -import path from 'path' -import { normalizeRegistries, DEFAULT_REGISTRIES } from '@pnpm/normalize-registries' -import { type StoreController } from '@pnpm/store-controller-types' -import { - type DependenciesField, - type ProjectManifest, - type Registries, -} from '@pnpm/types' -import { type ReporterFunction } from '../types' - -interface StrictLinkOptions { - autoInstallPeers: boolean - binsDir: string - excludeLinksFromLockfile: boolean - force: boolean - useLockfile: boolean - lockfileDir: string - nodeLinker: 'isolated' | 'hoisted' | 'pnp' - pinnedVersion: 'major' | 'minor' | 'patch' - storeController: StoreController - manifest: ProjectManifest - registries: Registries - storeDir: string - reporter: ReporterFunction - targetDependenciesField?: DependenciesField - dir: string - preferSymlinkedExecutables: boolean - - hoistPattern: string[] | undefined - forceHoistPattern: boolean - - publicHoistPattern: string[] | undefined - forcePublicHoistPattern: boolean - - useGitBranchLockfile: boolean - mergeGitBranchLockfiles: boolean - virtualStoreDirMaxLength: number - peersSuffixMaxLength: number -} - -export type LinkOptions = - & Partial - & Pick - -export async function extendOptions (opts: LinkOptions): Promise { - if (opts) { - for (const key in opts) { - if (opts[key as keyof LinkOptions] === undefined) { - delete opts[key as keyof LinkOptions] - } - } - } - const defaultOpts = await defaults(opts) - const extendedOpts = { ...defaultOpts, ...opts, storeDir: defaultOpts.storeDir } - extendedOpts.registries = normalizeRegistries(extendedOpts.registries) - return extendedOpts -} - -async function defaults (opts: LinkOptions): Promise { - const dir = opts.dir ?? process.cwd() - return { - binsDir: path.join(dir, 'node_modules', '.bin'), - dir, - force: false, - hoistPattern: undefined, - lockfileDir: opts.lockfileDir ?? dir, - nodeLinker: 'isolated', - registries: DEFAULT_REGISTRIES, - storeController: opts.storeController, - storeDir: opts.storeDir, - useLockfile: true, - virtualStoreDirMaxLength: 120, - } as StrictLinkOptions -} diff --git a/pkg-manager/core/test/api.ts b/pkg-manager/core/test/api.ts index 416351b38e0..81f9c25ec42 100644 --- a/pkg-manager/core/test/api.ts +++ b/pkg-manager/core/test/api.ts @@ -3,7 +3,6 @@ import { testDefaults } from './utils' test('API', () => { expect(typeof pnpm.install).toBe('function') - expect(typeof pnpm.link).toBe('function') }) // TODO: some sort of this validation might need to exist diff --git a/pkg-manager/core/test/brokenLockfileIntegrity.ts b/pkg-manager/core/test/brokenLockfileIntegrity.ts index c92616d5cfd..662e2c93c47 100644 --- a/pkg-manager/core/test/brokenLockfileIntegrity.ts +++ b/pkg-manager/core/test/brokenLockfileIntegrity.ts @@ -34,13 +34,13 @@ test('installation breaks if the lockfile contains the wrong checksum', async () manifest, mutation: 'install', rootDir: process.cwd() as ProjectRootDir, - }, testDefaults({ frozenLockfile: true }))).rejects.toThrowError(/Package name mismatch found while reading/) + }, testDefaults({ frozenLockfile: true }, { retry: { retries: 0 } }))).rejects.toThrowError(/Got unexpected checksum for/) await mutateModulesInSingleProject({ manifest, mutation: 'install', rootDir: process.cwd() as ProjectRootDir, - }, testDefaults()) + }, testDefaults({}, { retry: { retries: 0 } })) expect(project.readLockfile()).toStrictEqual(correctLockfile) @@ -53,7 +53,7 @@ test('installation breaks if the lockfile contains the wrong checksum', async () manifest, mutation: 'install', rootDir: process.cwd() as ProjectRootDir, - }, testDefaults({ preferFrozenLockfile: false })) + }, testDefaults({ preferFrozenLockfile: false }, { retry: { retries: 0 } })) expect(project.readLockfile()).toStrictEqual(correctLockfile) }) diff --git a/pkg-manager/core/test/install/fromRepo.ts b/pkg-manager/core/test/install/fromRepo.ts index 393890184f5..5ae32375118 100644 --- a/pkg-manager/core/test/install/fromRepo.ts +++ b/pkg-manager/core/test/install/fromRepo.ts @@ -339,3 +339,14 @@ test('from subdirectories of a git repo', async () => { '@my-namespace/simple-react-app': 'github:RexSkz/test-git-subfolder-fetch#path:/packages/simple-react-app', }) }) + +test('no hash character for github subdirectory install', async () => { + prepareEmpty() + + await addDependenciesToPackage({}, [ + 'github:pnpm/only-allow#path:/&v1.2.1', + ], testDefaults()) + + expect(fs.readdirSync('./node_modules/.pnpm')) + .toContain('only-allow@https+++codeload.github.com+pnpm+only-allow+tar.gz+4d577a5a5862a43e752df37a1e8a0c71c3a0084a+path++') +}) diff --git a/pkg-manager/core/test/install/packageExtensions.ts b/pkg-manager/core/test/install/packageExtensions.ts index a21b61a215b..df635cda87d 100644 --- a/pkg-manager/core/test/install/packageExtensions.ts +++ b/pkg-manager/core/test/install/packageExtensions.ts @@ -1,12 +1,16 @@ import { PnpmError } from '@pnpm/error' import { prepareEmpty } from '@pnpm/prepare' import { addDependenciesToPackage, mutateModulesInSingleProject, install } from '@pnpm/core' +import { hashObject as _hashObject } from '@pnpm/crypto.object-hasher' import { type ProjectRootDir, type PackageExtension, type ProjectManifest } from '@pnpm/types' -import { createObjectChecksum } from '../../lib/install/index' import { testDefaults, } from '../utils' +function hashObject (obj: Record): string { + return `sha256-${_hashObject(obj)}` +} + test('manifests are extended with fields specified by packageExtensions', async () => { const project = prepareEmpty() @@ -26,7 +30,7 @@ test('manifests are extended with fields specified by packageExtensions', async { const lockfile = project.readLockfile() expect(lockfile.snapshots['is-positive@1.0.0'].dependencies?.['@pnpm.e2e/bar']).toBe('100.1.0') - expect(lockfile.packageExtensionsChecksum).toStrictEqual(createObjectChecksum({ + expect(lockfile.packageExtensionsChecksum).toStrictEqual(hashObject({ 'is-positive': { dependencies: { '@pnpm.e2e/bar': '100.1.0', @@ -48,7 +52,7 @@ test('manifests are extended with fields specified by packageExtensions', async { const lockfile = project.readLockfile() expect(lockfile.snapshots['is-positive@1.0.0'].dependencies?.['@pnpm.e2e/foobar']).toBe('100.0.0') - expect(lockfile.packageExtensionsChecksum).toStrictEqual(createObjectChecksum({ + expect(lockfile.packageExtensionsChecksum).toStrictEqual(hashObject({ 'is-positive': { dependencies: { '@pnpm.e2e/bar': '100.1.0', @@ -68,7 +72,7 @@ test('manifests are extended with fields specified by packageExtensions', async { const lockfile = project.readLockfile() - expect(lockfile.packageExtensionsChecksum).toStrictEqual(createObjectChecksum({ + expect(lockfile.packageExtensionsChecksum).toStrictEqual(hashObject({ 'is-positive': { dependencies: { '@pnpm.e2e/bar': '100.1.0', diff --git a/pkg-manager/core/test/install/patch.ts b/pkg-manager/core/test/install/patch.ts index 62c4dabbf7c..1e6691fc8d1 100644 --- a/pkg-manager/core/test/install/patch.ts +++ b/pkg-manager/core/test/install/patch.ts @@ -3,6 +3,7 @@ import path from 'path' import { type PackageFilesIndex } from '@pnpm/store.cafs' import { ENGINE_NAME } from '@pnpm/constants' import { install } from '@pnpm/core' +import { createHexHashFromFile } from '@pnpm/crypto.hash' import { prepareEmpty } from '@pnpm/prepare' import { fixtures } from '@pnpm/test-fixtures' import { sync as rimraf } from '@zkochan/rimraf' @@ -32,7 +33,7 @@ test('patch package', async () => { expect(fs.readFileSync('node_modules/is-positive/index.js', 'utf8')).toContain('// patched') - const patchFileHash = 'jnbpamcxayl5i4ehrkoext3any' + const patchFileHash = await createHexHashFromFile(patchPath) const lockfile = project.readLockfile() expect(lockfile.patchedDependencies).toStrictEqual({ 'is-positive@1.0.0': { @@ -42,10 +43,10 @@ test('patch package', async () => { }) expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy() - const filesIndexFile = path.join(opts.storeDir, 'files/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812c5d8da8a735e94c2a1ccb77b4583808ee8405313951e7146ac83ede3671dc292-index.json') + const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.json') const filesIndex = loadJsonFile.sync(filesIndexFile) - const sideEffectsKey = `${ENGINE_NAME}-${patchFileHash}` - const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey]['index.js']?.integrity + const sideEffectsKey = `${ENGINE_NAME};patch=${patchFileHash}` + const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey].added?.['index.js']?.integrity expect(patchedFileIntegrity).toBeTruthy() const originalFileIntegrity = filesIndex.files['index.js'].integrity expect(originalFileIntegrity).toBeTruthy() @@ -199,7 +200,7 @@ test('patch package when scripts are ignored', async () => { expect(fs.readFileSync('node_modules/is-positive/index.js', 'utf8')).toContain('// patched') - const patchFileHash = 'jnbpamcxayl5i4ehrkoext3any' + const patchFileHash = await createHexHashFromFile(patchPath) const lockfile = project.readLockfile() expect(lockfile.patchedDependencies).toStrictEqual({ 'is-positive@1.0.0': { @@ -209,10 +210,10 @@ test('patch package when scripts are ignored', async () => { }) expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy() - const filesIndexFile = path.join(opts.storeDir, 'files/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812c5d8da8a735e94c2a1ccb77b4583808ee8405313951e7146ac83ede3671dc292-index.json') + const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.json') const filesIndex = loadJsonFile.sync(filesIndexFile) - const sideEffectsKey = `${ENGINE_NAME}-${patchFileHash}` - const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey]['index.js']?.integrity + const sideEffectsKey = `${ENGINE_NAME};patch=${patchFileHash}` + const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey].added?.['index.js']?.integrity expect(patchedFileIntegrity).toBeTruthy() const originalFileIntegrity = filesIndex.files['index.js'].integrity expect(originalFileIntegrity).toBeTruthy() @@ -286,7 +287,7 @@ test('patch package when the package is not in onlyBuiltDependencies list', asyn expect(fs.readFileSync('node_modules/is-positive/index.js', 'utf8')).toContain('// patched') - const patchFileHash = 'jnbpamcxayl5i4ehrkoext3any' + const patchFileHash = await createHexHashFromFile(patchPath) const lockfile = project.readLockfile() expect(lockfile.patchedDependencies).toStrictEqual({ 'is-positive@1.0.0': { @@ -296,10 +297,10 @@ test('patch package when the package is not in onlyBuiltDependencies list', asyn }) expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy() - const filesIndexFile = path.join(opts.storeDir, 'files/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812c5d8da8a735e94c2a1ccb77b4583808ee8405313951e7146ac83ede3671dc292-index.json') + const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.json') const filesIndex = loadJsonFile.sync(filesIndexFile) - const sideEffectsKey = `${ENGINE_NAME}-${patchFileHash}` - const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey]['index.js']?.integrity + const sideEffectsKey = `${ENGINE_NAME};patch=${patchFileHash}` + const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey].added?.['index.js']?.integrity expect(patchedFileIntegrity).toBeTruthy() const originalFileIntegrity = filesIndex.files['index.js'].integrity expect(originalFileIntegrity).toBeTruthy() @@ -376,10 +377,11 @@ test('patch package when the patched package has no dependencies and appears mul expect(fs.readFileSync('node_modules/is-positive/index.js', 'utf8')).toContain('// patched') + const patchFileHash = await createHexHashFromFile(patchPath) const lockfile = project.readLockfile() expect(Object.keys(lockfile.snapshots).sort()).toStrictEqual([ 'is-not-positive@1.0.0', - 'is-positive@1.0.0(patch_hash=jnbpamcxayl5i4ehrkoext3any)', + `is-positive@1.0.0(patch_hash=${patchFileHash})`, ].sort()) }) diff --git a/pkg-manager/core/test/install/sideEffects.ts b/pkg-manager/core/test/install/sideEffects.ts index 23504b5a511..92a589a2df5 100644 --- a/pkg-manager/core/test/install/sideEffects.ts +++ b/pkg-manager/core/test/install/sideEffects.ts @@ -82,14 +82,13 @@ test('using side effects cache', async () => { }, {}, {}, { packageImportMethod: 'copy' }) const manifest = await addDependenciesToPackage({}, ['@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0'], opts) - const cafsDir = path.join(opts.storeDir, 'files') - const filesIndexFile = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')) + const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') const filesIndex = loadJsonFile.sync(filesIndexFile) expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects - const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}` - expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'generated-by-preinstall.js']) - expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'generated-by-postinstall.js']) - delete filesIndex.sideEffects![sideEffectsKey]['generated-by-postinstall.js'] + const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}` + expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'added', 'generated-by-preinstall.js']) + expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'added', 'generated-by-postinstall.js']) + delete filesIndex.sideEffects![sideEffectsKey].added?.['generated-by-postinstall.js'] writeJsonFile.sync(filesIndexFile, filesIndex) rimraf('node_modules') @@ -156,8 +155,7 @@ test('uploading errors do not interrupt installation', async () => { expect(fs.existsSync('node_modules/@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-postinstall.js')).toBeTruthy() - const cafsDir = path.join(opts.storeDir, 'files') - const filesIndexFile = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')) + const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') const filesIndex = loadJsonFile.sync(filesIndexFile) expect(filesIndex.sideEffects).toBeFalsy() }) @@ -174,17 +172,16 @@ test('a postinstall script does not modify the original sources added to the sto expect(fs.readFileSync('node_modules/@pnpm/postinstall-modifies-source/empty-file.txt', 'utf8')).toContain('hello') - const cafsDir = path.join(opts.storeDir, 'files') - const filesIndexFile = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm/postinstall-modifies-source', '1.0.0')) + const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm/postinstall-modifies-source', '1.0.0'), '@pnpm/postinstall-modifies-source@1.0.0') const filesIndex = loadJsonFile.sync(filesIndexFile) - const patchedFileIntegrity = filesIndex.sideEffects?.[`${ENGINE_NAME}-${hashObject({})}`]['empty-file.txt']?.integrity + const patchedFileIntegrity = filesIndex.sideEffects?.[`${ENGINE_NAME};deps=${hashObject({})}`].added?.['empty-file.txt']?.integrity expect(patchedFileIntegrity).toBeTruthy() const originalFileIntegrity = filesIndex.files['empty-file.txt'].integrity expect(originalFileIntegrity).toBeTruthy() // The integrity of the original file differs from the integrity of the patched file expect(originalFileIntegrity).not.toEqual(patchedFileIntegrity) - expect(fs.readFileSync(getFilePathByModeInCafs(cafsDir, originalFileIntegrity, 420), 'utf8')).toEqual('') + expect(fs.readFileSync(getFilePathByModeInCafs(opts.storeDir, originalFileIntegrity, 420), 'utf8')).toEqual('') }) test('a corrupted side-effects cache is ignored', async () => { @@ -197,14 +194,13 @@ test('a corrupted side-effects cache is ignored', async () => { }) const manifest = await addDependenciesToPackage({}, ['@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0'], opts) - const cafsDir = path.join(opts.storeDir, 'files') - const filesIndexFile = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')) + const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') const filesIndex = loadJsonFile.sync(filesIndexFile) expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects - const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}` - expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'generated-by-preinstall.js']) - const sideEffectFileStat = filesIndex.sideEffects![sideEffectsKey]['generated-by-preinstall.js'] - const sideEffectFile = getFilePathByModeInCafs(cafsDir, sideEffectFileStat.integrity, sideEffectFileStat.mode) + const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}` + expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'added', 'generated-by-preinstall.js']) + const sideEffectFileStat = filesIndex.sideEffects![sideEffectsKey].added!['generated-by-preinstall.js'] + const sideEffectFile = getFilePathByModeInCafs(opts.storeDir, sideEffectFileStat.integrity, sideEffectFileStat.mode) expect(fs.existsSync(sideEffectFile)).toBeTruthy() rimraf(sideEffectFile) // we remove the side effect file to break the store diff --git a/pkg-manager/core/test/install/store.ts b/pkg-manager/core/test/install/store.ts deleted file mode 100644 index c5eeaadd939..00000000000 --- a/pkg-manager/core/test/install/store.ts +++ /dev/null @@ -1,25 +0,0 @@ -import path from 'path' -import { prepareEmpty } from '@pnpm/prepare' -import { addDependenciesToPackage, install } from '@pnpm/core' -import { sync as rimraf } from '@zkochan/rimraf' -import writeJsonFile from 'write-json-file' -import { testDefaults } from '../utils' - -test('repeat install with corrupted `store.json` should work', async () => { - const project = prepareEmpty() - - const opts = testDefaults() - const manifest = await addDependenciesToPackage({}, ['is-negative@1.0.0'], opts) - - rimraf('node_modules') - - // When a package reference is missing from `store.json` - // we assume that it is not in the store. - // The package is downloaded and in case there is a folder - // in the store, it is overwritten. - writeJsonFile.sync(path.join(opts.storeDir, 'v3/store.json'), {}) - - await install(manifest, opts) - - project.has('is-negative') -}) diff --git a/pkg-manager/core/test/link.ts b/pkg-manager/core/test/link.ts index 4023022de4a..f82fb47f347 100644 --- a/pkg-manager/core/test/link.ts +++ b/pkg-manager/core/test/link.ts @@ -1,52 +1,13 @@ -import fs from 'fs' import path from 'path' -import { - addDependenciesToPackage, - install, - link, -} from '@pnpm/core' +import { addDependenciesToPackage, install } from '@pnpm/core' import { fixtures } from '@pnpm/test-fixtures' import { prepareEmpty } from '@pnpm/prepare' import { addDistTag } from '@pnpm/registry-mock' -import { type RootLog } from '@pnpm/core-loggers' -import sinon from 'sinon' -import writeJsonFile from 'write-json-file' -import symlink from 'symlink-dir' +import symlinkDir from 'symlink-dir' import { testDefaults } from './utils' const f = fixtures(__dirname) -test('relative link', async () => { - const project = prepareEmpty() - - const linkedPkgName = 'hello-world-js-bin' - const linkedPkgPath = path.resolve('..', linkedPkgName) - - f.copy(linkedPkgName, linkedPkgPath) - await link([`../${linkedPkgName}`], path.join(process.cwd(), 'node_modules'), testDefaults({ - dir: process.cwd(), - manifest: { - dependencies: { - '@pnpm.e2e/hello-world-js-bin': '*', - }, - }, - })) - - project.isExecutable('.bin/hello-world-js-bin') - - const wantedLockfile = project.readLockfile() - expect(wantedLockfile.importers['.'].dependencies?.['@pnpm.e2e/hello-world-js-bin']).toStrictEqual({ - version: 'link:../hello-world-js-bin', - specifier: '*', - }) - - const currentLockfile = project.readCurrentLockfile() - expect(currentLockfile.importers['.'].dependencies?.['@pnpm.e2e/hello-world-js-bin']).toStrictEqual({ - version: 'link:../hello-world-js-bin', - specifier: '*', - }) -}) - test('relative link is linked by the name of the alias', async () => { const linkedPkgName = 'hello-world-js-bin' @@ -80,34 +41,10 @@ test('relative link is not rewritten by argumentless install', async () => { const linkedPkgName = 'hello-world-js-bin' const linkedPkgPath = path.resolve('..', linkedPkgName) - const reporter = sinon.spy() - const opts = testDefaults() - f.copy(linkedPkgName, linkedPkgPath) - const manifest = await link( - [linkedPkgPath], - path.join(process.cwd(), 'node_modules'), - { - ...opts, - dir: process.cwd(), - manifest: {}, - reporter, - }) - - expect(reporter.calledWithMatch({ - added: { - dependencyType: undefined, - linkedFrom: linkedPkgPath, - name: '@pnpm.e2e/hello-world-js-bin', - realName: '@pnpm.e2e/hello-world-js-bin', - version: '1.0.0', - }, - level: 'debug', - name: 'pnpm:root', - prefix: process.cwd(), - } as RootLog)).toBeTruthy() + symlinkDir.sync(linkedPkgPath, path.resolve('node_modules/@pnpm.e2e/hello-world-js-bin')) - await install(manifest, opts) + await install({}, testDefaults()) expect(project.requireModule('@pnpm.e2e/hello-world-js-bin/package.json').isLocal).toBeTruthy() }) @@ -119,35 +56,12 @@ test('relative link is rewritten by named installation to regular dependency', a const linkedPkgName = 'hello-world-js-bin' const linkedPkgPath = path.resolve('..', linkedPkgName) - const reporter = sinon.spy() const opts = testDefaults({ fastUnpack: false }) f.copy(linkedPkgName, linkedPkgPath) - let manifest = await link( - [linkedPkgPath], - path.join(process.cwd(), 'node_modules'), - { - ...opts, - dir: process.cwd(), - manifest: {}, - reporter, - } - ) - - expect(reporter.calledWithMatch({ - added: { - dependencyType: undefined, - linkedFrom: linkedPkgPath, - name: '@pnpm.e2e/hello-world-js-bin', - realName: '@pnpm.e2e/hello-world-js-bin', - version: '1.0.0', - }, - level: 'debug', - name: 'pnpm:root', - prefix: process.cwd(), - } as RootLog)).toBeTruthy() + symlinkDir.sync(linkedPkgPath, path.resolve('node_modules/@pnpm.e2e/hello-world-js-bin')) - manifest = await addDependenciesToPackage(manifest, ['@pnpm.e2e/hello-world-js-bin'], opts) + const manifest = await addDependenciesToPackage({}, ['@pnpm.e2e/hello-world-js-bin'], opts) expect(manifest.dependencies).toStrictEqual({ '@pnpm.e2e/hello-world-js-bin': '^1.0.0' }) @@ -160,91 +74,6 @@ test('relative link is rewritten by named installation to regular dependency', a expect(currentLockfile.importers['.'].dependencies?.['@pnpm.e2e/hello-world-js-bin'].version).toBe('1.0.0') }) -test('relative link uses realpath when contained in a symlinked dir', async () => { - prepareEmpty() - - // `process.cwd()` is now `.tmp/X/project`. - - f.copy('symlink-workspace', path.resolve('../symlink-workspace')) - - const app1RelPath = '../symlink-workspace/app1' - const app2RelPath = '../symlink-workspace/app2' - - const app1 = path.resolve(app1RelPath) - const app2 = path.resolve(app2RelPath) - - const dest = path.join(app2, 'packages/public') - const src = path.resolve(app1, 'packages/public') - - console.log(`${dest}->${src}`) - - // We must manually create the symlink so it works in Windows too. - await symlink(src, dest) - - process.chdir(path.join(app2, '/packages/public/foo')) - - // `process.cwd()` is now `.tmp/X/symlink-workspace/app2/packages/public/foo`. - - const linkFrom = path.join(app1, '/packages/public/bar') - const linkTo = path.join(app2, '/packages/public/foo', 'node_modules') - - await link([linkFrom], linkTo, testDefaults({ manifest: {}, dir: process.cwd() })) - - const linkToRelLink = fs.readlinkSync(path.join(linkTo, 'bar')) - - if (process.platform === 'win32') { - expect(path.relative(linkToRelLink, path.join(src, 'bar'))).toBe('') // link points to real location - } else { - expect(linkToRelLink).toBe('../../bar') - - // If we don't use real paths we get a link like this. - expect(linkToRelLink).not.toBe('../../../../../app1/packages/public/bar') - } -}) - -test('throws error is package name is not defined', async () => { - prepareEmpty() - - writeJsonFile.sync('../is-positive/package.json', { version: '1.0.0' }) - - const manifest = await addDependenciesToPackage({}, ['is-positive@1.0.0'], testDefaults()) - - try { - await link(['../is-positive'], path.resolve('node_modules'), testDefaults({ manifest, dir: process.cwd() })) - throw new Error('link package should fail') - } catch (err: any) { // eslint-disable-line - expect(err.message).toBe('Package in ../is-positive must have a name field to be linked') - expect(err.code).toBe('ERR_PNPM_INVALID_PACKAGE_NAME') - } -}) - -test('link should not change the type of the dependency', async () => { - const project = prepareEmpty() - - const linkedPkgName = 'hello-world-js-bin' - const linkedPkgPath = path.resolve('..', linkedPkgName) - - f.copy(linkedPkgName, linkedPkgPath) - await link([`../${linkedPkgName}`], path.join(process.cwd(), 'node_modules'), testDefaults({ - dir: process.cwd(), - manifest: { - devDependencies: { - '@pnpm.e2e/hello-world-js-bin': '*', - }, - }, - })) - - project.isExecutable('.bin/hello-world-js-bin') - - const wantedLockfile = project.readLockfile() - expect(wantedLockfile.importers['.'].devDependencies).toStrictEqual({ - '@pnpm.e2e/hello-world-js-bin': { - version: 'link:../hello-world-js-bin', - specifier: '*', - }, - }) -}) - // test.skip('relative link when an external lockfile is used', async () => { // const projects = prepare(t, [ // { diff --git a/pkg-manager/core/test/lockfile.ts b/pkg-manager/core/test/lockfile.ts index d7595d3faed..b072f7d07ee 100644 --- a/pkg-manager/core/test/lockfile.ts +++ b/pkg-manager/core/test/lockfile.ts @@ -1574,5 +1574,5 @@ test('setting a custom peersSuffixMaxLength', async () => { const lockfile = project.readLockfile() expect(lockfile.settings.peersSuffixMaxLength).toBe(10) - expect(lockfile.importers['.']?.dependencies?.['@pnpm.e2e/abc']?.version?.length).toBe(33) + expect(lockfile.importers['.']?.dependencies?.['@pnpm.e2e/abc']?.version?.length).toBe(39) }) diff --git a/pkg-manager/core/test/prune.ts b/pkg-manager/core/test/prune.ts index 98ac17de9d2..b5d6799953f 100644 --- a/pkg-manager/core/test/prune.ts +++ b/pkg-manager/core/test/prune.ts @@ -5,11 +5,11 @@ import { fixtures } from '@pnpm/test-fixtures' import { addDependenciesToPackage, install, - link, mutateModulesInSingleProject, } from '@pnpm/core' import { type ProjectRootDir } from '@pnpm/types' import sinon from 'sinon' +import symlinkDir from 'symlink-dir' import { testDefaults } from './utils' const f = fixtures(__dirname) @@ -23,7 +23,7 @@ test('prune removes extraneous packages', async () => { manifest = await addDependenciesToPackage(manifest, ['applyq@0.2.1'], { ...opts, targetDependenciesField: 'devDependencies' }) manifest = await addDependenciesToPackage(manifest, ['fnumber@0.1.0'], { ...opts, targetDependenciesField: 'optionalDependencies' }) manifest = await addDependenciesToPackage(manifest, ['is-positive@2.0.0', '@zkochan/logger@0.1.0'], opts) - manifest = await link([linkedPkg], path.resolve('node_modules'), { ...opts, manifest, dir: process.cwd() }) + symlinkDir.sync(linkedPkg, path.resolve('node_modules/@pnpm.e2e/hello-world-js-bin')) project.has('@pnpm.e2e/hello-world-js-bin') // external link added diff --git a/pkg-manager/core/test/uninstall.ts b/pkg-manager/core/test/uninstall.ts index d4ee0923757..9c3f5134916 100644 --- a/pkg-manager/core/test/uninstall.ts +++ b/pkg-manager/core/test/uninstall.ts @@ -12,9 +12,9 @@ import { REGISTRY_MOCK_PORT } from '@pnpm/registry-mock' import { fixtures } from '@pnpm/test-fixtures' import { type ProjectRootDir, type PackageManifest } from '@pnpm/types' import { sync as readYamlFile } from 'read-yaml-file' +import symlinkDir from 'symlink-dir' import { addDependenciesToPackage, - link, mutateModules, mutateModulesInSingleProject, } from '@pnpm/core' @@ -195,15 +195,16 @@ test('relative link is uninstalled', async () => { const linkedPkgPath = path.resolve('..', linkedPkgName) f.copy(linkedPkgName, linkedPkgPath) - const manifest = await link([`../${linkedPkgName}`], path.join(process.cwd(), 'node_modules'), opts as (typeof opts & { dir: string, manifest: PackageManifest })) + symlinkDir.sync(linkedPkgPath, path.resolve('node_modules/@pnpm.e2e/hello-world-js-bin')) + project.has('@pnpm.e2e/hello-world-js-bin') await mutateModulesInSingleProject({ - dependencyNames: [linkedPkgName], - manifest, + dependencyNames: ['@pnpm.e2e/hello-world-js-bin'], + manifest: {}, mutation: 'uninstallSome', rootDir: process.cwd() as ProjectRootDir, }, opts) - project.hasNot(linkedPkgName) + project.hasNot('@pnpm.e2e/hello-world-js-bin') }) test('pendingBuilds gets updated after uninstall', async () => { diff --git a/pkg-manager/core/test/unlink.ts b/pkg-manager/core/test/unlink.ts deleted file mode 100644 index c254427a8c9..00000000000 --- a/pkg-manager/core/test/unlink.ts +++ /dev/null @@ -1,296 +0,0 @@ -import fs from 'fs' -import path from 'path' -import { - addDependenciesToPackage, - install, - link, - mutateModulesInSingleProject, -} from '@pnpm/core' -import { prepareEmpty } from '@pnpm/prepare' -import { WANTED_LOCKFILE } from '@pnpm/constants' -import { addDistTag } from '@pnpm/registry-mock' -import { type ProjectRootDir } from '@pnpm/types' -import sinon from 'sinon' -import writeJsonFile from 'write-json-file' -import isInnerLink from 'is-inner-link' -import { testDefaults } from './utils' - -test('unlink 1 package that exists in package.json', async () => { - const project = prepareEmpty() - process.chdir('..') - - await Promise.all([ - writeJsonFile('is-subdir/package.json', { - dependencies: { - 'is-windows': '^1.0.0', - }, - name: 'is-subdir', - version: '1.0.0', - }), - writeJsonFile('is-positive/package.json', { - name: 'is-positive', - version: '1.0.0', - }), - ]) - - // TODO: unset useLockfileV6 - const opts = testDefaults({ fastUnpack: false, store: path.resolve('.store'), useLockfileV6: false }) - - let manifest = await link( - ['is-subdir', 'is-positive'], - path.join('project', 'node_modules'), - { - ...opts, - dir: path.resolve('project'), - manifest: { - dependencies: { - 'is-positive': '^1.0.0', - 'is-subdir': '^1.0.0', - }, - }, - } - ) - - process.chdir('project') - - manifest = await install(manifest, opts) - - await mutateModulesInSingleProject({ - dependencyNames: ['is-subdir'], - manifest, - mutation: 'unlinkSome', - rootDir: process.cwd() as ProjectRootDir, - }, opts) - - expect(typeof project.requireModule('is-subdir')).toBe('function') - expect((await isInnerLink('node_modules', 'is-positive')).isInner).toBeFalsy() -}) - -test("don't update package when unlinking", async () => { - const project = prepareEmpty() - - await addDistTag({ package: '@pnpm.e2e/foo', version: '100.0.0', distTag: 'latest' }) - const opts = testDefaults({ dir: process.cwd() }) - let manifest = await addDependenciesToPackage({}, ['@pnpm.e2e/foo'], opts) - - process.chdir('..') - - writeJsonFile.sync('foo/package.json', { - name: '@pnpm.e2e/foo', - version: '100.0.0', - }) - - manifest = await link(['foo'], path.join('project', 'node_modules'), { ...opts, manifest }) - await addDistTag({ package: '@pnpm.e2e/foo', version: '100.1.0', distTag: 'latest' }) - - process.chdir('project') - await mutateModulesInSingleProject({ - dependencyNames: ['@pnpm.e2e/foo'], - manifest, - mutation: 'unlinkSome', - rootDir: process.cwd() as ProjectRootDir, - }, opts) - - expect(project.requireModule('@pnpm.e2e/foo/package.json').version).toBe('100.0.0') -}) - -test(`don't update package when unlinking. Initial link is done on a package w/o ${WANTED_LOCKFILE}`, async () => { - const project = prepareEmpty() - - const opts = testDefaults({ dir: process.cwd(), resolutionMode: 'lowest-direct' }) - process.chdir('..') - - writeJsonFile.sync('foo/package.json', { - name: '@pnpm.e2e/foo', - version: '100.0.0', - }) - - const manifest = await link(['foo'], path.join('project', 'node_modules'), { - ...opts, - manifest: { - dependencies: { - '@pnpm.e2e/foo': '^100.0.0', - }, - }, - }) - await addDistTag({ package: '@pnpm.e2e/foo', version: '100.1.0', distTag: 'latest' }) - - process.chdir('project') - const unlinkResult = await mutateModulesInSingleProject({ - dependencyNames: ['@pnpm.e2e/foo'], - manifest, - mutation: 'unlinkSome', - rootDir: process.cwd() as ProjectRootDir, - }, opts) - - expect(project.requireModule('@pnpm.e2e/foo/package.json').version).toBe('100.0.0') - expect(unlinkResult.manifest.dependencies).toStrictEqual({ '@pnpm.e2e/foo': '^100.0.0' }) -}) - -test('unlink 2 packages. One of them exists in package.json', async () => { - const project = prepareEmpty() - const opts = testDefaults({ fastUnpack: false, dir: process.cwd() }) - process.chdir('..') - - await Promise.all([ - writeJsonFile('is-subdir/package.json', { - dependencies: { - 'is-windows': '^1.0.0', - }, - name: 'is-subdir', - version: '1.0.0', - }), - writeJsonFile('is-positive/package.json', { - name: 'is-positive', - version: '1.0.0', - }), - ]) - - const manifest = await link(['is-subdir', 'is-positive'], path.join('project', 'node_modules'), { - ...opts, - manifest: { - dependencies: { - 'is-subdir': '^1.0.0', - }, - }, - }) - - process.chdir('project') - await mutateModulesInSingleProject({ - dependencyNames: ['is-subdir', 'is-positive'], - manifest, - mutation: 'unlinkSome', - rootDir: process.cwd() as ProjectRootDir, - }, opts) - - expect(typeof project.requireModule('is-subdir')).toBe('function') - expect(fs.existsSync(path.join('node_modules', 'is-positive'))).toBeFalsy() -}) - -test('unlink all packages', async () => { - const project = prepareEmpty() - const opts = testDefaults({ fastUnpack: false, dir: process.cwd() }) - process.chdir('..') - - await Promise.all([ - writeJsonFile('is-subdir/package.json', { - dependencies: { - 'is-windows': '^1.0.0', - }, - name: 'is-subdir', - version: '1.0.0', - }), - writeJsonFile('logger/package.json', { - name: '@zkochan/logger', - version: '0.1.0', - }), - ]) - - const manifest = await link(['is-subdir', 'logger'], path.join('project', 'node_modules'), { - ...opts, - manifest: { - dependencies: { - '@zkochan/logger': '^0.1.0', - 'is-subdir': '^1.0.0', - }, - }, - }) - - await mutateModulesInSingleProject({ - manifest, - mutation: 'unlink', - rootDir: path.resolve('project') as ProjectRootDir, - }, opts) - - expect(typeof project.requireModule('is-subdir')).toBe('function') - expect(typeof project.requireModule('@zkochan/logger')).toBe('object') -}) - -test("don't warn about scoped packages when running unlink w/o params", async () => { - prepareEmpty() - - const manifest = await addDependenciesToPackage({}, ['@zkochan/logger'], testDefaults()) - - const reporter = sinon.spy() - await mutateModulesInSingleProject({ - manifest, - mutation: 'unlink', - rootDir: process.cwd() as ProjectRootDir, - }, testDefaults({ reporter })) - - expect(reporter.calledWithMatch({ - level: 'warn', - message: '@zkochan/logger is not an external link', - })).toBeFalsy() -}) - -test("don't unlink package that is not a link", async () => { - prepareEmpty() - - const reporter = sinon.spy() - - const manifest = await addDependenciesToPackage({}, ['is-positive'], testDefaults()) - - await mutateModulesInSingleProject({ - dependencyNames: ['is-positive'], - manifest, - mutation: 'unlinkSome', - rootDir: process.cwd() as ProjectRootDir, - }, testDefaults({ reporter })) - - expect(reporter.calledWithMatch({ - level: 'warn', - message: 'is-positive is not an external link', - })).toBeTruthy() -}) - -test('unlink would remove global bin', async () => { - prepareEmpty() - process.chdir('..') - fs.mkdirSync('bin') - fs.mkdirSync('is-subdir') - fs.writeFileSync('is-subdir/index.js', ' ') - - await Promise.all([ - writeJsonFile('is-subdir/package.json', { - bin: 'index.js', - dependencies: { - 'is-windows': '^1.0.0', - }, - name: 'is-subdir', - version: '1.0.0', - }), - ]) - - const opts = testDefaults({ - fastUnpack: false, - globalBin: path.resolve('bin'), - linkToBin: path.resolve('bin'), - store: path.resolve('.store'), - }) - - const manifest = await link( - ['is-subdir'], - path.join('project', 'node_modules'), - { - ...opts, - dir: path.resolve('project'), - manifest: { - dependencies: { - 'is-subdir': '^1.0.0', - }, - name: 'is-subdir', - }, - } - ) - expect(fs.existsSync(path.resolve('bin/is-subdir'))).toBeTruthy() - - await mutateModulesInSingleProject({ - dependencyNames: ['is-subdir'], - manifest, - mutation: 'unlinkSome', - rootDir: process.cwd() as ProjectRootDir, - }, opts) - - expect(fs.existsSync(path.resolve('bin/is-subdir'))).toBeFalsy() -}) diff --git a/pkg-manager/core/tsconfig.json b/pkg-manager/core/tsconfig.json index f42393f0f8b..e8666199394 100644 --- a/pkg-manager/core/tsconfig.json +++ b/pkg-manager/core/tsconfig.json @@ -40,10 +40,10 @@ "path": "../../config/parse-overrides" }, { - "path": "../../crypto/object-hasher" + "path": "../../crypto/hash" }, { - "path": "../../crypto/polyfill" + "path": "../../crypto/object-hasher" }, { "path": "../../deps/graph-sequencer" @@ -105,9 +105,6 @@ { "path": "../../packages/core-loggers" }, - { - "path": "../../packages/crypto.base32-hash" - }, { "path": "../../packages/dependency-path" }, diff --git a/pkg-manager/get-context/package.json b/pkg-manager/get-context/package.json index 29314708f29..3aed8559852 100644 --- a/pkg-manager/get-context/package.json +++ b/pkg-manager/get-context/package.json @@ -40,15 +40,12 @@ "dependencies": { "@pnpm/constants": "workspace:*", "@pnpm/core-loggers": "workspace:*", - "@pnpm/error": "workspace:*", "@pnpm/lockfile.fs": "workspace:*", "@pnpm/modules-yaml": "workspace:*", "@pnpm/read-projects-context": "workspace:*", "@pnpm/resolver-base": "workspace:*", "@pnpm/types": "workspace:*", - "@zkochan/rimraf": "catalog:", "ci-info": "catalog:", - "enquirer": "catalog:", "path-absolute": "catalog:", "ramda": "catalog:" }, diff --git a/pkg-manager/get-context/src/index.ts b/pkg-manager/get-context/src/index.ts index d29f73042e2..9dcad4dac8c 100644 --- a/pkg-manager/get-context/src/index.ts +++ b/pkg-manager/get-context/src/index.ts @@ -1,9 +1,7 @@ import { promises as fs } from 'fs' import path from 'path' import { contextLogger, packageManifestLogger } from '@pnpm/core-loggers' -import { PnpmError } from '@pnpm/error' import { type Lockfile } from '@pnpm/lockfile.fs' -import { logger } from '@pnpm/logger' import { type IncludedDependencies, type Modules, @@ -12,7 +10,6 @@ import { readProjectsContext } from '@pnpm/read-projects-context' import { type WorkspacePackages } from '@pnpm/resolver-base' import { type DepPath, - DEPENDENCIES_FIELDS, type HoistedDependencies, type ProjectId, type ProjectManifest, @@ -22,19 +19,14 @@ import { type ProjectRootDir, type ProjectRootDirRealPath, } from '@pnpm/types' -import rimraf from '@zkochan/rimraf' -import { isCI } from 'ci-info' -import enquirer from 'enquirer' import pathAbsolute from 'path-absolute' import clone from 'ramda/src/clone' -import equals from 'ramda/src/equals' -import { checkCompatibility } from './checkCompatibility' -import { UnexpectedStoreError } from './checkCompatibility/UnexpectedStoreError' -import { UnexpectedVirtualStoreDirError } from './checkCompatibility/UnexpectedVirtualStoreDirError' import { readLockfiles } from './readLockfiles' -export { UnexpectedStoreError, UnexpectedVirtualStoreDirError } - +/** + * Note that some fields are affected by modules directory state. Such fields should be used for + * mutating the modules directory only or in a manner that does not influence dependency resolution. + */ export interface PnpmContext { currentLockfile: Lockfile currentLockfileIsUpToDate: boolean @@ -42,9 +34,11 @@ export interface PnpmContext { existsWantedLockfile: boolean existsNonEmptyWantedLockfile: boolean extraBinPaths: string[] + /** Affected by existing modules directory, if it exists. */ extraNodePaths: string[] lockfileHadConflicts: boolean hoistedDependencies: HoistedDependencies + /** Required included dependencies or dependencies currently included by the modules directory. */ include: IncludedDependencies modulesFile: Modules | null pendingBuilds: string[] @@ -54,11 +48,17 @@ export interface PnpmContext { } & HookOptions & Required> rootModulesDir: string hoistPattern: string[] | undefined + /** As applied to existing modules directory, if it exists. */ + currentHoistPattern: string[] | undefined hoistedModulesDir: string publicHoistPattern: string[] | undefined + /** As applied to existing modules directory, if it exists. */ + currentPublicHoistPattern: string[] | undefined lockfileDir: string virtualStoreDir: string + /** As applied to existing modules directory, otherwise options. */ virtualStoreDirMaxLength: number + /** As applied to existing modules directory, if it exists. */ skipped: Set storeDir: string wantedLockfile: Lockfile @@ -87,7 +87,6 @@ export interface GetContextOptions { allProjects: Array confirmModulesPurge?: boolean force: boolean - forceNewModules?: boolean frozenLockfile?: boolean extraBinPaths: string[] extendNodePath?: boolean @@ -112,47 +111,14 @@ export interface GetContextOptions { forcePublicHoistPattern?: boolean global?: boolean } -interface ImporterToPurge { - modulesDir: string - rootDir: ProjectRootDir -} export async function getContext ( opts: GetContextOptions ): Promise { const modulesDir = opts.modulesDir ?? 'node_modules' - let importersContext = await readProjectsContext(opts.allProjects, { lockfileDir: opts.lockfileDir, modulesDir }) + const importersContext = await readProjectsContext(opts.allProjects, { lockfileDir: opts.lockfileDir, modulesDir }) const virtualStoreDir = pathAbsolute(opts.virtualStoreDir ?? path.join(modulesDir, '.pnpm'), opts.lockfileDir) - if (importersContext.modules != null) { - const { purged } = await validateModules(importersContext.modules, importersContext.projects, { - currentHoistPattern: importersContext.currentHoistPattern, - currentPublicHoistPattern: importersContext.currentPublicHoistPattern, - forceNewModules: opts.forceNewModules === true, - include: opts.include, - lockfileDir: opts.lockfileDir, - modulesDir, - registries: opts.registries, - storeDir: opts.storeDir, - virtualStoreDir, - virtualStoreDirMaxLength: opts.virtualStoreDirMaxLength, - confirmModulesPurge: opts.confirmModulesPurge && !isCI, - - forceHoistPattern: opts.forceHoistPattern, - hoistPattern: opts.hoistPattern, - - forcePublicHoistPattern: opts.forcePublicHoistPattern, - publicHoistPattern: opts.publicHoistPattern, - global: opts.global, - }) - if (purged) { - importersContext = await readProjectsContext(opts.allProjects, { - lockfileDir: opts.lockfileDir, - modulesDir, - }) - } - } - await fs.mkdir(opts.storeDir, { recursive: true }) for (const project of opts.allProjects) { @@ -175,19 +141,20 @@ export async function getContext ( if (opts.hoistPattern?.length) { extraBinPaths.unshift(path.join(hoistedModulesDir, '.bin')) } - const hoistPattern = importersContext.currentHoistPattern ?? opts.hoistPattern const ctx: PnpmContext = { extraBinPaths, - extraNodePaths: getExtraNodePaths({ extendNodePath: opts.extendNodePath, nodeLinker: opts.nodeLinker, hoistPattern, virtualStoreDir }), + extraNodePaths: getExtraNodePaths({ extendNodePath: opts.extendNodePath, nodeLinker: opts.nodeLinker, hoistPattern: importersContext.currentHoistPattern ?? opts.hoistPattern, virtualStoreDir }), hoistedDependencies: importersContext.hoistedDependencies, hoistedModulesDir, - hoistPattern, + hoistPattern: opts.hoistPattern, + currentHoistPattern: importersContext.currentHoistPattern, include: opts.include ?? importersContext.include, lockfileDir: opts.lockfileDir, modulesFile: importersContext.modules, pendingBuilds: importersContext.pendingBuilds, projects: Object.fromEntries(importersContext.projects.map((project) => [project.rootDir, project])), - publicHoistPattern: importersContext.currentPublicHoistPattern ?? opts.publicHoistPattern, + publicHoistPattern: opts.publicHoistPattern, + currentPublicHoistPattern: importersContext.currentPublicHoistPattern, registries: opts.registries, rootModulesDir: importersContext.rootModulesDir, skipped: importersContext.skipped, @@ -218,192 +185,13 @@ export async function getContext ( return ctx } -async function validateModules ( - modules: Modules, - projects: Array<{ - modulesDir: string - id: string - rootDir: ProjectRootDir - }>, - opts: { - currentHoistPattern?: string[] - currentPublicHoistPattern?: string[] - forceNewModules: boolean - include?: IncludedDependencies - lockfileDir: string - modulesDir: string - registries: Registries - storeDir: string - virtualStoreDir: string - virtualStoreDirMaxLength: number - confirmModulesPurge?: boolean - - hoistPattern?: string[] | undefined - forceHoistPattern?: boolean - - publicHoistPattern?: string[] | undefined - forcePublicHoistPattern?: boolean - global?: boolean - } -): Promise<{ purged: boolean }> { - const rootProject = projects.find(({ id }) => id === '.') - if (opts.virtualStoreDirMaxLength !== modules.virtualStoreDirMaxLength) { - if (opts.forceNewModules && (rootProject != null)) { - await purgeModulesDirsOfImporter(opts, rootProject) - return { purged: true } - } - throw new PnpmError( - 'VIRTUAL_STORE_DIR_MAX_LENGTH_DIFF', - 'This modules directory was created using a different virtual-store-dir-max-length value.' + - ' Run "pnpm install" to recreate the modules directory.' - ) - } - if ( - opts.forcePublicHoistPattern && - // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing - !equals(modules.publicHoistPattern, opts.publicHoistPattern || undefined) - ) { - if (opts.forceNewModules && (rootProject != null)) { - await purgeModulesDirsOfImporter(opts, rootProject) - return { purged: true } - } - throw new PnpmError( - 'PUBLIC_HOIST_PATTERN_DIFF', - 'This modules directory was created using a different public-hoist-pattern value.' + - ' Run "pnpm install" to recreate the modules directory.' - ) - } - - const importersToPurge: ImporterToPurge[] = [] - - if (opts.forceHoistPattern && (rootProject != null)) { - try { - // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing - if (!equals(opts.currentHoistPattern, opts.hoistPattern || undefined)) { - throw new PnpmError( - 'HOIST_PATTERN_DIFF', - 'This modules directory was created using a different hoist-pattern value.' + - ' Run "pnpm install" to recreate the modules directory.' - ) - } - } catch (err: any) { // eslint-disable-line - if (!opts.forceNewModules) throw err - importersToPurge.push(rootProject) - } - } - for (const project of projects) { - try { - checkCompatibility(modules, { - modulesDir: project.modulesDir, - storeDir: opts.storeDir, - virtualStoreDir: opts.virtualStoreDir, - }) - if (opts.lockfileDir !== project.rootDir && (opts.include != null) && modules.included) { - for (const depsField of DEPENDENCIES_FIELDS) { - if (opts.include[depsField] !== modules.included[depsField]) { - throw new PnpmError('INCLUDED_DEPS_CONFLICT', - `modules directory (at "${opts.lockfileDir}") was installed with ${stringifyIncludedDeps(modules.included)}. ` + - `Current install wants ${stringifyIncludedDeps(opts.include)}.` - ) - } - } - } - } catch (err: any) { // eslint-disable-line - if (!opts.forceNewModules) throw err - importersToPurge.push(project) - } - } - if (importersToPurge.length > 0 && (rootProject == null)) { - importersToPurge.push({ - modulesDir: path.join(opts.lockfileDir, opts.modulesDir), - rootDir: opts.lockfileDir as ProjectRootDir, - }) - } - - const purged = importersToPurge.length > 0 - if (purged) { - await purgeModulesDirsOfImporters(opts, importersToPurge) - } - - return { purged } -} - -async function purgeModulesDirsOfImporter ( - opts: { - confirmModulesPurge?: boolean - virtualStoreDir: string - }, - importer: ImporterToPurge -): Promise { - return purgeModulesDirsOfImporters(opts, [importer]) -} - -async function purgeModulesDirsOfImporters ( - opts: { - confirmModulesPurge?: boolean - virtualStoreDir: string - }, - importers: ImporterToPurge[] -): Promise { - if (opts.confirmModulesPurge ?? true) { - const confirmed = await enquirer.prompt<{ question: boolean }>({ - type: 'confirm', - name: 'question', - message: importers.length === 1 - ? `The modules directory at "${importers[0].modulesDir}" will be removed and reinstalled from scratch. Proceed?` - : 'The modules directories will be removed and reinstalled from scratch. Proceed?', - initial: true, - }) - if (!confirmed.question) { - throw new PnpmError('ABORTED_REMOVE_MODULES_DIR', 'Aborted removal of modules directory') - } - } - await Promise.all(importers.map(async (importer) => { - logger.info({ - message: `Recreating ${importer.modulesDir}`, - prefix: importer.rootDir, - }) - try { - // We don't remove the actual modules directory, just the contents of it. - // 1. we will need the directory anyway. - // 2. in some setups, pnpm won't even have permission to remove the modules directory. - await removeContentsOfDir(importer.modulesDir, opts.virtualStoreDir) - } catch (err: any) { // eslint-disable-line - if (err.code !== 'ENOENT') throw err - } - })) -} - -async function removeContentsOfDir (dir: string, virtualStoreDir: string): Promise { - const items = await fs.readdir(dir) - await Promise.all(items.map(async (item) => { - // The non-pnpm related hidden files are kept - if ( - item.startsWith('.') && - item !== '.bin' && - item !== '.modules.yaml' && - !dirsAreEqual(path.join(dir, item), virtualStoreDir) - ) { - return - } - await rimraf(path.join(dir, item)) - })) -} - -function dirsAreEqual (dir1: string, dir2: string): boolean { - return path.relative(dir1, dir2) === '' -} - -function stringifyIncludedDeps (included: IncludedDependencies): string { - return DEPENDENCIES_FIELDS.filter((depsField) => included[depsField]).join(', ') -} - export interface PnpmSingleContext { currentLockfile: Lockfile currentLockfileIsUpToDate: boolean existsCurrentLockfile: boolean existsWantedLockfile: boolean existsNonEmptyWantedLockfile: boolean + /** Affected by existing modules directory, if it exists. */ extraBinPaths: string[] extraNodePaths: string[] lockfileHadConflicts: boolean @@ -414,6 +202,7 @@ export interface PnpmSingleContext { modulesDir: string importerId: string prefix: string + /** Required included dependencies or dependencies currently included by the modules directory. */ include: IncludedDependencies modulesFile: Modules | null pendingBuilds: string[] @@ -422,6 +211,7 @@ export interface PnpmSingleContext { rootModulesDir: string lockfileDir: string virtualStoreDir: string + /** As applied to existing modules directory, if it exists. */ skipped: Set storeDir: string wantedLockfile: Lockfile @@ -435,7 +225,6 @@ export async function getContextForSingleImporter ( excludeLinksFromLockfile: boolean peersSuffixMaxLength: number force: boolean - forceNewModules?: boolean confirmModulesPurge?: boolean extraBinPaths: string[] extendNodePath?: boolean @@ -458,12 +247,10 @@ export async function getContextForSingleImporter ( publicHoistPattern?: string[] | undefined forcePublicHoistPattern?: boolean - }, - alreadyPurged: boolean = false + } ): Promise { const { currentHoistPattern, - currentPublicHoistPattern, hoistedDependencies, projects, include, @@ -491,31 +278,6 @@ export async function getContextForSingleImporter ( const importerId = importer.id const virtualStoreDir = pathAbsolute(opts.virtualStoreDir ?? 'node_modules/.pnpm', opts.lockfileDir) - if ((modules != null) && !alreadyPurged) { - const { purged } = await validateModules(modules, projects, { - currentHoistPattern, - currentPublicHoistPattern, - forceNewModules: opts.forceNewModules === true, - include: opts.include, - lockfileDir: opts.lockfileDir, - modulesDir: opts.modulesDir ?? 'node_modules', - registries: opts.registries, - storeDir: opts.storeDir, - virtualStoreDir, - virtualStoreDirMaxLength: opts.virtualStoreDirMaxLength, - confirmModulesPurge: opts.confirmModulesPurge && !isCI, - - forceHoistPattern: opts.forceHoistPattern, - hoistPattern: opts.hoistPattern, - - forcePublicHoistPattern: opts.forcePublicHoistPattern, - publicHoistPattern: opts.publicHoistPattern, - }) - if (purged) { - return getContextForSingleImporter(manifest, opts, true) - } - } - await fs.mkdir(storeDir, { recursive: true }) const extraBinPaths = [ ...opts.extraBinPaths || [], @@ -524,13 +286,12 @@ export async function getContextForSingleImporter ( if (opts.hoistPattern?.length) { extraBinPaths.unshift(path.join(hoistedModulesDir, '.bin')) } - const hoistPattern = currentHoistPattern ?? opts.hoistPattern const ctx: PnpmSingleContext = { extraBinPaths, - extraNodePaths: getExtraNodePaths({ extendNodePath: opts.extendNodePath, nodeLinker: opts.nodeLinker, hoistPattern, virtualStoreDir }), + extraNodePaths: getExtraNodePaths({ extendNodePath: opts.extendNodePath, nodeLinker: opts.nodeLinker, hoistPattern: currentHoistPattern ?? opts.hoistPattern, virtualStoreDir }), hoistedDependencies, hoistedModulesDir, - hoistPattern, + hoistPattern: opts.hoistPattern, importerId, include: opts.include ?? include, lockfileDir: opts.lockfileDir, @@ -539,7 +300,7 @@ export async function getContextForSingleImporter ( modulesFile: modules, pendingBuilds, prefix: opts.dir, - publicHoistPattern: currentPublicHoistPattern ?? opts.publicHoistPattern, + publicHoistPattern: opts.publicHoistPattern, registries: { ...opts.registries, ...registries, diff --git a/pkg-manager/get-context/tsconfig.json b/pkg-manager/get-context/tsconfig.json index fd19ba4562a..7566b8f34ee 100644 --- a/pkg-manager/get-context/tsconfig.json +++ b/pkg-manager/get-context/tsconfig.json @@ -18,9 +18,6 @@ { "path": "../../packages/core-loggers" }, - { - "path": "../../packages/error" - }, { "path": "../../packages/logger" }, diff --git a/pkg-manager/headless/test/index.ts b/pkg-manager/headless/test/index.ts index 9dc1fe5d512..060d4708116 100644 --- a/pkg-manager/headless/test/index.ts +++ b/pkg-manager/headless/test/index.ts @@ -677,15 +677,14 @@ test.each([['isolated'], ['hoisted']])('using side effects cache with nodeLinker }, {}, {}, { packageImportMethod: 'copy' }) await headlessInstall(opts) - const cafsDir = path.join(opts.storeDir, 'files') - const cacheIntegrityPath = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')) + const cacheIntegrityPath = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') const cacheIntegrity = loadJsonFile.sync(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any expect(cacheIntegrity!.sideEffects).toBeTruthy() - const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}` - expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'generated-by-postinstall.js']) - delete cacheIntegrity!.sideEffects[sideEffectsKey]['generated-by-postinstall.js'] + const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}` + expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'added', 'generated-by-postinstall.js']) + delete cacheIntegrity!.sideEffects[sideEffectsKey].added['generated-by-postinstall.js'] - expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'generated-by-preinstall.js']) + expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'added', 'generated-by-preinstall.js']) writeJsonFile.sync(cacheIntegrityPath, cacheIntegrity) prefix = f.prepare('side-effects') diff --git a/pkg-manager/package-requester/src/packageRequester.ts b/pkg-manager/package-requester/src/packageRequester.ts index 0c7c5ae4626..af82d520154 100644 --- a/pkg-manager/package-requester/src/packageRequester.ts +++ b/pkg-manager/package-requester/src/packageRequester.ts @@ -107,14 +107,13 @@ export function createPackageRequester ( concurrency: networkConcurrency, }) - const cafsDir = path.join(opts.storeDir, 'files') - const getIndexFilePathInCafs = _getIndexFilePathInCafs.bind(null, cafsDir) + const getIndexFilePathInCafs = _getIndexFilePathInCafs.bind(null, opts.storeDir) const fetch = fetcher.bind(null, opts.fetchers, opts.cafs) const fetchPackageToStore = fetchToStore.bind(null, { - readPkgFromCafs: _readPkgFromCafs.bind(null, cafsDir, opts.verifyStoreIntegrity), + readPkgFromCafs: _readPkgFromCafs.bind(null, opts.storeDir, opts.verifyStoreIntegrity), fetch, fetchingLocker: new Map(), - getFilePathByModeInCafs: _getFilePathByModeInCafs.bind(null, cafsDir), + getFilePathByModeInCafs: _getFilePathByModeInCafs.bind(null, opts.storeDir), getIndexFilePathInCafs, requestsQueue: Object.assign(requestsQueue, { counter: 0, @@ -308,7 +307,7 @@ interface FetchLock { function getFilesIndexFilePath ( ctx: { - getIndexFilePathInCafs: (integrity: string) => string + getIndexFilePathInCafs: (integrity: string, pkgId: string) => string storeDir: string virtualStoreDirMaxLength: number }, @@ -317,7 +316,7 @@ function getFilesIndexFilePath ( const targetRelative = depPathToFilename(opts.pkg.id, ctx.virtualStoreDirMaxLength) const target = path.join(ctx.storeDir, targetRelative) const filesIndexFile = (opts.pkg.resolution as TarballResolution).integrity - ? ctx.getIndexFilePathInCafs((opts.pkg.resolution as TarballResolution).integrity!) + ? ctx.getIndexFilePathInCafs((opts.pkg.resolution as TarballResolution).integrity!, opts.pkg.id) : path.join(target, opts.ignoreScripts ? 'integrity-not-built.json' : 'integrity.json') return { filesIndexFile, target } } @@ -334,7 +333,7 @@ function fetchToStore ( opts: FetchOptions ) => Promise fetchingLocker: Map - getIndexFilePathInCafs: (integrity: string) => string + getIndexFilePathInCafs: (integrity: string, pkgId: string) => string getFilePathByModeInCafs: (integrity: string, mode: number) => string requestsQueue: { add: (fn: () => Promise, opts: { priority: number }) => Promise diff --git a/pkg-manager/plugin-commands-installation/package.json b/pkg-manager/plugin-commands-installation/package.json index 591b7fea13d..eb86fe37e4f 100644 --- a/pkg-manager/plugin-commands-installation/package.json +++ b/pkg-manager/plugin-commands-installation/package.json @@ -81,6 +81,7 @@ "@pnpm/plugin-commands-rebuild": "workspace:*", "@pnpm/pnpmfile": "workspace:*", "@pnpm/read-project-manifest": "workspace:*", + "@pnpm/write-project-manifest": "workspace:*", "@pnpm/resolver-base": "workspace:*", "@pnpm/semver-diff": "catalog:", "@pnpm/sort-packages": "workspace:*", @@ -101,7 +102,6 @@ "mem": "catalog:", "p-filter": "catalog:", "p-limit": "catalog:", - "path-absolute": "catalog:", "ramda": "catalog:", "render-help": "catalog:", "version-selector-type": "catalog:", diff --git a/pkg-manager/plugin-commands-installation/src/install.ts b/pkg-manager/plugin-commands-installation/src/install.ts index d22b34c8097..d306a8b7260 100644 --- a/pkg-manager/plugin-commands-installation/src/install.ts +++ b/pkg-manager/plugin-commands-installation/src/install.ts @@ -308,6 +308,7 @@ export type InstallCommandOptions = Pick { const fetchFullMetadata: true | undefined = opts.rootProjectManifest?.pnpm?.supportedArchitectures?.libc && true const installDepsOptions: InstallDepsOptions = { ...opts, - frozenLockfileIfExists: isCI && !opts.lockfileOnly && + frozenLockfileIfExists: opts.frozenLockfileIfExists ?? ( + isCI && !opts.lockfileOnly && typeof opts.rawLocalConfig['frozen-lockfile'] === 'undefined' && - typeof opts.rawLocalConfig['prefer-frozen-lockfile'] === 'undefined', + typeof opts.rawLocalConfig['prefer-frozen-lockfile'] === 'undefined' + ), include, includeDirect: include, prepareExecutionEnv: prepareExecutionEnv.bind(null, opts), diff --git a/pkg-manager/plugin-commands-installation/src/link.ts b/pkg-manager/plugin-commands-installation/src/link.ts index 991a7768641..76b97747eda 100644 --- a/pkg-manager/plugin-commands-installation/src/link.ts +++ b/pkg-manager/plugin-commands-installation/src/link.ts @@ -1,53 +1,44 @@ import path from 'path' import { docsUrl, - getConfig, - readProjectManifest, - readProjectManifestOnly, tryReadProjectManifest, + type ReadProjectManifestOpts, } from '@pnpm/cli-utils' import { UNIVERSAL_OPTIONS } from '@pnpm/common-cli-options-help' -import { type Config, getOptionsFromRootManifest, types as allTypes } from '@pnpm/config' +import { type Config, types as allTypes } from '@pnpm/config' +import { DEPENDENCIES_FIELDS, type ProjectManifest, type Project } from '@pnpm/types' import { PnpmError } from '@pnpm/error' -import { findWorkspaceDir } from '@pnpm/find-workspace-dir' import { arrayOfWorkspacePackagesToMap } from '@pnpm/get-context' import { findWorkspacePackages } from '@pnpm/workspace.find-packages' -import { type StoreController } from '@pnpm/package-store' -import { createOrConnectStoreControllerCached, type CreateStoreControllerOptions } from '@pnpm/store-connection-manager' +import { writeProjectManifest } from '@pnpm/write-project-manifest' import { - addDependenciesToPackage, - install, - type InstallOptions, - link, - type LinkFunctionOptions, type WorkspacePackages, } from '@pnpm/core' import { logger } from '@pnpm/logger' -import { type Project } from '@pnpm/types' -import pLimit from 'p-limit' -import pathAbsolute from 'path-absolute' import pick from 'ramda/src/pick' import partition from 'ramda/src/partition' import renderHelp from 'render-help' -import * as installCommand from './install' import { getSaveType } from './getSaveType' +import * as install from './install' // @ts-expect-error const isWindows = process.platform === 'win32' || global['FAKE_WINDOWS'] const isFilespec = isWindows ? /^(?:[.]|~[/]|[/\\]|[a-zA-Z]:)/ : /^(?:[.]|~[/]|[/]|[a-zA-Z]:)/ -const installLimit = pLimit(4) -type LinkOpts = CreateStoreControllerOptions & Pick & Partial> +| 'globalPkgDir' +> & Partial> & install.InstallCommandOptions export const rcOptionsTypes = cliOptionsTypes @@ -77,21 +68,13 @@ export function help (): string { { title: 'Options', - list: [ - ...UNIVERSAL_OPTIONS, - { - description: 'Link package to/from global node_modules', - name: '--global', - shortAlias: '-g', - }, - ], + list: UNIVERSAL_OPTIONS, }, ], url: docsUrl('link'), usages: [ - 'pnpm link ', - 'pnpm link --global (in package dir)', - 'pnpm link --global ', + 'pnpm link ', + 'pnpm link', ], }) } @@ -121,9 +104,6 @@ export async function handler ( opts: LinkOpts, params?: string[] ): Promise { - const cwd = process.cwd() - - const storeControllerCache = new Map>() let workspacePackagesArr: Project[] let workspacePackages!: WorkspacePackages if (opts.workspaceDir) { @@ -136,12 +116,10 @@ export async function handler ( workspacePackages = new Map() } - const store = await createOrConnectStoreControllerCached(storeControllerCache, opts) const linkOpts = Object.assign(opts, { - storeController: store.ctrl, - storeDir: store.dir, targetDependenciesField: getSaveType(opts), workspacePackages, + binsDir: opts.bin, }) if (opts.cliOptions?.global && !opts.bin) { @@ -150,113 +128,60 @@ export async function handler ( }) } - const linkCwdDir = opts.cliOptions?.dir && opts.cliOptions?.global ? path.resolve(opts.cliOptions.dir) : cwd - // pnpm link if ((params == null) || (params.length === 0)) { + const cwd = process.cwd() if (path.relative(linkOpts.dir, cwd) === '') { throw new PnpmError('LINK_BAD_PARAMS', 'You must provide a parameter') } - await checkPeerDeps(linkCwdDir, opts) + await checkPeerDeps(cwd, opts) - const { manifest, writeProjectManifest } = await tryReadProjectManifest(opts.dir, opts) - const newManifest = await addDependenciesToPackage( - manifest ?? {}, - [`link:${linkCwdDir}`], - linkOpts - ) - await writeProjectManifest(newManifest) + const newManifest = opts.rootProjectManifest ?? {} + await addLinkToManifest(opts, newManifest, cwd, linkOpts.dir) + await writeProjectManifest(path.join(opts.rootProjectManifestDir, 'package.json'), newManifest) + await install.handler({ + ...linkOpts, + frozenLockfileIfExists: false, + rootProjectManifest: newManifest, + }) return } const [pkgPaths, pkgNames] = partition((inp) => isFilespec.test(inp), params) - await Promise.all( - pkgPaths.map(async (dir) => installLimit(async () => { - const s = await createOrConnectStoreControllerCached(storeControllerCache, opts) - const config = await getConfig( - { ...opts.cliOptions, dir }, - { - excludeReporter: true, - rcOptionsTypes: installCommand.rcOptionsTypes(), - workspaceDir: await findWorkspaceDir(dir), - } - ) - await install( - await readProjectManifestOnly(dir, opts), { - ...config, - ...getOptionsFromRootManifest(config.rootProjectManifestDir, config.rootProjectManifest ?? {}), - include: { - dependencies: config.production !== false, - devDependencies: config.dev !== false, - optionalDependencies: config.optional !== false, - }, - storeController: s.ctrl, - storeDir: s.dir, - workspacePackages, - } as InstallOptions - ) - })) - ) - - if (pkgNames.length > 0) { - let globalPkgNames!: string[] - if (opts.workspaceDir) { - workspacePackagesArr = await findWorkspacePackages(opts.workspaceDir, { - ...opts, - patterns: opts.workspacePackagePatterns, - }) - - const pkgsFoundInWorkspace = workspacePackagesArr - .filter(({ manifest }) => manifest.name && pkgNames.includes(manifest.name)) - pkgsFoundInWorkspace.forEach((pkgFromWorkspace) => pkgPaths.push(pkgFromWorkspace.rootDir)) - - if ((pkgsFoundInWorkspace.length > 0) && !linkOpts.targetDependenciesField) { - linkOpts.targetDependenciesField = 'dependencies' - } - - globalPkgNames = pkgNames.filter((pkgName) => !pkgsFoundInWorkspace.some((pkgFromWorkspace) => pkgFromWorkspace.manifest.name === pkgName)) - } else { - globalPkgNames = pkgNames - } - const globalPkgPath = pathAbsolute(opts.dir) - globalPkgNames.forEach((pkgName) => pkgPaths.push(path.join(globalPkgPath, 'node_modules', pkgName))) - } - - const { manifest, writeProjectManifest } = await readProjectManifest(linkCwdDir, opts) + pkgNames.forEach((pkgName) => pkgPaths.push(path.join(opts.globalPkgDir, 'node_modules', pkgName))) + const newManifest = opts.rootProjectManifest ?? {} await Promise.all( pkgPaths.map(async (dir) => { + await addLinkToManifest(opts, newManifest, dir, opts.dir) await checkPeerDeps(dir, opts) }) ) - const linkConfig = await getConfig( - { ...opts.cliOptions, dir: cwd }, - { - excludeReporter: true, - rcOptionsTypes: installCommand.rcOptionsTypes(), - workspaceDir: await findWorkspaceDir(cwd), + await writeProjectManifest(path.join(opts.rootProjectManifestDir, 'package.json'), newManifest) + await install.handler({ + ...linkOpts, + frozenLockfileIfExists: false, + rootProjectManifest: newManifest, + }) +} + +async function addLinkToManifest (opts: ReadProjectManifestOpts, manifest: ProjectManifest, linkedDepDir: string, dependentDir: string) { + if (!manifest.pnpm) { + manifest.pnpm = { + overrides: {}, } - ) - const storeL = await createOrConnectStoreControllerCached(storeControllerCache, linkConfig) - const newManifest = await link(pkgPaths, path.join(linkCwdDir, 'node_modules'), { - ...linkConfig, - targetDependenciesField: linkOpts.targetDependenciesField, - storeController: storeL.ctrl, - storeDir: storeL.dir, - manifest, - } as LinkFunctionOptions) - if (!opts.cliOptions?.global) { - await writeProjectManifest(newManifest) + } else if (!manifest.pnpm.overrides) { + manifest.pnpm.overrides = {} + } + const { manifest: linkedManifest } = await tryReadProjectManifest(linkedDepDir, opts) + const linkedPkgName = linkedManifest?.name ?? path.basename(linkedDepDir) + const linkedPkgSpec = `link:${path.relative(dependentDir, linkedDepDir)}` + manifest.pnpm.overrides![linkedPkgName] = linkedPkgSpec + if (DEPENDENCIES_FIELDS.every((depField) => manifest[depField]?.[linkedPkgName] == null)) { + manifest.dependencies = manifest.dependencies ?? {} + manifest.dependencies[linkedPkgName] = linkedPkgSpec } - - await Promise.all( - Array.from(storeControllerCache.values()) - .map(async (storeControllerPromise) => { - const storeControllerHolder = await storeControllerPromise - await storeControllerHolder.ctrl.close() - }) - ) } diff --git a/pkg-manager/plugin-commands-installation/src/recursive.ts b/pkg-manager/plugin-commands-installation/src/recursive.ts index 0e1d29d4ed0..08f72c4597e 100755 --- a/pkg-manager/plugin-commands-installation/src/recursive.ts +++ b/pkg-manager/plugin-commands-installation/src/recursive.ts @@ -29,7 +29,6 @@ import { type InstallOptions, type MutatedProject, mutateModules, - type MutateModulesResult, type ProjectOptions, type UpdateMatchingFunction, type WorkspacePackages, @@ -103,7 +102,7 @@ export async function recursive ( allProjects: Project[], params: string[], opts: RecursiveOptions, - cmdFullName: 'install' | 'add' | 'remove' | 'unlink' | 'update' | 'import' + cmdFullName: 'install' | 'add' | 'remove' | 'update' | 'import' ): Promise { if (allProjects.length === 0) { // It might make sense to throw an exception in this case @@ -121,9 +120,7 @@ export async function recursive ( const store = await createOrConnectStoreController(opts) - const workspacePackages: WorkspacePackages = cmdFullName !== 'unlink' - ? arrayOfWorkspacePackagesToMap(allProjects) as WorkspacePackages - : new Map() + const workspacePackages: WorkspacePackages = arrayOfWorkspacePackagesToMap(allProjects) as WorkspacePackages const targetDependenciesField = getSaveType(opts) const rootManifestDir = (opts.lockfileDir ?? opts.dir) as ProjectRootDir const installOpts = Object.assign(opts, { @@ -313,9 +310,6 @@ export async function recursive ( let action!: any // eslint-disable-line @typescript-eslint/no-explicit-any switch (cmdFullName) { - case 'unlink': - action = (currentInput.length === 0 ? unlink : unlinkPkgs.bind(null, currentInput)) - break case 'remove': action = async (manifest: PackageManifest, opts: any) => { // eslint-disable-line @typescript-eslint/no-explicit-any const mutationResult = await mutateModules([ @@ -385,8 +379,7 @@ export async function recursive ( !opts.lockfileOnly && !opts.ignoreScripts && ( cmdFullName === 'add' || cmdFullName === 'install' || - cmdFullName === 'update' || - cmdFullName === 'unlink' + cmdFullName === 'update' ) ) { await rebuild.handler({ @@ -406,31 +399,6 @@ export async function recursive ( return true } -async function unlink (manifest: ProjectManifest, opts: any): Promise { // eslint-disable-line @typescript-eslint/no-explicit-any - return mutateModules( - [ - { - mutation: 'unlink', - rootDir: opts.dir, - }, - ], - opts - ) -} - -async function unlinkPkgs (dependencyNames: string[], manifest: ProjectManifest, opts: any): Promise { // eslint-disable-line @typescript-eslint/no-explicit-any - return mutateModules( - [ - { - dependencyNames, - mutation: 'unlinkSome', - rootDir: opts.dir, - }, - ], - opts - ) -} - function calculateRepositoryRoot ( workspaceDir: string, projectDirs: string[] diff --git a/pkg-manager/plugin-commands-installation/src/unlink.ts b/pkg-manager/plugin-commands-installation/src/unlink.ts index a4cc558dce1..ae3b5ee6718 100644 --- a/pkg-manager/plugin-commands-installation/src/unlink.ts +++ b/pkg-manager/plugin-commands-installation/src/unlink.ts @@ -1,14 +1,13 @@ -import { docsUrl, readProjectManifestOnly } from '@pnpm/cli-utils' +import path from 'path' +import { docsUrl } from '@pnpm/cli-utils' import { UNIVERSAL_OPTIONS } from '@pnpm/common-cli-options-help' -import { type Config, getOptionsFromRootManifest } from '@pnpm/config' -import { createOrConnectStoreController, type CreateStoreControllerOptions } from '@pnpm/store-connection-manager' -import { mutateModulesInSingleProject } from '@pnpm/core' -import { type ProjectRootDir } from '@pnpm/types' +import { writeProjectManifest } from '@pnpm/write-project-manifest' import renderHelp from 'render-help' -import { cliOptionsTypes, rcOptionsTypes } from './install' -import { recursive } from './recursive' +import * as install from './install' -export { cliOptionsTypes, rcOptionsTypes } +export const cliOptionsTypes = install.cliOptionsTypes + +export const rcOptionsTypes = install.rcOptionsTypes export const commandNames = ['unlink', 'dislink'] @@ -41,57 +40,25 @@ For options that may be used with `-r`, see "pnpm help recursive"', } export async function handler ( - opts: CreateStoreControllerOptions & - Pick & { - recursive?: boolean - }, + opts: install.InstallCommandOptions, params: string[] -): Promise { - if (opts.recursive && (opts.allProjects != null) && (opts.selectedProjectsGraph != null) && opts.workspaceDir) { - await recursive(opts.allProjects, params, { - ...opts, - allProjectsGraph: opts.allProjectsGraph!, - selectedProjectsGraph: opts.selectedProjectsGraph, - workspaceDir: opts.workspaceDir, - }, 'unlink') - return - } - const store = await createOrConnectStoreController(opts) - const unlinkOpts = Object.assign(opts, { - ...getOptionsFromRootManifest(opts.rootProjectManifestDir, opts.rootProjectManifest ?? {}), - globalBin: opts.bin, - storeController: store.ctrl, - storeDir: store.dir, - }) +): Promise { + if (!opts.rootProjectManifest?.pnpm?.overrides) return 'Nothing to unlink' if (!params || (params.length === 0)) { - await mutateModulesInSingleProject({ - dependencyNames: params, - manifest: await readProjectManifestOnly(opts.dir, opts), - mutation: 'unlinkSome', - rootDir: opts.dir as ProjectRootDir, - }, unlinkOpts) - return + for (const selector in opts.rootProjectManifest.pnpm.overrides) { + if (opts.rootProjectManifest.pnpm.overrides[selector].startsWith('link:')) { + delete opts.rootProjectManifest.pnpm.overrides[selector] + } + } + } else { + for (const selector in opts.rootProjectManifest.pnpm.overrides) { + if (opts.rootProjectManifest.pnpm.overrides[selector].startsWith('link:') && params.includes(selector)) { + delete opts.rootProjectManifest.pnpm.overrides[selector] + } + } } - await mutateModulesInSingleProject({ - manifest: await readProjectManifestOnly(opts.dir, opts), - mutation: 'unlink', - rootDir: opts.dir as ProjectRootDir, - }, unlinkOpts) + await writeProjectManifest(path.join(opts.rootProjectManifestDir, 'package.json'), opts.rootProjectManifest) + await install.handler(opts) + return undefined } diff --git a/pkg-manager/plugin-commands-installation/test/add.ts b/pkg-manager/plugin-commands-installation/test/add.ts index 30fa8d976c4..8a83793cc3c 100644 --- a/pkg-manager/plugin-commands-installation/test/add.ts +++ b/pkg-manager/plugin-commands-installation/test/add.ts @@ -37,7 +37,7 @@ const DEFAULT_OPTIONS = { storeDir: path.join(tmp, 'store'), userConfig: {}, workspaceConcurrency: 1, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, } test('installing with "workspace:" should work even if link-workspace-packages is off', async () => { diff --git a/pkg-manager/plugin-commands-installation/test/fetch.ts b/pkg-manager/plugin-commands-installation/test/fetch.ts index 61aaeff6dfb..388112aadc5 100644 --- a/pkg-manager/plugin-commands-installation/test/fetch.ts +++ b/pkg-manager/plugin-commands-installation/test/fetch.ts @@ -32,7 +32,7 @@ const DEFAULT_OPTIONS = { sort: true, userConfig: {}, workspaceConcurrency: 1, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, } test('fetch dependencies', async () => { diff --git a/pkg-manager/plugin-commands-installation/test/global.ts b/pkg-manager/plugin-commands-installation/test/global.ts index 9394bf193e8..fe285645229 100644 --- a/pkg-manager/plugin-commands-installation/test/global.ts +++ b/pkg-manager/plugin-commands-installation/test/global.ts @@ -36,7 +36,7 @@ const DEFAULT_OPTIONS = { storeDir: path.join(tmp, 'store'), userConfig: {}, workspaceConcurrency: 1, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, } test('globally installed package is linked with active version of Node.js', async () => { diff --git a/pkg-manager/plugin-commands-installation/test/import.ts b/pkg-manager/plugin-commands-installation/test/import.ts index f19579017dc..5ce154de824 100644 --- a/pkg-manager/plugin-commands-installation/test/import.ts +++ b/pkg-manager/plugin-commands-installation/test/import.ts @@ -40,7 +40,7 @@ const DEFAULT_OPTS = { userConfig: {}, useRunningStoreServer: false, useStoreServer: false, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, } test('import from package-lock.json', async () => { diff --git a/pkg-manager/plugin-commands-installation/test/importRecursive.ts b/pkg-manager/plugin-commands-installation/test/importRecursive.ts index 8ac517afbff..e0dee5af8bc 100644 --- a/pkg-manager/plugin-commands-installation/test/importRecursive.ts +++ b/pkg-manager/plugin-commands-installation/test/importRecursive.ts @@ -38,7 +38,7 @@ const DEFAULT_OPTS = { userConfig: {}, useRunningStoreServer: false, useStoreServer: false, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, } test('import from shared yarn.lock of monorepo', async () => { diff --git a/pkg-manager/plugin-commands-installation/test/install.ts b/pkg-manager/plugin-commands-installation/test/install.ts index d9019aaea46..7fd5481771f 100644 --- a/pkg-manager/plugin-commands-installation/test/install.ts +++ b/pkg-manager/plugin-commands-installation/test/install.ts @@ -1,6 +1,7 @@ import fs from 'fs' import delay from 'delay' import path from 'path' +import { STORE_VERSION } from '@pnpm/constants' import { add, install } from '@pnpm/plugin-commands-installation' import { prepare, prepareEmpty } from '@pnpm/prepare' import { sync as rimraf } from '@zkochan/rimraf' @@ -41,7 +42,7 @@ test('install with no store integrity validation', async () => { // We should have a short delay before modifying the file in the store. // Otherwise pnpm will not consider it to be modified. await delay(200) - const readmePath = path.join(DEFAULT_OPTS.storeDir, 'v3/files/9a/f6af85f55c111108eddf1d7ef7ef224b812e7c7bfabae41c79cf8bc9a910352536963809463e0af2799abacb975f22418a35a1d170055ef3fdc3b2a46ef1c5') + const readmePath = path.join(DEFAULT_OPTS.storeDir, STORE_VERSION, 'files/9a/f6af85f55c111108eddf1d7ef7ef224b812e7c7bfabae41c79cf8bc9a910352536963809463e0af2799abacb975f22418a35a1d170055ef3fdc3b2a46ef1c5') fs.writeFileSync(readmePath, 'modified', 'utf8') rimraf('node_modules') diff --git a/pkg-manager/plugin-commands-installation/test/link.ts b/pkg-manager/plugin-commands-installation/test/link.ts index 663af12a2f7..d2911848d93 100644 --- a/pkg-manager/plugin-commands-installation/test/link.ts +++ b/pkg-manager/plugin-commands-installation/test/link.ts @@ -1,9 +1,8 @@ import fs from 'fs' import path from 'path' -import { sync as readYamlFile } from 'read-yaml-file' import { install, link } from '@pnpm/plugin-commands-installation' import { prepare, preparePackages } from '@pnpm/prepare' -import { assertProject, isExecutable } from '@pnpm/assert-project' +import { isExecutable } from '@pnpm/assert-project' import { fixtures } from '@pnpm/test-fixtures' import { logger } from '@pnpm/logger' import { sync as loadJsonFile } from 'load-json-file' @@ -26,28 +25,27 @@ test('linking multiple packages', async () => { process.chdir('linked-foo') - console.log('linking linked-foo to global package') - const linkOpts = { + // linking linked-foo to global package + await link.handler({ ...DEFAULT_OPTS, bin: path.join(globalDir, 'bin'), dir: globalDir, - } - await link.handler({ - ...linkOpts, + globalPkgDir: globalDir, + rootProjectManifestDir: globalDir, }) process.chdir('..') process.chdir('project') await link.handler({ - ...linkOpts, + ...DEFAULT_OPTS, + dir: process.cwd(), + globalPkgDir: globalDir, + rootProjectManifestDir: process.cwd(), }, ['linked-foo', '../linked-bar']) project.has('linked-foo') project.has('linked-bar') - - const modules = readYamlFile('../linked-bar/node_modules/.modules.yaml') // eslint-disable-line @typescript-eslint/no-explicit-any - expect(modules.hoistPattern).toStrictEqual(['*']) // the linked package used its own configs during installation // eslint-disable-line @typescript-eslint/dot-notation }) test('link global bin', async function () { @@ -67,11 +65,10 @@ test('link global bin', async function () { await link.handler({ ...DEFAULT_OPTS, - cliOptions: { - global: true, - }, bin: globalBin, dir: globalDir, + globalPkgDir: globalDir, + rootProjectManifestDir: globalDir, }) process.env[PATH] = oldPath @@ -80,35 +77,6 @@ test('link global bin', async function () { }, path.join(globalBin, 'package-with-bin')) }) -test('link to global bin from the specified directory', async function () { - prepare() - process.chdir('..') - - const globalDir = path.resolve('global') - const globalBin = path.join(globalDir, 'bin') - const oldPath = process.env[PATH] - process.env[PATH] = `${globalBin}${path.delimiter}${oldPath ?? ''}` - fs.mkdirSync(globalBin, { recursive: true }) - - await writePkg('./dir/package-with-bin-in-dir', { name: 'package-with-bin-in-dir', version: '1.0.0', bin: 'bin.js' }) - fs.writeFileSync('./dir/package-with-bin-in-dir/bin.js', '#!/usr/bin/env node\nconsole.log(/hi/)\n', 'utf8') - - await link.handler({ - ...DEFAULT_OPTS, - cliOptions: { - global: true, - dir: path.resolve('./dir/package-with-bin-in-dir'), - }, - bin: globalBin, - dir: globalDir, - }) - process.env[PATH] = oldPath - - isExecutable((value) => { - expect(value).toBeTruthy() - }, path.join(globalBin, 'package-with-bin-in-dir')) -}) - test('link a global package to the specified directory', async function () { const project = prepare({ dependencies: { 'global-package-with-bin': '0.0.0' } }) process.chdir('..') @@ -127,11 +95,10 @@ test('link a global package to the specified directory', async function () { // link to global await link.handler({ ...DEFAULT_OPTS, - cliOptions: { - global: true, - }, bin: globalBin, dir: globalDir, + globalPkgDir: globalDir, + rootProjectManifestDir: globalDir, }) process.chdir('..') @@ -140,13 +107,12 @@ test('link a global package to the specified directory', async function () { // link from global await link.handler({ ...DEFAULT_OPTS, - cliOptions: { - global: true, - dir: projectDir, - }, - bin: globalBin, - dir: globalDir, + // bin: globalBin, + dir: projectDir, saveProd: true, // @pnpm/config sets this setting to true when global is true. This should probably be changed. + globalPkgDir: globalDir, + rootProjectManifest: { dependencies: { 'global-package-with-bin': '0.0.0' } }, + rootProjectManifestDir: projectDir, }, ['global-package-with-bin']) process.env[PATH] = oldPath @@ -170,19 +136,21 @@ test('relative link', async () => { await link.handler({ ...DEFAULT_OPTS, dir: process.cwd(), + globalPkgDir: '', + rootProjectManifest: { + dependencies: { + '@pnpm.e2e/hello-world-js-bin': '*', + }, + }, + rootProjectManifestDir: process.cwd(), }, [`../${linkedPkgName}`]) project.isExecutable('.bin/hello-world-js-bin') - // The linked package has been installed successfully as well with bins linked - // to node_modules/.bin - const linkedProject = assertProject(linkedPkgPath) - linkedProject.isExecutable('.bin/cowsay') - const wantedLockfile = project.readLockfile() expect(wantedLockfile.importers['.'].dependencies?.['@pnpm.e2e/hello-world-js-bin']).toStrictEqual({ - specifier: '*', // specifier of linked dependency added to ${WANTED_LOCKFILE} - version: 'link:../hello-world-js-bin', // link added to wanted lockfile + specifier: 'link:../hello-world-js-bin', + version: 'link:../hello-world-js-bin', }) const currentLockfile = project.readCurrentLockfile() @@ -203,18 +171,20 @@ test('absolute link', async () => { await link.handler({ ...DEFAULT_OPTS, dir: process.cwd(), + globalPkgDir: '', + rootProjectManifestDir: process.cwd(), + rootProjectManifest: { + dependencies: { + '@pnpm.e2e/hello-world-js-bin': '*', + }, + }, }, [linkedPkgPath]) project.isExecutable('.bin/hello-world-js-bin') - // The linked package has been installed successfully as well with bins linked - // to node_modules/.bin - const linkedProject = assertProject(linkedPkgPath) - linkedProject.isExecutable('.bin/cowsay') - const wantedLockfile = project.readLockfile() expect(wantedLockfile.importers['.'].dependencies?.['@pnpm.e2e/hello-world-js-bin']).toStrictEqual({ - specifier: '*', // specifier of linked dependency added to ${WANTED_LOCKFILE} + specifier: 'link:../hello-world-js-bin', // specifier of linked dependency added to ${WANTED_LOCKFILE} version: 'link:../hello-world-js-bin', // link added to wanted lockfile }) @@ -223,18 +193,19 @@ test('absolute link', async () => { }) test('link --production', async () => { - const projects = preparePackages([ - { - name: 'target', - version: '1.0.0', + const targetManifest = { + name: 'target', + version: '1.0.0', - dependencies: { - 'is-positive': '1.0.0', - }, - devDependencies: { - 'is-negative': '1.0.0', - }, + dependencies: { + 'is-positive': '1.0.0', }, + devDependencies: { + 'is-negative': '1.0.0', + }, + } + const projects = preparePackages([ + targetManifest, { name: 'source', version: '1.0.0', @@ -258,11 +229,11 @@ test('link --production', async () => { ...DEFAULT_OPTS, cliOptions: { production: true }, dir: process.cwd(), + globalPkgDir: '', + rootProjectManifestDir: process.cwd(), + rootProjectManifest: targetManifest, }, ['../source']) - projects['source'].has('is-positive') - projects['source'].hasNot('is-negative') - // --production should not have effect on the target projects['target'].has('is-positive') projects['target'].has('is-negative') @@ -275,6 +246,7 @@ test('link fails if nothing is linked', async () => { link.handler({ ...DEFAULT_OPTS, dir: '', + globalPkgDir: '', }, []) ).rejects.toThrow(/You must provide a parameter/) }) @@ -297,20 +269,21 @@ test('logger warns about peer dependencies when linking', async () => { process.chdir('linked-with-peer-deps') - const linkOpts = { + await link.handler({ ...DEFAULT_OPTS, bin: path.join(globalDir, 'bin'), dir: globalDir, - } - await link.handler({ - ...linkOpts, + globalPkgDir: globalDir, + rootProjectManifestDir: globalDir, }) process.chdir('..') process.chdir('project') await link.handler({ - ...linkOpts, + ...DEFAULT_OPTS, + dir: process.cwd(), + globalPkgDir: globalDir, }, ['linked-with-peer-deps']) expect(warnMock).toHaveBeenCalledWith(expect.objectContaining({ @@ -336,20 +309,22 @@ test('logger should not warn about peer dependencies when it is an empty object' process.chdir('linked-with-empty-peer-deps') - const linkOpts = { + await link.handler({ ...DEFAULT_OPTS, + globalPkgDir: '', bin: path.join(globalDir, 'bin'), dir: globalDir, - } - await link.handler({ - ...linkOpts, + rootProjectManifestDir: globalDir, }) process.chdir('..') process.chdir('project') await link.handler({ - ...linkOpts, + ...DEFAULT_OPTS, + globalPkgDir: globalDir, + dir: process.cwd(), + rootProjectManifestDir: process.cwd(), }, ['linked-with-empty-peer-deps']) expect(warnMock).not.toHaveBeenCalledWith(expect.objectContaining({ @@ -370,6 +345,7 @@ test('link: fail when global bin directory is not found', async () => { ...DEFAULT_OPTS, bin: undefined as any, // eslint-disable-line @typescript-eslint/no-explicit-any dir: globalDir, + globalPkgDir: globalDir, cliOptions: { global: true, }, diff --git a/pkg-manager/plugin-commands-installation/test/linkRecursive.ts b/pkg-manager/plugin-commands-installation/test/linkRecursive.ts deleted file mode 100644 index 100f181f73e..00000000000 --- a/pkg-manager/plugin-commands-installation/test/linkRecursive.ts +++ /dev/null @@ -1,137 +0,0 @@ -import fs from 'fs' -import path from 'path' -import { LOCKFILE_VERSION } from '@pnpm/constants' -import { filterPackagesFromDir } from '@pnpm/workspace.filter-packages-from-dir' -import { install, unlink } from '@pnpm/plugin-commands-installation' -import { preparePackages } from '@pnpm/prepare' -import { DEFAULT_OPTS } from './utils' - -test('recursive linking/unlinking', async () => { - const projects = preparePackages([ - { - name: 'project-1', - version: '1.0.0', - - devDependencies: { - 'is-positive': '1.0.0', - }, - }, - { - name: 'is-positive', - version: '1.0.0', - - dependencies: { - 'is-negative': '1.0.0', - }, - }, - ]) - - const { allProjects, allProjectsGraph, selectedProjectsGraph } = await filterPackagesFromDir(process.cwd(), []) - await install.handler({ - ...DEFAULT_OPTS, - allProjects, - allProjectsGraph, - dir: process.cwd(), - recursive: true, - saveWorkspaceProtocol: false, - selectedProjectsGraph, - workspaceDir: process.cwd(), - }) - - expect(projects['is-positive'].requireModule('is-negative')).toBeTruthy() - expect(projects['project-1'].requireModule('is-positive/package.json').author).toBeFalsy() - - { - const project1Lockfile = projects['project-1'].readLockfile() - expect(project1Lockfile.importers['.'].devDependencies?.['is-positive'].version).toBe('link:../is-positive') - } - - await unlink.handler({ - ...DEFAULT_OPTS, - allProjects, - allProjectsGraph, - dir: process.cwd(), - recursive: true, - saveWorkspaceProtocol: false, - selectedProjectsGraph, - workspaceDir: process.cwd(), - }, []) - - process.chdir('project-1') - expect(fs.existsSync(path.resolve('node_modules', 'is-positive', 'index.js'))).toBeTruthy() - - { - const project1Lockfile = projects['project-1'].readLockfile() - expect(project1Lockfile.lockfileVersion).toBe(LOCKFILE_VERSION) - expect(project1Lockfile.importers['.'].devDependencies?.['is-positive'].version).toBe('1.0.0') - expect(project1Lockfile.packages['is-positive@1.0.0']).toBeTruthy() - } - - const isPositiveLockfile = projects['is-positive'].readLockfile() - expect(isPositiveLockfile.lockfileVersion).toBe(LOCKFILE_VERSION) -}) - -test('recursive unlink specific package', async () => { - const projects = preparePackages([ - { - name: 'project-1', - version: '1.0.0', - - devDependencies: { - 'is-positive': '1.0.0', - }, - }, - { - name: 'is-positive', - version: '1.0.0', - - dependencies: { - 'is-negative': '1.0.0', - }, - }, - ]) - - const { allProjects, allProjectsGraph, selectedProjectsGraph } = await filterPackagesFromDir(process.cwd(), []) - await install.handler({ - ...DEFAULT_OPTS, - allProjects, - allProjectsGraph, - dir: process.cwd(), - recursive: true, - saveWorkspaceProtocol: false, - selectedProjectsGraph, - workspaceDir: process.cwd(), - }) - - expect(projects['is-positive'].requireModule('is-negative')).toBeTruthy() - expect(projects['project-1'].requireModule('is-positive/package.json').author).toBeFalsy() - - { - const project1Lockfile = projects['project-1'].readLockfile() - expect(project1Lockfile.importers['.'].devDependencies?.['is-positive'].version).toBe('link:../is-positive') - } - - await unlink.handler({ - ...DEFAULT_OPTS, - allProjects, - allProjectsGraph, - dir: process.cwd(), - recursive: true, - saveWorkspaceProtocol: false, - selectedProjectsGraph, - workspaceDir: process.cwd(), - }, ['is-positive']) - - process.chdir('project-1') - expect(fs.existsSync(path.resolve('node_modules', 'is-positive', 'index.js'))).toBeTruthy() - - { - const project1Lockfile = projects['project-1'].readLockfile() - expect(project1Lockfile.lockfileVersion).toBe(LOCKFILE_VERSION) - expect(project1Lockfile.importers['.'].devDependencies?.['is-positive'].version).toBe('1.0.0') - expect(project1Lockfile.packages['is-positive@1.0.0']).toBeTruthy() - } - - const isPositiveLockfile = projects['is-positive'].readLockfile() - expect(isPositiveLockfile.lockfileVersion).toBe(LOCKFILE_VERSION) -}) diff --git a/pkg-manager/plugin-commands-installation/test/peerDependencies.ts b/pkg-manager/plugin-commands-installation/test/peerDependencies.ts index 0b3dad7c48e..aa4addfbdc6 100644 --- a/pkg-manager/plugin-commands-installation/test/peerDependencies.ts +++ b/pkg-manager/plugin-commands-installation/test/peerDependencies.ts @@ -35,7 +35,7 @@ const DEFAULT_OPTIONS = { storeDir: path.join(TMP, 'store'), userConfig: {}, workspaceConcurrency: 1, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, } test('root dependency that has a peer is correctly updated after its version changes', async () => { diff --git a/pkg-manager/plugin-commands-installation/test/prune.ts b/pkg-manager/plugin-commands-installation/test/prune.ts index 0b32075710b..04aa9616307 100644 --- a/pkg-manager/plugin-commands-installation/test/prune.ts +++ b/pkg-manager/plugin-commands-installation/test/prune.ts @@ -1,9 +1,10 @@ import path from 'path' -import { add, install, link, prune } from '@pnpm/plugin-commands-installation' +import { add, install, prune } from '@pnpm/plugin-commands-installation' import { prepare } from '@pnpm/prepare' import { REGISTRY_MOCK_PORT } from '@pnpm/registry-mock' import { fixtures } from '@pnpm/test-fixtures' import { createTestIpcServer } from '@pnpm/test-ipc-server' +import symlinkDir from 'symlink-dir' import fs from 'fs' const REGISTRY_URL = `http://localhost:${REGISTRY_MOCK_PORT}` @@ -36,7 +37,7 @@ const DEFAULT_OPTIONS = { sort: true, userConfig: {}, workspaceConcurrency: 1, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, } test('prune removes external link that is not in package.json', async () => { @@ -44,12 +45,7 @@ test('prune removes external link that is not in package.json', async () => { const storeDir = path.resolve('store') f.copy('local-pkg', 'local') - await link.handler({ - ...DEFAULT_OPTIONS, - cacheDir: path.resolve('cache'), - dir: process.cwd(), - storeDir, - }, ['./local']) + symlinkDir.sync(path.resolve('local'), path.join('node_modules/local-pkg')) project.has('local-pkg') diff --git a/pkg-manager/plugin-commands-installation/test/update/interactive.ts b/pkg-manager/plugin-commands-installation/test/update/interactive.ts index 202e0fe164a..1bfa558ba26 100644 --- a/pkg-manager/plugin-commands-installation/test/update/interactive.ts +++ b/pkg-manager/plugin-commands-installation/test/update/interactive.ts @@ -41,7 +41,7 @@ const DEFAULT_OPTIONS = { sort: true, userConfig: {}, workspaceConcurrency: 1, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, } test('interactively update', async () => { diff --git a/pkg-manager/plugin-commands-installation/test/update/issue-7415.ts b/pkg-manager/plugin-commands-installation/test/update/issue-7415.ts index 26ac781f209..2a3cbab9f0a 100644 --- a/pkg-manager/plugin-commands-installation/test/update/issue-7415.ts +++ b/pkg-manager/plugin-commands-installation/test/update/issue-7415.ts @@ -38,7 +38,7 @@ const DEFAULT_OPTIONS = { sort: true, userConfig: {}, workspaceConcurrency: 1, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, } test('interactive recursive should not error on git specifier override', async () => { diff --git a/pkg-manager/plugin-commands-installation/test/utils/index.ts b/pkg-manager/plugin-commands-installation/test/utils/index.ts index 021d4d8c268..c72a61d863e 100644 --- a/pkg-manager/plugin-commands-installation/test/utils/index.ts +++ b/pkg-manager/plugin-commands-installation/test/utils/index.ts @@ -49,5 +49,5 @@ export const DEFAULT_OPTS = { useRunningStoreServer: false, useStoreServer: false, workspaceConcurrency: 4, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, } diff --git a/pkg-manager/plugin-commands-installation/tsconfig.json b/pkg-manager/plugin-commands-installation/tsconfig.json index 1379df162ab..62e99080499 100644 --- a/pkg-manager/plugin-commands-installation/tsconfig.json +++ b/pkg-manager/plugin-commands-installation/tsconfig.json @@ -75,6 +75,9 @@ { "path": "../../pkg-manifest/read-project-manifest" }, + { + "path": "../../pkg-manifest/write-project-manifest" + }, { "path": "../../resolving/resolver-base" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 915d69f75b2..5dec1fd7c40 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -181,11 +181,11 @@ catalogs: specifier: npm:@types/table@6.0.0 version: 6.0.0 '@yarnpkg/core': - specifier: 4.0.3 - version: 4.0.3 + specifier: 4.0.5 + version: 4.0.5 '@yarnpkg/extensions': - specifier: 2.0.1 - version: 2.0.1 + specifier: 2.0.3 + version: 2.0.3 '@yarnpkg/lockfile': specifier: ^1.1.0 version: 1.1.0 @@ -528,9 +528,6 @@ catalogs: resolve-link-target: specifier: ^2.0.0 version: 2.0.0 - rfc4648: - specifier: ^1.5.3 - version: 1.5.3 rimraf: specifier: ^3.0.2 version: 3.0.2 @@ -879,6 +876,9 @@ importers: '@pnpm/assert-store': specifier: workspace:* version: 'link:' + '@pnpm/constants': + specifier: workspace:* + version: link:../../packages/constants __utils__/eslint-config: dependencies: @@ -1641,11 +1641,27 @@ importers: specifier: 'catalog:' version: 1.3.31 + crypto/hash: + dependencies: + '@pnpm/crypto.polyfill': + specifier: workspace:* + version: link:../polyfill + devDependencies: + '@pnpm/crypto.hash': + specifier: workspace:* + version: 'link:' + '@pnpm/prepare': + specifier: workspace:* + version: link:../../__utils__/prepare + crypto/object-hasher: dependencies: object-hash: specifier: 'catalog:' version: 3.0.0 + ramda: + specifier: 'catalog:' + version: '@pnpm/ramda@0.28.1' devDependencies: '@pnpm/crypto.object-hasher': specifier: workspace:* @@ -1653,6 +1669,9 @@ importers: '@types/object-hash': specifier: 'catalog:' version: 3.0.6 + '@types/ramda': + specifier: 'catalog:' + version: 0.29.12 crypto/polyfill: devDependencies: @@ -2265,9 +2284,9 @@ importers: '@pnpm/core-loggers': specifier: workspace:* version: link:../../packages/core-loggers - '@pnpm/crypto.base32-hash': + '@pnpm/crypto.hash': specifier: workspace:* - version: link:../../packages/crypto.base32-hash + version: link:../../crypto/hash '@pnpm/env.path': specifier: workspace:* version: link:../../env/path @@ -2779,9 +2798,9 @@ importers: '@pnpm/core-loggers': specifier: workspace:* version: link:../../packages/core-loggers - '@pnpm/crypto.base32-hash': + '@pnpm/crypto.hash': specifier: workspace:* - version: link:../../packages/crypto.base32-hash + version: link:../../crypto/hash '@pnpm/error': specifier: workspace:* version: link:../../packages/error @@ -2833,7 +2852,7 @@ importers: version: link:../../packages/types '@yarnpkg/extensions': specifier: 'catalog:' - version: 2.0.1(@yarnpkg/core@4.0.3(typanion@3.14.0)) + version: 2.0.3(@yarnpkg/core@4.0.5(typanion@3.14.0)) normalize-path: specifier: 'catalog:' version: 3.0.0 @@ -2858,7 +2877,7 @@ importers: version: 7.5.3 '@yarnpkg/core': specifier: 'catalog:' - version: 4.0.3(typanion@3.14.0) + version: 4.0.5(typanion@3.14.0) hooks/types: dependencies: @@ -3283,9 +3302,9 @@ importers: lockfile/settings-checker: dependencies: - '@pnpm/crypto.base32-hash': + '@pnpm/crypto.hash': specifier: workspace:* - version: link:../../packages/crypto.base32-hash + version: link:../../crypto/hash '@pnpm/lockfile.types': specifier: workspace:* version: link:../types @@ -3482,6 +3501,9 @@ importers: specifier: 'catalog:' version: 2.2.6 devDependencies: + '@pnpm/constants': + specifier: workspace:* + version: link:../../packages/constants '@pnpm/mount-modules': specifier: workspace:* version: 'link:' @@ -3602,27 +3624,11 @@ importers: specifier: workspace:* version: link:../logger - packages/crypto.base32-hash: - dependencies: - '@pnpm/crypto.polyfill': - specifier: workspace:* - version: link:../../crypto/polyfill - rfc4648: - specifier: 'catalog:' - version: 1.5.3 - devDependencies: - '@pnpm/crypto.base32-hash': - specifier: workspace:* - version: 'link:' - '@pnpm/prepare': - specifier: workspace:* - version: link:../../__utils__/prepare - packages/dependency-path: dependencies: - '@pnpm/crypto.base32-hash': + '@pnpm/crypto.hash': specifier: workspace:* - version: link:../crypto.base32-hash + version: link:../../crypto/hash '@pnpm/types': specifier: workspace:* version: link:../types @@ -4103,12 +4109,12 @@ importers: '@pnpm/core-loggers': specifier: workspace:* version: link:../../packages/core-loggers - '@pnpm/crypto.base32-hash': + '@pnpm/crypto.hash': specifier: workspace:* - version: link:../../packages/crypto.base32-hash - '@pnpm/crypto.polyfill': + version: link:../../crypto/hash + '@pnpm/crypto.object-hasher': specifier: workspace:* - version: link:../../crypto/polyfill + version: link:../../crypto/object-hasher '@pnpm/dependency-path': specifier: workspace:* version: link:../../packages/dependency-path @@ -4229,6 +4235,12 @@ importers: '@zkochan/rimraf': specifier: 'catalog:' version: 3.0.2 + ci-info: + specifier: 'catalog:' + version: 3.9.0 + enquirer: + specifier: 'catalog:' + version: 2.4.1 is-inner-link: specifier: 'catalog:' version: 4.0.0 @@ -4259,9 +4271,6 @@ importers: semver: specifier: 'catalog:' version: 7.6.2 - sort-keys: - specifier: 'catalog:' - version: 4.2.0 devDependencies: '@pnpm/assert-project': specifier: workspace:* @@ -4275,9 +4284,6 @@ importers: '@pnpm/core': specifier: workspace:* version: 'link:' - '@pnpm/crypto.object-hasher': - specifier: workspace:* - version: link:../../crypto/object-hasher '@pnpm/git-utils': specifier: workspace:* version: link:../../packages/git-utils @@ -4331,10 +4337,7 @@ importers: version: 10.0.20 '@yarnpkg/core': specifier: 'catalog:' - version: 4.0.3(typanion@3.14.0) - ci-info: - specifier: 'catalog:' - version: 3.9.0 + version: 4.0.5(typanion@3.14.0) deep-require-cwd: specifier: 'catalog:' version: 1.0.0 @@ -4411,9 +4414,6 @@ importers: '@pnpm/core-loggers': specifier: workspace:* version: link:../../packages/core-loggers - '@pnpm/error': - specifier: workspace:* - version: link:../../packages/error '@pnpm/lockfile.fs': specifier: workspace:* version: link:../../lockfile/fs @@ -4429,15 +4429,9 @@ importers: '@pnpm/types': specifier: workspace:* version: link:../../packages/types - '@zkochan/rimraf': - specifier: 'catalog:' - version: 3.0.2 ci-info: specifier: 'catalog:' version: 3.9.0 - enquirer: - specifier: 'catalog:' - version: 2.4.1 path-absolute: specifier: 'catalog:' version: 1.0.1 @@ -5070,9 +5064,12 @@ importers: '@pnpm/workspace.pkgs-graph': specifier: workspace:* version: link:../../workspace/pkgs-graph + '@pnpm/write-project-manifest': + specifier: workspace:* + version: link:../../pkg-manifest/write-project-manifest '@yarnpkg/core': specifier: 'catalog:' - version: 4.0.3(typanion@3.14.0) + version: 4.0.5(typanion@3.14.0) '@yarnpkg/lockfile': specifier: 'catalog:' version: 1.1.0 @@ -5109,9 +5106,6 @@ importers: p-limit: specifier: 'catalog:' version: 3.1.0 - path-absolute: - specifier: 'catalog:' - version: 1.0.1 ramda: specifier: 'catalog:' version: '@pnpm/ramda@0.28.1' @@ -5364,7 +5358,7 @@ importers: version: link:../../workspace/spec-parser '@yarnpkg/core': specifier: 'catalog:' - version: 4.0.3(typanion@3.14.0) + version: 4.0.5(typanion@3.14.0) filenamify: specifier: 'catalog:' version: 4.3.0 @@ -5643,9 +5637,9 @@ importers: '@pnpm/core-loggers': specifier: workspace:* version: link:../packages/core-loggers - '@pnpm/crypto.base32-hash': + '@pnpm/crypto.hash': specifier: workspace:* - version: link:../packages/crypto.base32-hash + version: link:../crypto/hash '@pnpm/default-reporter': specifier: workspace:* version: link:../cli/default-reporter @@ -6313,9 +6307,9 @@ importers: '@pnpm/core-loggers': specifier: workspace:* version: link:../../packages/core-loggers - '@pnpm/crypto.polyfill': + '@pnpm/crypto.hash': specifier: workspace:* - version: link:../../crypto/polyfill + version: link:../../crypto/hash '@pnpm/error': specifier: workspace:* version: link:../../packages/error @@ -7222,6 +7216,9 @@ importers: '@pnpm/assert-store': specifier: workspace:* version: link:../../__utils__/assert-store + '@pnpm/constants': + specifier: workspace:* + version: link:../../packages/constants '@pnpm/lockfile.fs': specifier: workspace:* version: link:../../lockfile/fs @@ -7439,6 +7436,9 @@ importers: store/store-path: dependencies: + '@pnpm/constants': + specifier: workspace:* + version: link:../../packages/constants '@pnpm/error': specifier: workspace:* version: link:../../packages/error @@ -9387,19 +9387,19 @@ packages: resolution: {integrity: sha512-KZVpiDKRi2gtrVtKwhz/ZUKBOicVNggxaYQzPBjULuOLJ/UypTmAz5a2g+utLMn+WogbLE3vLfmC+TWp8v3+aQ==} hasBin: true - '@yarnpkg/core@4.0.3': - resolution: {integrity: sha512-O+WGCjB9aIBxdRMBxXdsIy08MW4RbxfCS2AfywWb8DPS9H0LICahUJgNAaE0fwCsW7/gNzQbLYlh9DQQwzONrA==} + '@yarnpkg/core@4.0.5': + resolution: {integrity: sha512-6ib9l8P30GxHvxZo3170hr5PCy2qQnI4N4lXwNJxJnV0i46UlgLA4hlGp/kFVttteATGeckfduIDyWZgjn9sPw==} engines: {node: '>=18.12.0'} '@yarnpkg/core@4.1.2': resolution: {integrity: sha512-2jcHRIJwjOOdl7mFysdz5FLxl4JWWJJEwesaKMRYmVwcqXolItX/gXdL3AjTD0umlBySccK9TulRD2/YL2Y0YQ==} engines: {node: '>=18.12.0'} - '@yarnpkg/extensions@2.0.1': - resolution: {integrity: sha512-keiaqrAA+bISwaerRFkV3H2/TXB4j0cYnKsW0YVOn+daEFVrKYdW+mOdqsyljCyS24ZBY131/+bW0I6n01yH8g==} + '@yarnpkg/extensions@2.0.3': + resolution: {integrity: sha512-aBzcipnRR6jJy4ta8KHBvq7EG/1tGa9RCJ3n7RG38l+cCuEf41O5mcMKssoL0FzaBqnEn6q5qWZ4xa24c0d1RQ==} engines: {node: '>=18.12.0'} peerDependencies: - '@yarnpkg/core': ^4.0.3 + '@yarnpkg/core': ^4.0.5 '@yarnpkg/fslib@3.1.0': resolution: {integrity: sha512-wsj7/sUVSdXOIX/qwaON/Ky5GsP5gs9ry9DKwgLbWT7k3qw4/EcHAtfTtPhBYu33UibzBFI+fgB4wBRVH2XVaw==} @@ -13915,6 +13915,7 @@ packages: uid-number@0.0.6: resolution: {integrity: sha512-c461FXIljswCuscZn67xq9PpszkPT6RjheWFQTgCyabJrTUozElanb0YEqv2UGgk247YpcJkFBuSGNvBlpXM9w==} + deprecated: This package is no longer supported. umask@1.1.0: resolution: {integrity: sha512-lE/rxOhmiScJu9L6RTNVgB/zZbF+vGC0/p6D3xnkAePI2o0sMyFG966iR5Ki50OI/0mNi2yaRnxfLsPmEZF/JA==} @@ -16352,7 +16353,7 @@ snapshots: - encoding - supports-color - '@yarnpkg/core@4.0.3(typanion@3.14.0)': + '@yarnpkg/core@4.0.5(typanion@3.14.0)': dependencies: '@arcanis/slice-ansi': 1.1.1 '@types/semver': 7.5.8 @@ -16414,9 +16415,9 @@ snapshots: transitivePeerDependencies: - typanion - '@yarnpkg/extensions@2.0.1(@yarnpkg/core@4.0.3(typanion@3.14.0))': + '@yarnpkg/extensions@2.0.3(@yarnpkg/core@4.0.5(typanion@3.14.0))': dependencies: - '@yarnpkg/core': 4.0.3(typanion@3.14.0) + '@yarnpkg/core': 4.0.5(typanion@3.14.0) '@yarnpkg/fslib@3.1.0': dependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 555b820ff8a..df933ca4390 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -97,8 +97,8 @@ catalog: "@types/write-file-atomic": ^4.0.3 "@types/yarnpkg__lockfile": ^1.1.9 "@types/zkochan__table": npm:@types/table@6.0.0 - "@yarnpkg/core": 4.0.3 - "@yarnpkg/extensions": 2.0.1 + "@yarnpkg/core": 4.0.5 + "@yarnpkg/extensions": 2.0.3 "@yarnpkg/lockfile": ^1.1.0 "@yarnpkg/nm": 4.0.2 "@yarnpkg/parsers": 3.0.0 diff --git a/pnpm/package.json b/pnpm/package.json index 5fcd3fbbad0..1649ebb4032 100644 --- a/pnpm/package.json +++ b/pnpm/package.json @@ -33,7 +33,7 @@ "@pnpm/config": "workspace:*", "@pnpm/constants": "workspace:*", "@pnpm/core-loggers": "workspace:*", - "@pnpm/crypto.base32-hash": "workspace:*", + "@pnpm/crypto.hash": "workspace:*", "@pnpm/default-reporter": "workspace:*", "@pnpm/dependency-path": "workspace:*", "@pnpm/env.path": "workspace:*", @@ -174,7 +174,7 @@ "compile": "tsc --build && pnpm run lint --fix && rimraf dist bin/nodes && pnpm run bundle && shx cp -r node-gyp-bin dist/node-gyp-bin && shx cp -r node_modules/@pnpm/tabtab/lib/templates dist/templates && shx cp -r node_modules/ps-list/vendor dist/vendor && shx cp pnpmrc dist/pnpmrc" }, "publishConfig": { - "tag": "next-9", + "tag": "next-10", "executableFiles": [ "./dist/node-gyp-bin/node-gyp", "./dist/node-gyp-bin/node-gyp.cmd", diff --git a/pnpm/src/cmd/index.ts b/pnpm/src/cmd/index.ts index 653017bf684..9cc5f4cda5b 100644 --- a/pnpm/src/cmd/index.ts +++ b/pnpm/src/cmd/index.ts @@ -21,7 +21,6 @@ import { exec, restart, run, - test, } from '@pnpm/plugin-commands-script-runners' import { server } from '@pnpm/plugin-commands-server' import { setup } from '@pnpm/plugin-commands-setup' @@ -150,7 +149,6 @@ const commands: CommandDefinition[] = [ catFile, catIndex, findHash, - test, unlink, update, why, diff --git a/pnpm/src/cmd/installTest.ts b/pnpm/src/cmd/installTest.ts index fc60fe9d788..49c4670ef8d 100644 --- a/pnpm/src/cmd/installTest.ts +++ b/pnpm/src/cmd/installTest.ts @@ -1,6 +1,6 @@ import { docsUrl } from '@pnpm/cli-utils' import { install } from '@pnpm/plugin-commands-installation' -import { test } from '@pnpm/plugin-commands-script-runners' +import { run } from '@pnpm/plugin-commands-script-runners' import renderHelp from 'render-help' import { type PnpmOptions } from '../types' @@ -21,5 +21,5 @@ export function help (): string { export async function handler (opts: PnpmOptions, params: string[]): Promise { await install.handler(opts) - await test.handler(opts as any, params) // eslint-disable-line + await run.handler(opts as any, ['test', ...params]) // eslint-disable-line } diff --git a/pnpm/src/parseCliArgs.ts b/pnpm/src/parseCliArgs.ts index bcb58d4e69d..650f03c4fbf 100644 --- a/pnpm/src/parseCliArgs.ts +++ b/pnpm/src/parseCliArgs.ts @@ -17,7 +17,7 @@ const RENAMED_OPTIONS = { export async function parseCliArgs (inputArgv: string[]): Promise { return parseCliArgsLib({ fallbackCommand: 'run', - escapeArgs: ['create', 'dlx', 'exec'], + escapeArgs: ['create', 'dlx', 'exec', 'test'], getCommandLongName: getCommandFullName, getTypesByCommandName: getCliOptionsTypes, renamedOptions: RENAMED_OPTIONS, diff --git a/pnpm/test/cli.ts b/pnpm/test/cli.ts index 9bbc9d47d30..f6e475c9a11 100644 --- a/pnpm/test/cli.ts +++ b/pnpm/test/cli.ts @@ -166,7 +166,7 @@ test.each([ { message: 'npm_command env available on special lifecycle hooks', script: 'prepare', command: 'install' }, { message: 'npm_command env available on special lifecycle hooks (alias)', script: 'prepare', command: 'i', expected: 'install' }, { message: 'npm_command env available on pre lifecycle hooks', script: 'prepack', command: 'pack' }, - { message: 'npm_command env available on special commands', script: 'test', command: 'test' }, + { message: 'npm_command env available on special commands', script: 'test', command: 'test', expected: 'run-script' }, { message: 'npm_command env available on scripts', script: 'dev', command: 'dev', expected: 'run-script' }, ])('$message', async ({ script, command, expected }) => { prepare({ diff --git a/pnpm/test/filterProd.test.ts b/pnpm/test/filterProd.test.ts index 3ac4363cdbf..d04105c3223 100644 --- a/pnpm/test/filterProd.test.ts +++ b/pnpm/test/filterProd.test.ts @@ -52,7 +52,7 @@ test.each([ writeYamlFile('pnpm-workspace.yaml', { packages: ['**', '!store/**'] }) await execPnpm(['install']) - await execPnpm(['recursive', 'test', filter, '...project-3']) + await execPnpm([filter, '...project-3', 'test']) expect(server.getLines().sort()).toEqual(expected) }) diff --git a/pnpm/test/hooks.ts b/pnpm/test/hooks.ts index bffe8caed57..af0c05386ef 100644 --- a/pnpm/test/hooks.ts +++ b/pnpm/test/hooks.ts @@ -1,6 +1,6 @@ import fs from 'fs' import path from 'path' -import { createBase32Hash } from '@pnpm/crypto.base32-hash' +import { createHash } from '@pnpm/crypto.hash' import { type PackageManifest } from '@pnpm/types' import { prepare, preparePackages } from '@pnpm/prepare' import { REGISTRY_MOCK_PORT } from '@pnpm/registry-mock' @@ -253,7 +253,7 @@ test('adding or changing pnpmfile should change pnpmfileChecksum and module stru await execPnpm(['install']) const lockfile1 = project.readLockfile() - expect(lockfile1.pnpmfileChecksum).toBe(createBase32Hash(pnpmfile1)) + expect(lockfile1.pnpmfileChecksum).toBe(createHash(pnpmfile1)) expect(lockfile1.packages).toHaveProperty(['@pnpm.e2e/pkg-with-good-optional@1.0.0']) expect(lockfile1.packages).not.toHaveProperty(['is-positive@1.0.0']) // this should be removed due to being optional dependency @@ -274,7 +274,7 @@ test('adding or changing pnpmfile should change pnpmfileChecksum and module stru await execPnpm(['install']) const lockfile2 = project.readLockfile() - expect(lockfile2.pnpmfileChecksum).toBe(createBase32Hash(pnpmfile2)) + expect(lockfile2.pnpmfileChecksum).toBe(createHash(pnpmfile2)) expect(lockfile2.snapshots).toMatchObject({ '@pnpm.e2e/foo@100.0.0': expect.any(Object), '@pnpm.e2e/bar@100.0.0': expect.any(Object), diff --git a/pnpm/test/install/misc.ts b/pnpm/test/install/misc.ts index 28f726ecf4b..e180e3aecaa 100644 --- a/pnpm/test/install/misc.ts +++ b/pnpm/test/install/misc.ts @@ -1,6 +1,6 @@ import fs from 'fs' import path from 'path' -import { WANTED_LOCKFILE } from '@pnpm/constants' +import { STORE_VERSION, WANTED_LOCKFILE } from '@pnpm/constants' import { type Lockfile } from '@pnpm/lockfile.types' import { prepare, prepareEmpty, preparePackages } from '@pnpm/prepare' import { readPackageJsonFromDir } from '@pnpm/read-package-json' @@ -270,9 +270,9 @@ test('install should not fail if the used pnpm version does not satisfy the pnpm packageManager: 'pnpm@0.0.0', }) - expect(execPnpmSync(['install']).status).toBe(0) + expect(execPnpmSync(['install', '--config.manage-package-manager-versions=false']).status).toBe(0) - const { status, stderr } = execPnpmSync(['install', '--config.package-manager-strict-version=true']) + const { status, stderr } = execPnpmSync(['install', '--config.manage-package-manager-versions=false', '--config.package-manager-strict-version=true']) expect(status).toBe(1) expect(stderr.toString()).toContain('This project is configured to use v0.0.0 of pnpm. Your current pnpm is') @@ -286,7 +286,7 @@ test('install should fail if the project requires a different package manager', packageManager: 'yarn@4.0.0', }) - const { status, stderr } = execPnpmSync(['install']) + const { status, stderr } = execPnpmSync(['install', '--config.manage-package-manager-versions=false']) expect(status).toBe(1) expect(stderr.toString()).toContain('This project is configured to use yarn') @@ -518,8 +518,7 @@ test('installation fails when the stored package name and version do not match t await execPnpm(['add', '@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0', ...settings]) - const cafsDir = path.join(storeDir, 'v3/files') - const cacheIntegrityPath = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/dep-of-pkg-with-1-dep', '100.1.0')) + const cacheIntegrityPath = getIndexFilePathInCafs(path.join(storeDir, STORE_VERSION), getIntegrity('@pnpm.e2e/dep-of-pkg-with-1-dep', '100.1.0'), '@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0') const cacheIntegrity = loadJsonFile.sync(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any cacheIntegrity.name = 'foo' writeJsonFile.sync(cacheIntegrityPath, { diff --git a/pnpm/test/install/selfUpdate.ts b/pnpm/test/install/selfUpdate.ts index 0e09bb6a641..38017e9bc6a 100644 --- a/pnpm/test/install/selfUpdate.ts +++ b/pnpm/test/install/selfUpdate.ts @@ -1,6 +1,7 @@ import fs from 'fs' import path from 'path' import PATH_NAME from 'path-name' +import { STORE_VERSION } from '@pnpm/constants' import { prepare } from '@pnpm/prepare' import isWindows from 'is-windows' import { @@ -16,7 +17,7 @@ skipOnWindows('self-update stops the store server', async () => { spawnPnpm(['server', 'start']) - const serverJsonPath = path.resolve('../store/v3/server/server.json') + const serverJsonPath = path.resolve(`../store/${STORE_VERSION}/server/server.json`) const serverJson = await retryLoadJsonFile<{ connectionOptions: object }>(serverJsonPath) expect(serverJson).toBeTruthy() expect(serverJson.connectionOptions).toBeTruthy() diff --git a/pnpm/test/monorepo/index.ts b/pnpm/test/monorepo/index.ts index 087ab10374e..f206cf1def2 100644 --- a/pnpm/test/monorepo/index.ts +++ b/pnpm/test/monorepo/index.ts @@ -76,47 +76,6 @@ test('incorrect workspace manifest', async () => { expect(status).toBe(1) }) -test('linking a package inside a monorepo', async () => { - const projects = preparePackages([ - { - name: 'project-1', - version: '1.0.0', - }, - { - name: 'project-2', - version: '2.0.0', - }, - { - name: 'project-3', - version: '3.0.0', - }, - { - name: 'project-4', - version: '4.0.0', - }, - ]) - - writeYamlFile('pnpm-workspace.yaml', { packages: ['**', '!store/**'] }) - - process.chdir('project-1') - - await execPnpm(['link', 'project-2']) - - await execPnpm(['link', 'project-3', '--save-dev']) - - await execPnpm(['link', 'project-4', '--save-optional']) - - const { default: pkg } = await import(path.resolve('package.json')) - - expect(pkg?.dependencies).toStrictEqual({ 'project-2': '^2.0.0' }) // spec of linked package added to dependencies - expect(pkg?.devDependencies).toStrictEqual({ 'project-3': '^3.0.0' }) // spec of linked package added to devDependencies - expect(pkg?.optionalDependencies).toStrictEqual({ 'project-4': '^4.0.0' }) // spec of linked package added to optionalDependencies - - projects['project-1'].has('project-2') - projects['project-1'].has('project-3') - projects['project-1'].has('project-4') -}) - test('linking a package inside a monorepo with --link-workspace-packages when installing new dependencies', async () => { const projects = preparePackages([ { @@ -335,7 +294,7 @@ test('topological order of packages with self-dependencies in monorepo is correc expect(server1.getLines()).toStrictEqual(['project-2', 'project-3', 'project-1']) - await execPnpm(['recursive', 'test']) + await execPnpm(['-r', 'test']) expect(server2.getLines()).toStrictEqual(['project-2', 'project-3', 'project-1']) }) @@ -400,7 +359,7 @@ test('test-pattern is respected by the test script', async () => { await execPnpm(['install']) - await execPnpm(['recursive', 'test', '--filter', '...[origin/main]']) + await execPnpm(['--filter', '...[origin/main]', 'test']) // Expecting only project-2 and project-4 to run since they were changed above. expect(server.getLines().sort()).toEqual(['project-2', 'project-4']) diff --git a/pnpm/test/recursive/misc.ts b/pnpm/test/recursive/misc.ts index 6d893988ea5..28d5045801a 100644 --- a/pnpm/test/recursive/misc.ts +++ b/pnpm/test/recursive/misc.ts @@ -1,5 +1,6 @@ import fs from 'fs' import path from 'path' +import { STORE_VERSION } from '@pnpm/constants' import { prepare, preparePackages } from '@pnpm/prepare' import { type LockfileV9 as Lockfile } from '@pnpm/lockfile.types' import { sync as readYamlFile } from 'read-yaml-file' @@ -122,7 +123,7 @@ skipOnWindows('recursive installation using server', async () => { const storeDir = path.resolve('store') spawnPnpm(['server', 'start'], { storeDir }) - const serverJsonPath = path.resolve(storeDir, 'v3/server/server.json') + const serverJsonPath = path.resolve(storeDir, STORE_VERSION, 'server/server.json') const serverJson = await retryLoadJsonFile<{ connectionOptions: object }>(serverJsonPath) expect(serverJson).toBeTruthy() expect(serverJson.connectionOptions).toBeTruthy() diff --git a/pnpm/test/run.ts b/pnpm/test/run.ts index 74820a4bf26..8c75f05bddf 100644 --- a/pnpm/test/run.ts +++ b/pnpm/test/run.ts @@ -86,7 +86,7 @@ test('exit code of child process is preserved', async () => { expect(result.status).toBe(87) }) -test('test -r: pass the args to the command that is specified in the build script of a package.json manifest', async () => { +test('recursive test: pass the args to the command that is specified in the build script of a package.json manifest', async () => { preparePackages([{ name: 'project', scripts: { @@ -94,7 +94,7 @@ test('test -r: pass the args to the command that is specified in the build scrip }, }]) - const result = execPnpmSync(['test', '-r', 'arg', '--', '--flag=true']) + const result = execPnpmSync(['-r', 'test', 'arg', '--flag=true']) expect((result.stdout as Buffer).toString('utf8')).toMatch(/ts-node test "arg" "--flag=true"/) }) diff --git a/pnpm/test/server.ts b/pnpm/test/server.ts index d2a5ba186a9..345455df922 100644 --- a/pnpm/test/server.ts +++ b/pnpm/test/server.ts @@ -4,6 +4,7 @@ import { type Readable } from 'stream' import { promisify } from 'util' import path from 'path' import byline from '@pnpm/byline' +import { STORE_VERSION } from '@pnpm/constants' import { type Project, prepare } from '@pnpm/prepare' import { REGISTRY_MOCK_PORT } from '@pnpm/registry-mock' import delay, { type ClearablePromise } from 'delay' @@ -47,7 +48,7 @@ function prepareServerTest (serverStartArgs?: readonly string[]): TestSetup { const project = prepare() spawnPnpm(['server', 'start', ...(serverStartArgs ?? [])]) - const serverJsonPath = path.resolve('..', 'store/v3/server/server.json') + const serverJsonPath = path.resolve('..', `store/${STORE_VERSION}/server/server.json`) async function onTestEnd () { await expect(execPnpm(['server', 'stop'])).resolves.not.toThrow() @@ -173,7 +174,7 @@ skipOnWindows('installation using store server started in the background', async await expect(execPnpm(['install', 'is-positive@1.0.0', '--use-store-server'])).resolves.not.toThrow() - const serverJsonPath = path.resolve('..', 'store/v3/server/server.json') + const serverJsonPath = path.resolve('..', `store/${STORE_VERSION}/server/server.json`) try { const serverJson = await retryLoadJsonFile<{ connectionOptions: object }>(serverJsonPath) @@ -194,7 +195,7 @@ skipOnWindows('store server started in the background should use store location await expect(execPnpm(['add', 'is-positive@1.0.0', '--use-store-server', '--store-dir', '../store2'])).resolves.not.toThrow() - const serverJsonPath = path.resolve('..', 'store2/v3/server/server.json') + const serverJsonPath = path.resolve('..', `store2/${STORE_VERSION}/server/server.json`) try { const serverJson = await retryLoadJsonFile<{ connectionOptions: object }>(serverJsonPath) @@ -310,7 +311,7 @@ skipOnWindows('parallel server starts against the same store should result in on }, timeoutMillis: 60000, })).resolves.not.toThrow() - const serverJsonPath = path.resolve('..', 'store/v3/server/server.json') + const serverJsonPath = path.resolve('..', `store/${STORE_VERSION}/server/server.json`) expect(fs.existsSync(serverJsonPath)).toBeFalsy() }) @@ -319,7 +320,7 @@ skipOnWindows('installation without store server running in the background', asy await expect(execPnpm(['install', 'is-positive@1.0.0', '--no-use-store-server'])).resolves.not.toThrow() - const serverJsonPath = path.resolve('..', 'store/v3/server/server.json') + const serverJsonPath = path.resolve('..', `store/${STORE_VERSION}/server/server.json`) expect(fs.existsSync(serverJsonPath)).toBeFalsy() expect(project.requireModule('is-positive')).toBeTruthy() @@ -332,7 +333,7 @@ skipOnWindows('installation without store server running in the background', asy test.skip('fail if the store server is run by a different version of pnpm', async () => { prepare() - const serverJsonPath = path.resolve('..', 'store/v3/server/server.json') + const serverJsonPath = path.resolve('..', `store/${STORE_VERSION}/server/server.json`) writeJsonFile.sync(serverJsonPath, { pnpmVersion: '2.0.0' }) const result = execPnpmSync(['install', 'is-positive@1.0.0']) diff --git a/pnpm/test/switchingVersions.test.ts b/pnpm/test/switchingVersions.test.ts index f4b7cef9158..623f6dede4e 100644 --- a/pnpm/test/switchingVersions.test.ts +++ b/pnpm/test/switchingVersions.test.ts @@ -6,11 +6,10 @@ import { sync as writeJsonFile } from 'write-json-file' import { execPnpmSync } from './utils' import isWindows from 'is-windows' -test('switch to the pnpm version specified in the packageManager field of package.json, when manager-package-manager=versions is true', async () => { +test('switch to the pnpm version specified in the packageManager field of package.json', async () => { prepare() const pnpmHome = path.resolve('pnpm') const env = { PNPM_HOME: pnpmHome } - fs.writeFileSync('.npmrc', 'manage-package-manager-versions=true') writeJsonFile('package.json', { packageManager: 'pnpm@9.3.0', }) @@ -20,10 +19,11 @@ test('switch to the pnpm version specified in the packageManager field of packag expect(stdout.toString()).toContain('Version 9.3.0') }) -test('do not switch to the pnpm version specified in the packageManager field of package.json', async () => { +test('do not switch to the pnpm version specified in the packageManager field of package.json, if manage-package-manager-versions is set to false', async () => { prepare() const pnpmHome = path.resolve('pnpm') const env = { PNPM_HOME: pnpmHome } + fs.writeFileSync('.npmrc', 'manage-package-manager-versions=false') writeJsonFile('package.json', { packageManager: 'pnpm@9.3.0', }) @@ -37,7 +37,6 @@ test('do not switch to pnpm version that is specified not with a semver version' prepare() const pnpmHome = path.resolve('pnpm') const env = { PNPM_HOME: pnpmHome } - fs.writeFileSync('.npmrc', 'manage-package-manager-versions=true') writeJsonFile('package.json', { packageManager: 'pnpm@kevva/is-positive', }) @@ -51,7 +50,6 @@ test('do not switch to pnpm version when a range is specified', async () => { prepare() const pnpmHome = path.resolve('pnpm') const env = { PNPM_HOME: pnpmHome } - fs.writeFileSync('.npmrc', 'manage-package-manager-versions=true') writeJsonFile('package.json', { packageManager: 'pnpm@^9.3.0', }) diff --git a/pnpm/tsconfig.json b/pnpm/tsconfig.json index c85c0d879da..04e039f209a 100644 --- a/pnpm/tsconfig.json +++ b/pnpm/tsconfig.json @@ -57,6 +57,9 @@ { "path": "../config/plugin-commands-config" }, + { + "path": "../crypto/hash" + }, { "path": "../env/path" }, @@ -84,9 +87,6 @@ { "path": "../packages/core-loggers" }, - { - "path": "../packages/crypto.base32-hash" - }, { "path": "../packages/dependency-path" }, diff --git a/releasing/plugin-commands-deploy/test/utils/index.ts b/releasing/plugin-commands-deploy/test/utils/index.ts index 021d4d8c268..c72a61d863e 100644 --- a/releasing/plugin-commands-deploy/test/utils/index.ts +++ b/releasing/plugin-commands-deploy/test/utils/index.ts @@ -49,5 +49,5 @@ export const DEFAULT_OPTS = { useRunningStoreServer: false, useStoreServer: false, workspaceConcurrency: 4, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, } diff --git a/releasing/plugin-commands-publishing/test/utils/index.ts b/releasing/plugin-commands-publishing/test/utils/index.ts index fed4cce537a..38312148fd7 100644 --- a/releasing/plugin-commands-publishing/test/utils/index.ts +++ b/releasing/plugin-commands-publishing/test/utils/index.ts @@ -46,7 +46,7 @@ export const DEFAULT_OPTS = { useRunningStoreServer: false, useStoreServer: false, workspaceConcurrency: 4, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, } export async function checkPkgExists (packageName: string, expectedVersion: string): Promise { diff --git a/resolving/npm-resolver/package.json b/resolving/npm-resolver/package.json index ffddef11af3..752ece42a3f 100644 --- a/resolving/npm-resolver/package.json +++ b/resolving/npm-resolver/package.json @@ -36,7 +36,7 @@ "dependencies": { "@pnpm/constants": "workspace:*", "@pnpm/core-loggers": "workspace:*", - "@pnpm/crypto.polyfill": "workspace:*", + "@pnpm/crypto.hash": "workspace:*", "@pnpm/error": "workspace:*", "@pnpm/fetching-types": "workspace:*", "@pnpm/graceful-fs": "workspace:*", diff --git a/resolving/npm-resolver/src/pickPackage.ts b/resolving/npm-resolver/src/pickPackage.ts index ffa75c1f6b2..bb21d401110 100644 --- a/resolving/npm-resolver/src/pickPackage.ts +++ b/resolving/npm-resolver/src/pickPackage.ts @@ -1,6 +1,6 @@ import { promises as fs } from 'fs' import path from 'path' -import * as crypto from '@pnpm/crypto.polyfill' +import { createHexHash } from '@pnpm/crypto.hash' import { PnpmError } from '@pnpm/error' import { logger } from '@pnpm/logger' import gfs from '@pnpm/graceful-fs' @@ -270,7 +270,7 @@ function clearMeta (pkg: PackageMeta): PackageMeta { function encodePkgName (pkgName: string): string { if (pkgName !== pkgName.toLowerCase()) { - return `${pkgName}_${crypto.hash('md5', pkgName, 'hex')}` + return `${pkgName}_${createHexHash(pkgName)}` } return pkgName } diff --git a/resolving/npm-resolver/test/index.ts b/resolving/npm-resolver/test/index.ts index 544a8756221..1a601f9095d 100644 --- a/resolving/npm-resolver/test/index.ts +++ b/resolving/npm-resolver/test/index.ts @@ -1,6 +1,7 @@ /// import fs from 'fs' import path from 'path' +import { createHexHash } from '@pnpm/crypto.hash' import { PnpmError } from '@pnpm/error' import { createFetchFromRegistry } from '@pnpm/fetch' import { @@ -110,7 +111,7 @@ test('resolveFromNpm() should save metadata to a unique file when the package na // The resolve function does not wait for the package meta cache file to be saved // so we must delay for a bit in order to read it - const meta = await retryLoadJsonFile(path.join(cacheDir, 'metadata/registry.npmjs.org/JSON_0ecd11c1d7a287401d148a23bbd7a2f8.json')) // eslint-disable-line @typescript-eslint/no-explicit-any + const meta = await retryLoadJsonFile(path.join(cacheDir, `metadata/registry.npmjs.org/JSON_${createHexHash('JSON')}.json`)) // eslint-disable-line @typescript-eslint/no-explicit-any expect(meta.name).toBeTruthy() expect(meta.versions).toBeTruthy() expect(meta['dist-tags']).toBeTruthy() diff --git a/resolving/npm-resolver/tsconfig.json b/resolving/npm-resolver/tsconfig.json index 21665b38cdf..7769c4a74af 100644 --- a/resolving/npm-resolver/tsconfig.json +++ b/resolving/npm-resolver/tsconfig.json @@ -13,7 +13,7 @@ "path": "../../__utils__/test-fixtures" }, { - "path": "../../crypto/polyfill" + "path": "../../crypto/hash" }, { "path": "../../fs/graceful-fs" diff --git a/reviewing/dependencies-hierarchy/test/index.ts b/reviewing/dependencies-hierarchy/test/index.ts index a639de3e724..07f0400bf2f 100644 --- a/reviewing/dependencies-hierarchy/test/index.ts +++ b/reviewing/dependencies-hierarchy/test/index.ts @@ -5,6 +5,7 @@ import { fixtures } from '@pnpm/test-fixtures' import { buildDependenciesHierarchy, type PackageNode } from '@pnpm/reviewing.dependencies-hierarchy' import { depPathToFilename } from '@pnpm/dependency-path' +const virtualStoreDirMaxLength = process.platform === 'win32' ? 60 : 120 const f = fixtures(__dirname) const generalFixture = f.find('general') const withPeerFixture = f.find('with-peer') @@ -19,7 +20,7 @@ const workspaceWithNestedWorkspaceDeps = f.find('workspace-with-nested-workspace const customModulesDirFixture = f.find('custom-modules-dir') test('one package depth 0', async () => { - const tree = await buildDependenciesHierarchy([generalFixture], { depth: 0, lockfileDir: generalFixture, virtualStoreDirMaxLength: 120 }) + const tree = await buildDependenciesHierarchy([generalFixture], { depth: 0, lockfileDir: generalFixture, virtualStoreDirMaxLength }) const modulesDir = path.join(generalFixture, 'node_modules') expect(tree).toStrictEqual({ @@ -80,7 +81,7 @@ test('one package depth 0', async () => { }) test('one package depth 1', async () => { - const tree = await buildDependenciesHierarchy([generalFixture], { depth: 1, lockfileDir: generalFixture, virtualStoreDirMaxLength: 120 }) + const tree = await buildDependenciesHierarchy([generalFixture], { depth: 1, lockfileDir: generalFixture, virtualStoreDirMaxLength }) const modulesDir = path.join(generalFixture, 'node_modules') expect(tree).toStrictEqual({ @@ -179,7 +180,7 @@ test('only prod depth 0', async () => { optionalDependencies: false, }, lockfileDir: generalFixture, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength, } ) const modulesDir = path.join(generalFixture, 'node_modules') @@ -225,7 +226,7 @@ test('only dev depth 0', async () => { optionalDependencies: false, }, lockfileDir: generalFixture, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength, } ) const modulesDir = path.join(generalFixture, 'node_modules') @@ -254,7 +255,7 @@ test('hierarchy for no packages', async () => { depth: 100, lockfileDir: generalFixture, search: () => false, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength, }) expect(tree).toStrictEqual({ @@ -273,7 +274,7 @@ test('filter 1 package with depth 0', async () => { depth: 0, lockfileDir: generalFixture, search: ({ name }) => name === 'rimraf', - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength, } ) const modulesDir = path.join(generalFixture, 'node_modules') @@ -304,7 +305,7 @@ test('circular dependency', async () => { const tree = await buildDependenciesHierarchy([circularFixture], { depth: 1000, lockfileDir: circularFixture, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength, }) const modulesDir = path.join(circularFixture, 'node_modules') @@ -340,7 +341,7 @@ test('local package depth 0', async () => { const tree = await buildDependenciesHierarchy([withFileDepFixture], { depth: 1, lockfileDir: withFileDepFixture, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength, }) const modulesDir = path.join(withFileDepFixture, 'node_modules') @@ -378,7 +379,7 @@ test('on a package that has only links', async () => { const tree = await buildDependenciesHierarchy([withLinksOnlyFixture], { depth: 1000, lockfileDir: withLinksOnlyFixture, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength, }) expect(tree).toStrictEqual({ @@ -407,7 +408,7 @@ test('on a package with nested workspace links', async () => { { depth: 1000, lockfileDir: workspaceWithNestedWorkspaceDeps, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength, } ) @@ -449,7 +450,7 @@ test('unsaved dependencies are listed', async () => { expect(await buildDependenciesHierarchy([withUnsavedDepsFixture], { depth: 0, lockfileDir: withUnsavedDepsFixture, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength, })) .toStrictEqual({ [withUnsavedDepsFixture]: { @@ -492,7 +493,7 @@ test('unsaved dependencies are listed and filtered', async () => { depth: 0, lockfileDir: withUnsavedDepsFixture, search: ({ name }) => name === 'symlink-dir', - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength, } ) ).toStrictEqual({ @@ -522,7 +523,7 @@ test(`do not fail on importers that are not in current ${WANTED_LOCKFILE}`, asyn expect(await buildDependenciesHierarchy([fixtureMonorepo], { depth: 0, lockfileDir: fixtureMonorepo, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength, })).toStrictEqual({ [fixtureMonorepo]: {} }) }) @@ -532,7 +533,7 @@ test('dependency with an alias', async () => { await buildDependenciesHierarchy([withAliasedDepFixture], { depth: 0, lockfileDir: withAliasedDepFixture, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength, }) ).toStrictEqual({ [withAliasedDepFixture]: { @@ -559,7 +560,7 @@ test('peer dependencies', async () => { const hierarchy = await buildDependenciesHierarchy([withPeerFixture], { depth: 1, lockfileDir: withPeerFixture, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength, }) expect(hierarchy[withPeerFixture].dependencies![1].dependencies![0].name).toEqual('ajv') expect(hierarchy[withPeerFixture].dependencies![1].dependencies![0].isPeer).toEqual(true) @@ -573,7 +574,7 @@ test('dependency without a package.json', async () => { const tree = await buildDependenciesHierarchy([withNonPackageDepFixture], { depth: 0, lockfileDir: withNonPackageDepFixture, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength, }) const resolved = `https://codeload.github.com/${org}/${pkg}/tar.gz/${commit}` expect(tree).toStrictEqual({ @@ -586,7 +587,7 @@ test('dependency without a package.json', async () => { isPeer: false, isSkipped: false, name: 'camelcase', - path: path.join(withNonPackageDepFixture, 'node_modules/.pnpm', `camelcase@${depPathToFilename(resolved, 120)}`, 'node_modules/camelcase'), + path: path.join(withNonPackageDepFixture, 'node_modules/.pnpm', depPathToFilename(`camelcase@${resolved}`, virtualStoreDirMaxLength), 'node_modules/camelcase'), resolved, version: '0.0.0', }, @@ -613,7 +614,7 @@ test('on custom modules-dir workspaces', async () => { depth: 1000, lockfileDir: customModulesDirFixture, modulesDir: 'fake_modules', - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength, } ) expect(tree).toEqual({ diff --git a/reviewing/license-scanner/src/getPkgInfo.ts b/reviewing/license-scanner/src/getPkgInfo.ts index b0435f99a54..0d69d04341a 100644 --- a/reviewing/license-scanner/src/getPkgInfo.ts +++ b/reviewing/license-scanner/src/getPkgInfo.ts @@ -8,6 +8,7 @@ import { type PackageManifest, type Registries } from '@pnpm/types' import { getFilePathByModeInCafs, getIndexFilePathInCafs, + type PackageFiles, type PackageFileInfo, type PackageFilesIndex, } from '@pnpm/store.cafs' @@ -155,9 +156,9 @@ async function parseLicense ( manifest: PackageManifest files: | { local: true, files: Record } - | { local: false, files: Record } + | { local: false, files: PackageFiles } }, - opts: { cafsDir: string } + opts: { storeDir: string } ): Promise { let licenseField: unknown = pkg.manifest.license if ('licenses' in pkg.manifest) { @@ -179,7 +180,7 @@ async function parseLicense ( if (pkg.files.local) { licenseContents = await readFile(licensePackageFileInfo as string) } else { - licenseContents = await readLicenseFileFromCafs(opts.cafsDir, licensePackageFileInfo as PackageFileInfo) + licenseContents = await readLicenseFileFromCafs(opts.storeDir, licensePackageFileInfo as PackageFileInfo) } const licenseContent = licenseContents?.toString('utf-8') let name = 'Unknown' @@ -202,22 +203,21 @@ async function parseLicense ( /** * Fetch a file by integrity id from the content-addressable store - * @param cafsDir the cafs directory + * @param storeDir the cafs directory * @param opts the options for reading file * @returns Promise */ -async function readLicenseFileFromCafs (cafsDir: string, { integrity, mode }: PackageFileInfo): Promise { - const fileName = getFilePathByModeInCafs(cafsDir, integrity, mode) +async function readLicenseFileFromCafs (storeDir: string, { integrity, mode }: PackageFileInfo): Promise { + const fileName = getFilePathByModeInCafs(storeDir, integrity, mode) const fileContents = await readFile(fileName) return fileContents } export type ReadPackageIndexFileResult = - | { local: false, files: Record } + | { local: false, files: PackageFiles } | { local: true, files: Record } export interface ReadPackageIndexFileOptions { - cafsDir: string storeDir: string lockfileDir: string virtualStoreDirMaxLength: number @@ -253,10 +253,12 @@ export async function readPackageIndexFile ( let pkgIndexFilePath if (isPackageWithIntegrity) { + const parsedId = parse(id) // Retrieve all the index file of all files included in the package pkgIndexFilePath = getIndexFilePathInCafs( - opts.cafsDir, - packageResolution.integrity as string + opts.storeDir, + packageResolution.integrity as string, + `${parsedId.name}@${parsedId.version}` ) } else if (!packageResolution.type && packageResolution.tarball) { const packageDirInStore = depPathToFilename(parse(id).nonSemverVersion ?? id, opts.virtualStoreDirMaxLength) @@ -282,7 +284,7 @@ export async function readPackageIndexFile ( if (err.code === 'ENOENT') { throw new PnpmError( 'MISSING_PACKAGE_INDEX_FILE', - `Failed to find package index file for ${id}, please consider running 'pnpm install'` + `Failed to find package index file for ${id} (at ${pkgIndexFilePath}), please consider running 'pnpm install'` ) } @@ -321,8 +323,6 @@ export async function getPkgInfo ( pkg: PackageInfo, opts: GetPackageInfoOptions ): Promise { - const cafsDir = path.join(opts.storeDir, 'files') - // Retrieve file index for the requested package const packageResolution = pkgSnapshotToResolution( pkg.depPath, @@ -334,7 +334,6 @@ export async function getPkgInfo ( packageResolution as Resolution, pkg.id, { - cafsDir, storeDir: opts.storeDir, lockfileDir: opts.dir, virtualStoreDirMaxLength: opts.virtualStoreDirMaxLength, @@ -352,7 +351,7 @@ export async function getPkgInfo ( > const packageManifestFile = packageFileIndex['package.json'] packageManifestDir = getFilePathByModeInCafs( - cafsDir, + opts.storeDir, packageManifestFile.integrity, packageManifestFile.mode ) @@ -388,7 +387,7 @@ export async function getPkgInfo ( const licenseInfo = await parseLicense( { manifest, files: packageFileIndexInfo }, - { cafsDir } + { storeDir: opts.storeDir } ) const packageInfo = { diff --git a/reviewing/license-scanner/test/getPkgInfo.spec.ts b/reviewing/license-scanner/test/getPkgInfo.spec.ts index 1ed067270e4..2dd7aff3d8e 100644 --- a/reviewing/license-scanner/test/getPkgInfo.spec.ts +++ b/reviewing/license-scanner/test/getPkgInfo.spec.ts @@ -1,3 +1,4 @@ +import path from 'path' import { getPkgInfo } from '../lib/getPkgInfo' export const DEFAULT_REGISTRIES = { @@ -11,8 +12,8 @@ describe('licences', () => { { name: 'bogus-package', version: '1.0.0', - id: '/bogus-package@1.0.0', - depPath: '/bogus-package@1.0.0', + id: 'bogus-package@1.0.0', + depPath: 'bogus-package@1.0.0', snapshot: { resolution: { integrity: 'integrity-sha', @@ -28,6 +29,6 @@ describe('licences', () => { virtualStoreDirMaxLength: 120, } ) - ).rejects.toThrow('Failed to find package index file for /bogus-package@1.0.0, please consider running \'pnpm install\'') + ).rejects.toThrow(`Failed to find package index file for bogus-package@1.0.0 (at ${path.join('store-dir', 'index', 'b2', '16-bogus-package@1.0.0.json')}), please consider running 'pnpm install'`) }) }) diff --git a/reviewing/plugin-commands-licenses/test/fixtures/with-git-protocol-patched-deps/pnpm-lock.yaml b/reviewing/plugin-commands-licenses/test/fixtures/with-git-protocol-patched-deps/pnpm-lock.yaml index f11b00ac0bb..e3111ae589c 100644 --- a/reviewing/plugin-commands-licenses/test/fixtures/with-git-protocol-patched-deps/pnpm-lock.yaml +++ b/reviewing/plugin-commands-licenses/test/fixtures/with-git-protocol-patched-deps/pnpm-lock.yaml @@ -6,7 +6,7 @@ settings: patchedDependencies: is-positive@3.1.0: - hash: 3wijq7ky55ptqab6aghokum24i + hash: b27bbf2d83e68cac4491a38dd8b846aadd55d9c7bf8a4971139465c4de3566ce path: patches/is-positive@3.1.0.patch importers: @@ -15,7 +15,7 @@ importers: dependencies: is-positive: specifier: github:kevva/is-positive - version: https://codeload.github.com/kevva/is-positive/tar.gz/97edff6f525f192a3f83cea1944765f769ae2678(patch_hash=3wijq7ky55ptqab6aghokum24i) + version: https://codeload.github.com/kevva/is-positive/tar.gz/97edff6f525f192a3f83cea1944765f769ae2678(patch_hash=b27bbf2d83e68cac4491a38dd8b846aadd55d9c7bf8a4971139465c4de3566ce) packages: @@ -26,5 +26,4 @@ packages: snapshots: - is-positive@https://codeload.github.com/kevva/is-positive/tar.gz/97edff6f525f192a3f83cea1944765f769ae2678(patch_hash=3wijq7ky55ptqab6aghokum24i): - dev: false + is-positive@https://codeload.github.com/kevva/is-positive/tar.gz/97edff6f525f192a3f83cea1944765f769ae2678(patch_hash=b27bbf2d83e68cac4491a38dd8b846aadd55d9c7bf8a4971139465c4de3566ce): {} diff --git a/reviewing/plugin-commands-licenses/test/index.ts b/reviewing/plugin-commands-licenses/test/index.ts index 1484b22dd26..cbdacd29f83 100644 --- a/reviewing/plugin-commands-licenses/test/index.ts +++ b/reviewing/plugin-commands-licenses/test/index.ts @@ -1,6 +1,7 @@ /// import path from 'path' import fs from 'fs' +import { STORE_VERSION } from '@pnpm/constants' import { licenses } from '@pnpm/plugin-commands-licenses' import { install } from '@pnpm/plugin-commands-installation' import { tempDir } from '@pnpm/prepare' @@ -29,9 +30,9 @@ test('pnpm licenses', async () => { dir: workspaceDir, pnpmHomeDir: '', long: false, - // we need to prefix it with v3 otherwise licenses tool can't find anything + // we need to prefix it with STORE_VERSION otherwise licenses tool can't find anything // in the content-addressable directory - storeDir: path.resolve(storeDir, 'v3'), + storeDir: path.resolve(storeDir, STORE_VERSION), }, ['list']) expect(exitCode).toBe(0) @@ -56,9 +57,9 @@ test('pnpm licenses: show details', async () => { dir: workspaceDir, pnpmHomeDir: '', long: true, - // we need to prefix it with v3 otherwise licenses tool can't find anything + // we need to prefix it with STORE_VERSION otherwise licenses tool can't find anything // in the content-addressable directory - storeDir: path.resolve(storeDir, 'v3'), + storeDir: path.resolve(storeDir, STORE_VERSION), }, ['list']) expect(exitCode).toBe(0) @@ -84,9 +85,9 @@ test('pnpm licenses: output as json', async () => { pnpmHomeDir: '', long: false, json: true, - // we need to prefix it with v3 otherwise licenses tool can't find anything + // we need to prefix it with STORE_VERSION otherwise licenses tool can't find anything // in the content-addressable directory - storeDir: path.resolve(storeDir, 'v3'), + storeDir: path.resolve(storeDir, STORE_VERSION), }, ['list']) expect(exitCode).toBe(0) @@ -148,7 +149,7 @@ test('pnpm licenses: path should be correct for workspaces', async () => { pnpmHomeDir: '', long: false, json: true, - storeDir: path.resolve(storeDir, 'v3'), + storeDir: path.resolve(storeDir, STORE_VERSION), }, ['list']) expect(exitCode).toBe(0) @@ -199,7 +200,7 @@ test('pnpm licenses: filter outputs', async () => { path.includes('bar') ) ), - storeDir: path.resolve(storeDir, 'v3'), + storeDir: path.resolve(storeDir, STORE_VERSION), }, ['list'] ) @@ -238,9 +239,9 @@ test('pnpm licenses: should correctly read LICENSE file with executable file mod dir: workspaceDir, pnpmHomeDir: '', long: true, - // we need to prefix it with v3 otherwise licenses tool can't find anything + // we need to prefix it with STORE_VERSION otherwise licenses tool can't find anything // in the content-addressable directory - storeDir: path.resolve(storeDir, 'v3'), + storeDir: path.resolve(storeDir, STORE_VERSION), }, ['list']) expect(exitCode).toBe(0) @@ -264,7 +265,7 @@ test('pnpm licenses should work with file protocol dependency', async () => { dir: workspaceDir, pnpmHomeDir: '', long: false, - storeDir: path.resolve(storeDir, 'v3'), + storeDir: path.resolve(storeDir, STORE_VERSION), }, ['list']) expect(exitCode).toBe(0) @@ -289,7 +290,7 @@ test('pnpm licenses should work with git protocol dep that have patches', async dir: workspaceDir, pnpmHomeDir: '', long: false, - storeDir: path.resolve(storeDir, 'v3'), + storeDir: path.resolve(storeDir, STORE_VERSION), }, ['list']) expect(exitCode).toBe(0) @@ -312,7 +313,7 @@ test('pnpm licenses should work with git protocol dep that have peerDependencies dir: workspaceDir, pnpmHomeDir: '', long: false, - storeDir: path.resolve(storeDir, 'v3'), + storeDir: path.resolve(storeDir, STORE_VERSION), }, ['list']) expect(exitCode).toBe(0) @@ -335,7 +336,7 @@ test('pnpm licenses should work git repository name containing capital letters', dir: workspaceDir, pnpmHomeDir: '', long: false, - storeDir: path.resolve(storeDir, 'v3'), + storeDir: path.resolve(storeDir, STORE_VERSION), }, ['list']) expect(exitCode).toBe(0) diff --git a/reviewing/plugin-commands-licenses/test/utils/index.ts b/reviewing/plugin-commands-licenses/test/utils/index.ts index 0da9ebdecaa..c900bf9f3f1 100644 --- a/reviewing/plugin-commands-licenses/test/utils/index.ts +++ b/reviewing/plugin-commands-licenses/test/utils/index.ts @@ -47,5 +47,5 @@ export const DEFAULT_OPTS = { useStoreServer: false, virtualStoreDir: 'node_modules/.pnpm', workspaceConcurrency: 4, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, } diff --git a/reviewing/plugin-commands-listing/test/index.ts b/reviewing/plugin-commands-listing/test/index.ts index babf699621b..ff17eabee14 100644 --- a/reviewing/plugin-commands-listing/test/index.ts +++ b/reviewing/plugin-commands-listing/test/index.ts @@ -28,7 +28,7 @@ test('listing packages', async () => { dev: false, dir: process.cwd(), optional: false, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, []) expect(stripAnsi(output)).toBe(`Legend: production dependency, optional only, dev only @@ -44,7 +44,7 @@ is-positive 1.0.0`) dir: process.cwd(), optional: false, production: false, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, []) expect(stripAnsi(output)).toBe(`Legend: production dependency, optional only, dev only @@ -58,7 +58,7 @@ is-negative 1.0.0`) { const output = await list.handler({ dir: process.cwd(), - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, []) expect(stripAnsi(output)).toBe(`Legend: production dependency, optional only, dev only @@ -95,7 +95,7 @@ test(`listing packages of a project that has an external ${WANTED_LOCKFILE}`, as const output = await list.handler({ dir: process.cwd(), lockfileDir: path.resolve('..'), - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, []) expect(stripAnsi(output)).toBe(`Legend: production dependency, optional only, dev only @@ -117,7 +117,7 @@ test.skip('list on a project with skipped optional dependencies', async () => { const output = await list.handler({ depth: 10, dir: process.cwd(), - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, []) expect(stripAnsi(output)).toBe(`Legend: production dependency, optional only, dev only @@ -134,7 +134,7 @@ pkg-with-optional 1.0.0 const output = await list.handler({ depth: 10, dir: process.cwd(), - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, ['not-compatible-with-any-os']) expect(stripAnsi(output)).toBe(`Legend: production dependency, optional only, dev only @@ -149,7 +149,7 @@ pkg-with-optional 1.0.0 { const output = await why.handler({ dir: process.cwd(), - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, ['not-compatible-with-any-os']) expect(stripAnsi(output)).toBe(`Legend: production dependency, optional only, dev only @@ -186,7 +186,7 @@ test('listing packages should not fail on package that has local file directory dev: false, dir: pkgDir, optional: false, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, []) expect(stripAnsi(output)).toBe(`Legend: production dependency, optional only, dev only diff --git a/reviewing/plugin-commands-listing/test/utils/index.ts b/reviewing/plugin-commands-listing/test/utils/index.ts index 82a3f646919..e03b6692d4a 100644 --- a/reviewing/plugin-commands-listing/test/utils/index.ts +++ b/reviewing/plugin-commands-listing/test/utils/index.ts @@ -47,5 +47,5 @@ export const DEFAULT_OPTS = { useRunningStoreServer: false, useStoreServer: false, workspaceConcurrency: 4, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, } diff --git a/reviewing/plugin-commands-listing/test/why.ts b/reviewing/plugin-commands-listing/test/why.ts index 82b2f1eabee..0a1519c22f9 100644 --- a/reviewing/plugin-commands-listing/test/why.ts +++ b/reviewing/plugin-commands-listing/test/why.ts @@ -15,7 +15,7 @@ test('`pnpm why` should fail if no package name was provided', async () => { try { await why.handler({ dir: process.cwd(), - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, []) } catch (_err: any) { // eslint-disable-line err = _err @@ -39,7 +39,7 @@ test('"why" should find non-direct dependency', async () => { dev: false, dir: process.cwd(), optional: false, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, ['@pnpm.e2e/dep-of-pkg-with-1-dep']) expect(stripAnsi(output)).toBe(`Legend: production dependency, optional only, dev only diff --git a/reviewing/plugin-commands-outdated/test/utils/index.ts b/reviewing/plugin-commands-outdated/test/utils/index.ts index 5ec219608ab..9ea80030ea5 100644 --- a/reviewing/plugin-commands-outdated/test/utils/index.ts +++ b/reviewing/plugin-commands-outdated/test/utils/index.ts @@ -51,7 +51,7 @@ export const DEFAULT_OPTS = { useRunningStoreServer: false, useStoreServer: false, workspaceConcurrency: 4, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, } export const DEFAULT_OUTDATED_OPTS = { diff --git a/store/cafs-types/src/index.ts b/store/cafs-types/src/index.ts index 2f1cbe1ba62..354c420dd19 100644 --- a/store/cafs-types/src/index.ts +++ b/store/cafs-types/src/index.ts @@ -1,6 +1,8 @@ import type { IntegrityLike } from 'ssri' import type { DependencyManifest } from '@pnpm/types' +export type PackageFiles = Record + export interface PackageFileInfo { checkedAt?: number // Nullable for backward compatibility integrity: string @@ -8,19 +10,26 @@ export interface PackageFileInfo { size: number } +export type SideEffects = Record + +export interface SideEffectsDiff { + deleted?: string[] + added?: PackageFiles +} + export type ResolvedFrom = 'store' | 'local-dir' | 'remote' export type PackageFilesResponse = { resolvedFrom: ResolvedFrom packageImportMethod?: 'auto' | 'hardlink' | 'copy' | 'clone' | 'clone-or-copy' - sideEffects?: Record> + sideEffects?: SideEffects requiresBuild: boolean } & ({ unprocessed?: false filesIndex: Record } | { unprocessed: true - filesIndex: Record + filesIndex: PackageFiles }) export interface ImportPackageOpts { @@ -63,7 +72,7 @@ export interface AddToStoreResult { } export interface Cafs { - cafsDir: string + storeDir: string addFilesFromDir: (dir: string) => AddToStoreResult addFilesFromTarball: (buffer: Buffer) => AddToStoreResult getIndexFilePathInCafs: (integrity: string | IntegrityLike, fileType: FileType) => string diff --git a/store/cafs/src/addFilesFromDir.ts b/store/cafs/src/addFilesFromDir.ts index f312f35d75e..cdacfbee3dc 100644 --- a/store/cafs/src/addFilesFromDir.ts +++ b/store/cafs/src/addFilesFromDir.ts @@ -48,10 +48,12 @@ export function addFilesFromDir ( if (opts.readManifest && relativePath === 'package.json') { manifest = parseJsonBufferSync(buffer) as DependencyManifest } + // Remove the file type information (regular file, directory, etc.) and leave just the permission bits (rwx for owner, group, and others) + const mode = stat.mode & 0o777 filesIndex[relativePath] = { - mode: stat.mode, + mode, size: stat.size, - ...addBuffer(buffer, stat.mode), + ...addBuffer(buffer, mode), } } return { manifest, filesIndex } @@ -78,7 +80,9 @@ function findFiles ( for (const file of files) { const relativeSubdir = `${relativeDir}${relativeDir ? '/' : ''}${file.name}` if (file.isDirectory()) { - findFiles(filesList, path.join(dir, file.name), relativeSubdir) + if (relativeDir !== '' || file.name !== 'node_modules') { + findFiles(filesList, path.join(dir, file.name), relativeSubdir) + } continue } const absolutePath = path.join(dir, file.name) diff --git a/store/cafs/src/checkPkgFilesIntegrity.ts b/store/cafs/src/checkPkgFilesIntegrity.ts index c1de93224f6..73781c83d59 100644 --- a/store/cafs/src/checkPkgFilesIntegrity.ts +++ b/store/cafs/src/checkPkgFilesIntegrity.ts @@ -1,6 +1,6 @@ import fs from 'fs' import util from 'util' -import type { PackageFileInfo } from '@pnpm/cafs-types' +import { type PackageFiles, type PackageFileInfo, type SideEffects } from '@pnpm/cafs-types' import gfs from '@pnpm/graceful-fs' import { type DependencyManifest } from '@pnpm/types' import rimraf from '@zkochan/rimraf' @@ -20,8 +20,6 @@ export interface VerifyResult { manifest?: DependencyManifest } -export type SideEffects = Record> - export interface PackageFilesIndex { // name and version are nullable for backward compatibility // the initial specs of pnpm store v3 did not require these fields. @@ -31,12 +29,12 @@ export interface PackageFilesIndex { version?: string requiresBuild?: boolean - files: Record + files: PackageFiles sideEffects?: SideEffects } export function checkPkgFilesIntegrity ( - cafsDir: string, + storeDir: string, pkgIndex: PackageFilesIndex, readManifest?: boolean ): VerifyResult { @@ -44,17 +42,19 @@ export function checkPkgFilesIntegrity ( // but there's a smaller chance that the same file will be checked twice // so it's probably not worth the memory (this assumption should be verified) const verifiedFilesCache = new Set() - const _checkFilesIntegrity = checkFilesIntegrity.bind(null, verifiedFilesCache, cafsDir) + const _checkFilesIntegrity = checkFilesIntegrity.bind(null, verifiedFilesCache, storeDir) const verified = _checkFilesIntegrity(pkgIndex.files, readManifest) if (!verified) return { passed: false } if (pkgIndex.sideEffects) { // We verify all side effects cache. We could optimize it to verify only the side effects cache // that satisfies the current os/arch/platform. // However, it likely won't make a big difference. - for (const [sideEffectName, files] of Object.entries(pkgIndex.sideEffects)) { - const { passed } = _checkFilesIntegrity(files) - if (!passed) { - delete pkgIndex.sideEffects![sideEffectName] + for (const [sideEffectName, { added }] of Object.entries(pkgIndex.sideEffects)) { + if (added) { + const { passed } = _checkFilesIntegrity(added) + if (!passed) { + delete pkgIndex.sideEffects![sideEffectName] + } } } } @@ -63,8 +63,8 @@ export function checkPkgFilesIntegrity ( function checkFilesIntegrity ( verifiedFilesCache: Set, - cafsDir: string, - files: Record, + storeDir: string, + files: PackageFiles, readManifest?: boolean ): VerifyResult { let allVerified = true @@ -73,7 +73,7 @@ function checkFilesIntegrity ( if (!fstat.integrity) { throw new Error(`Integrity checksum is missing for ${f}`) } - const filename = getFilePathByModeInCafs(cafsDir, fstat.integrity, fstat.mode) + const filename = getFilePathByModeInCafs(storeDir, fstat.integrity, fstat.mode) const readFile = readManifest && f === 'package.json' if (!readFile && verifiedFilesCache.has(filename)) continue const verifyResult = verifyFile(filename, fstat, readFile) diff --git a/store/cafs/src/getFilePathInCafs.ts b/store/cafs/src/getFilePathInCafs.ts index 0546c80a48d..31f5249c58a 100644 --- a/store/cafs/src/getFilePathInCafs.ts +++ b/store/cafs/src/getFilePathInCafs.ts @@ -19,19 +19,27 @@ export const modeIsExecutable = (mode: number): boolean => (mode & 0o111) !== 0 export type FileType = 'exec' | 'nonexec' | 'index' export function getFilePathByModeInCafs ( - cafsDir: string, + storeDir: string, integrity: string | IntegrityLike, mode: number ): string { const fileType = modeIsExecutable(mode) ? 'exec' : 'nonexec' - return path.join(cafsDir, contentPathFromIntegrity(integrity, fileType)) + return path.join(storeDir, contentPathFromIntegrity(integrity, fileType)) } export function getIndexFilePathInCafs ( - cafsDir: string, - integrity: string | IntegrityLike + storeDir: string, + integrity: string | IntegrityLike, + pkgId: string ): string { - return path.join(cafsDir, contentPathFromIntegrity(integrity, 'index')) + const hex = ssri.parse(integrity, { single: true }).hexDigest().substring(0, 64) + // Some registries allow identical content to be published under different package names or versions. + // To accommodate this, index files are stored using both the content hash and package identifier. + // This approach ensures that we can: + // 1. Validate that the integrity in the lockfile corresponds to the correct package, + // which might not be the case after a poorly resolved Git conflict. + // 2. Allow the same content to be referenced by different packages or different versions of the same package. + return path.join(storeDir, `index/${path.join(hex.slice(0, 2), hex.slice(2))}-${pkgId.replace(/[\\/:*?"<>|]/g, '+')}.json`) } function contentPathFromIntegrity ( @@ -43,7 +51,7 @@ function contentPathFromIntegrity ( } export function contentPathFromHex (fileType: FileType, hex: string): string { - const p = path.join(hex.slice(0, 2), hex.slice(2)) + const p = path.join('files', hex.slice(0, 2), hex.slice(2)) switch (fileType) { case 'exec': return `${p}-exec` diff --git a/store/cafs/src/index.ts b/store/cafs/src/index.ts index c45b83831b1..fc77862f929 100644 --- a/store/cafs/src/index.ts +++ b/store/cafs/src/index.ts @@ -1,11 +1,10 @@ -import { type AddToStoreResult, type FileWriteResult, type PackageFileInfo, type FilesIndex } from '@pnpm/cafs-types' +import { type AddToStoreResult, type FileWriteResult, type PackageFiles, type PackageFileInfo, type FilesIndex } from '@pnpm/cafs-types' import ssri from 'ssri' import { addFilesFromDir } from './addFilesFromDir' import { addFilesFromTarball } from './addFilesFromTarball' import { checkPkgFilesIntegrity, type PackageFilesIndex, - type SideEffects, type VerifyResult, } from './checkPkgFilesIntegrity' import { readManifestFromStore } from './readManifestFromStore' @@ -27,8 +26,8 @@ export { getFilePathByModeInCafs, getIndexFilePathInCafs, type PackageFileInfo, + type PackageFiles, type PackageFilesIndex, - type SideEffects, optimisticRenameOverwrite, type FilesIndex, type VerifyResult, @@ -48,14 +47,14 @@ export interface CafsFunctions { getFilePathByModeInCafs: (integrity: string | ssri.IntegrityLike, mode: number) => string } -export function createCafs (cafsDir: string, { ignoreFile, cafsLocker }: CreateCafsOpts = {}): CafsFunctions { - const _writeBufferToCafs = writeBufferToCafs.bind(null, cafsLocker ?? new Map(), cafsDir) +export function createCafs (storeDir: string, { ignoreFile, cafsLocker }: CreateCafsOpts = {}): CafsFunctions { + const _writeBufferToCafs = writeBufferToCafs.bind(null, cafsLocker ?? new Map(), storeDir) const addBuffer = addBufferToCafs.bind(null, _writeBufferToCafs) return { addFilesFromDir: addFilesFromDir.bind(null, addBuffer), addFilesFromTarball: addFilesFromTarball.bind(null, addBuffer, ignoreFile ?? null), - getIndexFilePathInCafs: getIndexFilePathInCafs.bind(null, cafsDir), - getFilePathByModeInCafs: getFilePathByModeInCafs.bind(null, cafsDir), + getIndexFilePathInCafs: getIndexFilePathInCafs.bind(null, storeDir), + getFilePathByModeInCafs: getFilePathByModeInCafs.bind(null, storeDir), } } diff --git a/store/cafs/src/readManifestFromStore.ts b/store/cafs/src/readManifestFromStore.ts index 4ae8936d749..bb494f43be9 100644 --- a/store/cafs/src/readManifestFromStore.ts +++ b/store/cafs/src/readManifestFromStore.ts @@ -4,10 +4,10 @@ import { type PackageFilesIndex } from './checkPkgFilesIntegrity' import { getFilePathByModeInCafs } from './getFilePathInCafs' import { parseJsonBufferSync } from './parseJson' -export function readManifestFromStore (cafsDir: string, pkgIndex: PackageFilesIndex): PackageManifest | undefined { +export function readManifestFromStore (storeDir: string, pkgIndex: PackageFilesIndex): PackageManifest | undefined { const pkg = pkgIndex.files['package.json'] if (pkg) { - const fileName = getFilePathByModeInCafs(cafsDir, pkg.integrity, pkg.mode) + const fileName = getFilePathByModeInCafs(storeDir, pkg.integrity, pkg.mode) return parseJsonBufferSync(gfs.readFileSync(fileName)) as PackageManifest } return undefined diff --git a/store/cafs/src/writeBufferToCafs.ts b/store/cafs/src/writeBufferToCafs.ts index 573d2ecc65b..7d4a8839a9f 100644 --- a/store/cafs/src/writeBufferToCafs.ts +++ b/store/cafs/src/writeBufferToCafs.ts @@ -9,13 +9,13 @@ import { writeFile } from './writeFile' export function writeBufferToCafs ( locker: Map, - cafsDir: string, + storeDir: string, buffer: Buffer, fileDest: string, mode: number | undefined, integrity: ssri.IntegrityLike ): { checkedAt: number, filePath: string } { - fileDest = path.join(cafsDir, fileDest) + fileDest = path.join(storeDir, fileDest) if (locker.has(fileDest)) { return { checkedAt: locker.get(fileDest)!, diff --git a/store/cafs/test/writeBufferToCafs.test.ts b/store/cafs/test/writeBufferToCafs.test.ts index f9526cbc4ed..1e63dcbd787 100644 --- a/store/cafs/test/writeBufferToCafs.test.ts +++ b/store/cafs/test/writeBufferToCafs.test.ts @@ -6,12 +6,12 @@ import { pathTemp, writeBufferToCafs } from '../src/writeBufferToCafs' describe('writeBufferToCafs', () => { it('should not fail if a file already exists at the temp file location', () => { - const cafsDir = tempy.directory() + const storeDir = tempy.directory() const fileDest = 'abc' const buffer = Buffer.from('abc') - const fullFileDest = path.join(cafsDir, fileDest) + const fullFileDest = path.join(storeDir, fileDest) fs.writeFileSync(pathTemp(fullFileDest), 'ccc', 'utf8') - writeBufferToCafs(new Map(), cafsDir, buffer, fileDest, 420, ssri.fromData(buffer)) + writeBufferToCafs(new Map(), storeDir, buffer, fileDest, 420, ssri.fromData(buffer)) expect(fs.readFileSync(fullFileDest, 'utf8')).toBe('abc') }) }) diff --git a/store/create-cafs-store/src/index.ts b/store/create-cafs-store/src/index.ts index f989a88a268..c1356333bc8 100644 --- a/store/create-cafs-store/src/index.ts +++ b/store/create-cafs-store/src/index.ts @@ -5,14 +5,13 @@ import { createCafs, getFilePathByModeInCafs, } from '@pnpm/store.cafs' -import type { Cafs, PackageFilesResponse } from '@pnpm/cafs-types' +import { type Cafs, type PackageFilesResponse, type PackageFiles, type SideEffectsDiff } from '@pnpm/cafs-types' import { createIndexedPkgImporter } from '@pnpm/fs.indexed-pkg-importer' import { type ImportIndexedPackage, type ImportIndexedPackageAsync, type ImportPackageFunction, type ImportPackageFunctionAsync, - type PackageFileInfo, } from '@pnpm/store-controller-types' import memoize from 'mem' import pathTemp from 'path-temp' @@ -24,14 +23,14 @@ export function createPackageImporterAsync ( opts: { importIndexedPackage?: ImportIndexedPackageAsync packageImportMethod?: 'auto' | 'hardlink' | 'copy' | 'clone' | 'clone-or-copy' - cafsDir: string + storeDir: string } ): ImportPackageFunctionAsync { const cachedImporterCreator = opts.importIndexedPackage ? () => opts.importIndexedPackage! : memoize(createIndexedPkgImporter) const packageImportMethod = opts.packageImportMethod - const gfm = getFlatMap.bind(null, opts.cafsDir) + const gfm = getFlatMap.bind(null, opts.storeDir) return async (to, opts) => { const { filesMap, isBuilt } = gfm(opts.filesResponse, opts.sideEffectsCacheKey) const willBeBuilt = !isBuilt && opts.requiresBuild @@ -54,14 +53,14 @@ function createPackageImporter ( opts: { importIndexedPackage?: ImportIndexedPackage packageImportMethod?: 'auto' | 'hardlink' | 'copy' | 'clone' | 'clone-or-copy' - cafsDir: string + storeDir: string } ): ImportPackageFunction { const cachedImporterCreator = opts.importIndexedPackage ? () => opts.importIndexedPackage! : memoize(createIndexedPkgImporter) const packageImportMethod = opts.packageImportMethod - const gfm = getFlatMap.bind(null, opts.cafsDir) + const gfm = getFlatMap.bind(null, opts.storeDir) return (to, opts) => { const { filesMap, isBuilt } = gfm(opts.filesResponse, opts.sideEffectsCacheKey) const willBeBuilt = !isBuilt && opts.requiresBuild @@ -81,14 +80,14 @@ function createPackageImporter ( } function getFlatMap ( - cafsDir: string, + storeDir: string, filesResponse: PackageFilesResponse, targetEngine?: string ): { filesMap: Record, isBuilt: boolean } { let isBuilt!: boolean - let filesIndex!: Record + let filesIndex!: PackageFiles if (targetEngine && ((filesResponse.sideEffects?.[targetEngine]) != null)) { - filesIndex = filesResponse.sideEffects?.[targetEngine] + filesIndex = applySideEffectsDiff(filesResponse.filesIndex as PackageFiles, filesResponse.sideEffects?.[targetEngine]) isBuilt = true } else if (!filesResponse.unprocessed) { return { @@ -99,10 +98,20 @@ function getFlatMap ( filesIndex = filesResponse.filesIndex isBuilt = false } - const filesMap = mapValues(({ integrity, mode }) => getFilePathByModeInCafs(cafsDir, integrity, mode), filesIndex) + const filesMap = mapValues(({ integrity, mode }) => getFilePathByModeInCafs(storeDir, integrity, mode), filesIndex) return { filesMap, isBuilt } } +function applySideEffectsDiff (baseFiles: PackageFiles, { added, deleted }: SideEffectsDiff): PackageFiles { + const filesWithSideEffects: PackageFiles = { ...added } + for (const fileName in baseFiles) { + if (!deleted?.includes(fileName) && !filesWithSideEffects[fileName]) { + filesWithSideEffects[fileName] = baseFiles[fileName] + } + } + return filesWithSideEffects +} + export function createCafsStore ( storeDir: string, opts?: { @@ -112,16 +121,15 @@ export function createCafsStore ( cafsLocker?: CafsLocker } ): Cafs { - const cafsDir = path.join(storeDir, 'files') const baseTempDir = path.join(storeDir, 'tmp') const importPackage = createPackageImporter({ importIndexedPackage: opts?.importPackage, packageImportMethod: opts?.packageImportMethod, - cafsDir, + storeDir, }) return { - ...createCafs(cafsDir, opts), - cafsDir, + ...createCafs(storeDir, opts), + storeDir, importPackage, tempDir: async () => { const tmpDir = pathTemp(baseTempDir) diff --git a/store/package-store/src/storeController/index.ts b/store/package-store/src/storeController/index.ts index 238b9e4c3e9..b8f0dcf9394 100644 --- a/store/package-store/src/storeController/index.ts +++ b/store/package-store/src/storeController/index.ts @@ -1,3 +1,5 @@ +import path from 'path' +import fs from 'fs' import { createCafsStore, createPackageImporterAsync, type CafsLocker } from '@pnpm/create-cafs-store' import { type Fetchers } from '@pnpm/fetcher-base' import { createPackageRequester } from '@pnpm/package-requester' @@ -6,7 +8,7 @@ import { type ImportIndexedPackageAsync, type StoreController, } from '@pnpm/store-controller-types' -import { addFilesFromDir, importPackage } from '@pnpm/worker' +import { addFilesFromDir, importPackage, initStoreDir } from '@pnpm/worker' import { prune } from './prune' export { type CafsLocker } @@ -33,6 +35,9 @@ export function createPackageStore ( } ): StoreController { const storeDir = initOpts.storeDir + if (!fs.existsSync(path.join(storeDir, 'files'))) { + initStoreDir(storeDir).catch() // eslint-disable-line @typescript-eslint/no-floating-promises + } const cafs = createCafsStore(storeDir, { cafsLocker: initOpts.cafsLocker, packageImportMethod: initOpts.packageImportMethod, @@ -58,7 +63,7 @@ export function createPackageStore ( fetchPackage: packageRequester.fetchPackageToStore, getFilesIndexFilePath: packageRequester.getFilesIndexFilePath, importPackage: initOpts.importPackage - ? createPackageImporterAsync({ importIndexedPackage: initOpts.importPackage, cafsDir: cafs.cafsDir }) + ? createPackageImporterAsync({ importIndexedPackage: initOpts.importPackage, storeDir: cafs.storeDir }) : (targetDir, opts) => importPackage({ ...opts, packageImportMethod: initOpts.packageImportMethod, @@ -73,7 +78,7 @@ export function createPackageStore ( async function upload (builtPkgLocation: string, opts: { filesIndexFile: string, sideEffectsCacheKey: string }): Promise { await addFilesFromDir({ - cafsDir: cafs.cafsDir, + storeDir: cafs.storeDir, dir: builtPkgLocation, sideEffectsCacheKey: opts.sideEffectsCacheKey, filesIndexFile: opts.filesIndexFile, diff --git a/store/package-store/src/storeController/prune.ts b/store/package-store/src/storeController/prune.ts index 9625dfcb326..7d555e43ea3 100644 --- a/store/package-store/src/storeController/prune.ts +++ b/store/package-store/src/storeController/prune.ts @@ -25,6 +25,16 @@ export async function prune ({ cacheDir, storeDir }: PruneOptions, removeAlienFi await rimraf(path.join(storeDir, 'tmp')) globalInfo('Removed all cached metadata files') const pkgIndexFiles = [] as string[] + const indexDir = path.join(storeDir, 'index') + await Promise.all((await getSubdirsSafely(indexDir)).map(async (dir) => { + const subdir = path.join(indexDir, dir) + await Promise.all((await fs.readdir(subdir)).map(async (fileName) => { + const filePath = path.join(subdir, fileName) + if (fileName.endsWith('.json')) { + pkgIndexFiles.push(filePath) + } + })) + })) const removedHashes = new Set() const dirs = await getSubdirsSafely(cafsDir) let fileCounter = 0 @@ -32,7 +42,7 @@ export async function prune ({ cacheDir, storeDir }: PruneOptions, removeAlienFi const subdir = path.join(cafsDir, dir) await Promise.all((await fs.readdir(subdir)).map(async (fileName) => { const filePath = path.join(subdir, fileName) - if (fileName.endsWith('-index.json')) { + if (fileName.endsWith('.json')) { pkgIndexFiles.push(filePath) return } diff --git a/store/plugin-commands-store-inspecting/src/catIndex.ts b/store/plugin-commands-store-inspecting/src/catIndex.ts index fccf9026f44..2ce4cb4afae 100644 --- a/store/plugin-commands-store-inspecting/src/catIndex.ts +++ b/store/plugin-commands-store-inspecting/src/catIndex.ts @@ -1,5 +1,3 @@ -import path from 'path' - import { type Config } from '@pnpm/config' import { createResolver } from '@pnpm/client' import { type TarballResolution } from '@pnpm/lockfile.types' @@ -68,7 +66,6 @@ export async function handler (opts: CatIndexCommandOptions, params: string[]): storePath: opts.storeDir, pnpmHomeDir: opts.pnpmHomeDir, }) - const cafsDir = path.join(storeDir, 'files') const { resolve } = createResolver({ ...opts, authConfig: opts.rawConfig, @@ -84,8 +81,9 @@ export async function handler (opts: CatIndexCommandOptions, params: string[]): ) const filesIndexFile = getIndexFilePathInCafs( - cafsDir, - (pkgSnapshot.resolution as TarballResolution).integrity!.toString() + storeDir, + (pkgSnapshot.resolution as TarballResolution).integrity!.toString(), + `${alias}@${pref}` ) try { const pkgFilesIndex = await loadJsonFile(filesIndexFile) diff --git a/store/plugin-commands-store-inspecting/src/findHash.ts b/store/plugin-commands-store-inspecting/src/findHash.ts index b4b7cd6b805..952d3559d6f 100644 --- a/store/plugin-commands-store-inspecting/src/findHash.ts +++ b/store/plugin-commands-store-inspecting/src/findHash.ts @@ -48,15 +48,15 @@ export async function handler (opts: FindHashCommandOptions, params: string[]): storePath: opts.storeDir, pnpmHomeDir: opts.pnpmHomeDir, }) - const cafsDir = path.join(storeDir, 'files') - const cafsChildrenDirs = fs.readdirSync(cafsDir, { withFileTypes: true }).filter(file => file.isDirectory()) + const indexDir = path.join(storeDir, 'index') + const cafsChildrenDirs = fs.readdirSync(indexDir, { withFileTypes: true }).filter(file => file.isDirectory()) const indexFiles: string[] = []; const result: FindHashResult[] = [] for (const { name: dirName } of cafsChildrenDirs) { const dirIndexFiles = fs - .readdirSync(`${cafsDir}/${dirName}`) - .filter((fileName) => fileName.includes('-index.json')) - ?.map((fileName) => `${cafsDir}/${dirName}/${fileName}`) + .readdirSync(`${indexDir}/${dirName}`) + .filter((fileName) => fileName.includes('.json')) + ?.map((fileName) => `${indexDir}/${dirName}/${fileName}`) indexFiles.push(...dirIndexFiles) } @@ -66,7 +66,7 @@ export async function handler (opts: FindHashCommandOptions, params: string[]): for (const [, file] of Object.entries(pkgFilesIndex.files)) { if (file?.integrity === hash) { - result.push({ name: pkgFilesIndex.name ?? 'unknown', version: pkgFilesIndex?.version ?? 'unknown', filesIndexFile: filesIndexFile.replace(cafsDir, '') }) + result.push({ name: pkgFilesIndex.name ?? 'unknown', version: pkgFilesIndex?.version ?? 'unknown', filesIndexFile: filesIndexFile.replace(indexDir, '') }) // a package is only found once. continue @@ -74,10 +74,11 @@ export async function handler (opts: FindHashCommandOptions, params: string[]): } if (pkgFilesIndex?.sideEffects) { - for (const [, files] of Object.entries(pkgFilesIndex.sideEffects)) { - for (const [, file] of Object.entries(files)) { + for (const { added } of Object.values(pkgFilesIndex.sideEffects)) { + if (!added) continue + for (const file of Object.values(added)) { if (file?.integrity === hash) { - result.push({ name: pkgFilesIndex.name ?? 'unknown', version: pkgFilesIndex?.version ?? 'unknown', filesIndexFile: filesIndexFile.replace(cafsDir, '') }) + result.push({ name: pkgFilesIndex.name ?? 'unknown', version: pkgFilesIndex?.version ?? 'unknown', filesIndexFile: filesIndexFile.replace(indexDir, '') }) // a package is only found once. continue diff --git a/store/plugin-commands-store-inspecting/test/findHash.ts b/store/plugin-commands-store-inspecting/test/findHash.ts index d8764b05c8d..54361b1097f 100644 --- a/store/plugin-commands-store-inspecting/test/findHash.ts +++ b/store/plugin-commands-store-inspecting/test/findHash.ts @@ -26,8 +26,8 @@ test('print index file path with hash', async () => { storeDir, }, ['sha512-fXs1pWlUdqT2jkeoEJW/+odKZ2NwAyYkWea+plJKZI2xmhRKQi2e+nKGcClyDblgLwCLD912oMaua0+sTwwIrw==']) - expect(output).toBe(`${PACKAGE_INFO_CLR('lodash')}@${PACKAGE_INFO_CLR('4.17.19')} ${INDEX_PATH_CLR('/24/dbddf17111f46417d2fdaa260b1a37f9b3142340e4145efe3f0937d77eb56c862d2a1d2901ca16271dc0d6335b0237c2346768a3ec1a3d579018f1fc5f7a0d-index.json')} -${PACKAGE_INFO_CLR('lodash')}@${PACKAGE_INFO_CLR('4.17.20')} ${INDEX_PATH_CLR('/3e/585d15c8a594e20d7de57b362ea81754c011acb2641a19f1b72c8531ea39825896bab344ae616a0a5a824cb9a381df0b3cddd534645cf305aba70a93dac698-index.json')} + expect(output).toBe(`${PACKAGE_INFO_CLR('lodash')}@${PACKAGE_INFO_CLR('4.17.19')} ${INDEX_PATH_CLR('/24/dbddf17111f46417d2fdaa260b1a37f9b3142340e4145efe3f0937d77eb56c-lodash@4.17.19.json')} +${PACKAGE_INFO_CLR('lodash')}@${PACKAGE_INFO_CLR('4.17.20')} ${INDEX_PATH_CLR('/3e/585d15c8a594e20d7de57b362ea81754c011acb2641a19f1b72c8531ea3982-lodash@4.17.20.json')} `) } }) @@ -49,4 +49,4 @@ test('print index file path with hash error', async () => { expect(err.code).toBe('ERR_PNPM_INVALID_FILE_HASH') expect(err.message).toBe('No package or index file matching this hash was found.') -}) \ No newline at end of file +}) diff --git a/store/plugin-commands-store/package.json b/store/plugin-commands-store/package.json index c611ddfc17d..4bf5e8e569e 100644 --- a/store/plugin-commands-store/package.json +++ b/store/plugin-commands-store/package.json @@ -31,6 +31,7 @@ "homepage": "https://github.com/pnpm/pnpm/blob/main/store/plugin-commands-store#readme", "devDependencies": { "@pnpm/assert-store": "workspace:*", + "@pnpm/constants": "workspace:*", "@pnpm/lockfile.fs": "workspace:*", "@pnpm/logger": "workspace:*", "@pnpm/plugin-commands-script-runners": "workspace:*", diff --git a/store/plugin-commands-store/src/storeStatus/index.ts b/store/plugin-commands-store/src/storeStatus/index.ts index 9c6194a04a8..81b577b5c72 100644 --- a/store/plugin-commands-store/src/storeStatus/index.ts +++ b/store/plugin-commands-store/src/storeStatus/index.ts @@ -48,10 +48,9 @@ export async function storeStatus (maybeOpts: StoreStatusOptions): Promise { const pkgIndexFilePath = integrity - ? getIndexFilePathInCafs(cafsDir, integrity) + ? getIndexFilePathInCafs(storeDir, integrity, id) : path.join(storeDir, dp.depPathToFilename(id, maybeOpts.virtualStoreDirMaxLength), 'integrity.json') const { files } = await loadJsonFile(pkgIndexFilePath) return (await dint.check(path.join(virtualStoreDir, dp.depPathToFilename(depPath, maybeOpts.virtualStoreDirMaxLength), 'node_modules', name), files)) === false diff --git a/store/plugin-commands-store/test/storeAdd.ts b/store/plugin-commands-store/test/storeAdd.ts index 73a75e071b4..f7b50ffb8a0 100644 --- a/store/plugin-commands-store/test/storeAdd.ts +++ b/store/plugin-commands-store/test/storeAdd.ts @@ -1,12 +1,11 @@ import fs from 'fs' import path from 'path' import { assertStore } from '@pnpm/assert-store' +import { STORE_VERSION } from '@pnpm/constants' import { store } from '@pnpm/plugin-commands-store' import { tempDir } from '@pnpm/prepare' import { REGISTRY_MOCK_PORT } from '@pnpm/registry-mock' -const STORE_VERSION = 'v3' - test('pnpm store add express@4.16.3', async () => { tempDir() @@ -24,11 +23,11 @@ test('pnpm store add express@4.16.3', async () => { storeDir, userConfig: {}, dlxCacheMaxAge: 0, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, ['add', 'express@4.16.3']) const { cafsHas } = assertStore(path.join(storeDir, STORE_VERSION)) - cafsHas('sha512-CDaOBMB9knI6vx9SpIxEMOJ6VBbC2U/tYNILs0qv1YOZc15K9U2EcF06v10F0JX6IYcWnKYZJwIDJspEHLvUaQ==') + cafsHas('express', '4.16.3') }) test('pnpm store add scoped package that uses not the standard registry', async () => { @@ -51,7 +50,7 @@ test('pnpm store add scoped package that uses not the standard registry', async storeDir, userConfig: {}, dlxCacheMaxAge: 0, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, ['add', '@foo/no-deps@1.0.0']) const { cafsHas } = assertStore(path.join(storeDir, STORE_VERSION)) @@ -81,7 +80,7 @@ test('should fail if some packages can not be added', async () => { storeDir, userConfig: {}, dlxCacheMaxAge: 0, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, ['add', '@pnpm/this-does-not-exist']) } catch (e: any) { // eslint-disable-line thrown = true diff --git a/store/plugin-commands-store/test/storePath.ts b/store/plugin-commands-store/test/storePath.ts index 422d45d50c6..118818eb2ae 100644 --- a/store/plugin-commands-store/test/storePath.ts +++ b/store/plugin-commands-store/test/storePath.ts @@ -1,5 +1,6 @@ import os from 'os' import path from 'path' +import { STORE_VERSION } from '@pnpm/constants' import { store } from '@pnpm/plugin-commands-store' import { prepare } from '@pnpm/prepare' import { REGISTRY_MOCK_PORT } from '@pnpm/registry-mock' @@ -20,12 +21,12 @@ test('CLI prints the current store path', async () => { storeDir: '/home/example/.pnpm-store', userConfig: {}, dlxCacheMaxAge: 0, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, ['path']) const expectedStorePath = os.platform() === 'win32' - ? '\\home\\example\\.pnpm-store\\v3' - : '/home/example/.pnpm-store/v3' + ? `\\home\\example\\.pnpm-store\\${STORE_VERSION}` + : `/home/example/.pnpm-store/${STORE_VERSION}` expect(candidateStorePath).toBe(expectedStorePath) }) diff --git a/store/plugin-commands-store/test/storePrune.ts b/store/plugin-commands-store/test/storePrune.ts index b64b6fa807f..be79b439890 100644 --- a/store/plugin-commands-store/test/storePrune.ts +++ b/store/plugin-commands-store/test/storePrune.ts @@ -1,15 +1,14 @@ import fs from 'fs' import path from 'path' import { assertStore } from '@pnpm/assert-store' +import { STORE_VERSION } from '@pnpm/constants' import { dlx } from '@pnpm/plugin-commands-script-runners' import { store } from '@pnpm/plugin-commands-store' import { prepare, prepareEmpty } from '@pnpm/prepare' import { REGISTRY_MOCK_PORT } from '@pnpm/registry-mock' import { sync as rimraf } from '@zkochan/rimraf' import execa from 'execa' -import ssri from 'ssri' -const STORE_VERSION = 'v3' const REGISTRY = `http://localhost:${REGISTRY_MOCK_PORT}/` const pnpmBin = path.join(__dirname, '../../../pnpm/bin/pnpm.cjs') @@ -51,7 +50,7 @@ test('remove unreferenced packages', async () => { storeDir, userConfig: {}, dlxCacheMaxAge: Infinity, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, ['prune']) expect(reporter).toHaveBeenCalledWith( @@ -76,7 +75,7 @@ test('remove unreferenced packages', async () => { storeDir, userConfig: {}, dlxCacheMaxAge: Infinity, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, ['prune']) expect(reporter).not.toHaveBeenCalledWith( @@ -98,7 +97,7 @@ test.skip('remove packages that are used by project that no longer exist', async rimraf('node_modules') - cafsHas(ssri.fromHex('f0d86377aa15a64c34961f38ac2a9be2b40a1187', 'sha1').toString()) + cafsHas('is-negative', '2.1.0') const reporter = jest.fn() await store.handler({ @@ -113,7 +112,7 @@ test.skip('remove packages that are used by project that no longer exist', async storeDir, userConfig: {}, dlxCacheMaxAge: Infinity, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, ['prune']) expect(reporter).toHaveBeenCalledWith( @@ -123,7 +122,7 @@ test.skip('remove packages that are used by project that no longer exist', async }) ) - cafsHasNot(ssri.fromHex('f0d86377aa15a64c34961f38ac2a9be2b40a1187', 'sha1').toString()) + cafsHasNot('is-negative', '2.1.0') }) test('keep dependencies used by others', async () => { @@ -153,7 +152,7 @@ test('keep dependencies used by others', async () => { storeDir, userConfig: {}, dlxCacheMaxAge: Infinity, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, ['prune']) project.storeHasNot('camelcase-keys', '3.0.0') @@ -179,7 +178,7 @@ test('keep dependency used by package', async () => { storeDir, userConfig: {}, dlxCacheMaxAge: Infinity, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, ['prune']) project.storeHas('is-positive', '3.1.0') @@ -203,7 +202,7 @@ test('prune will skip scanning non-directory in storeDir', async () => { storeDir, userConfig: {}, dlxCacheMaxAge: Infinity, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, ['prune']) }) @@ -215,7 +214,7 @@ test('prune does not fail if the store contains an unexpected directory', async await execa('node', [pnpmBin, 'add', 'is-negative@2.1.0', '--store-dir', storeDir, '--registry', REGISTRY]) project.storeHas('is-negative', '2.1.0') - const alienDir = path.join(storeDir, 'v3/files/44/directory') + const alienDir = path.join(storeDir, STORE_VERSION, 'files/44/directory') fs.mkdirSync(alienDir) const reporter = jest.fn() @@ -231,7 +230,7 @@ test('prune does not fail if the store contains an unexpected directory', async storeDir, userConfig: {}, dlxCacheMaxAge: Infinity, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, ['prune']) expect(reporter).toHaveBeenCalledWith( @@ -253,7 +252,7 @@ test('prune removes alien files from the store if the --force flag is used', asy await execa('node', [pnpmBin, 'add', 'is-negative@2.1.0', '--store-dir', storeDir, '--registry', REGISTRY]) project.storeHas('is-negative', '2.1.0') - const alienDir = path.join(storeDir, 'v3/files/44/directory') + const alienDir = path.join(storeDir, STORE_VERSION, 'files/44/directory') fs.mkdirSync(alienDir) const reporter = jest.fn() @@ -270,7 +269,7 @@ test('prune removes alien files from the store if the --force flag is used', asy userConfig: {}, force: true, dlxCacheMaxAge: Infinity, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, ['prune']) expect(reporter).toHaveBeenCalledWith( expect.objectContaining({ @@ -374,8 +373,8 @@ test('prune removes cache directories that outlives dlx-cache-max-age', async () const cacheDir = path.resolve('cache') const storeDir = path.resolve('store') - fs.mkdirSync(path.join(storeDir, 'v3', 'files'), { recursive: true }) - fs.mkdirSync(path.join(storeDir, 'v3', 'tmp'), { recursive: true }) + fs.mkdirSync(path.join(storeDir, STORE_VERSION, 'files'), { recursive: true }) + fs.mkdirSync(path.join(storeDir, STORE_VERSION, 'tmp'), { recursive: true }) const now = new Date() @@ -397,7 +396,7 @@ test('prune removes cache directories that outlives dlx-cache-max-age', async () storeDir, userConfig: {}, dlxCacheMaxAge: 7, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, ['prune']) expect( diff --git a/store/plugin-commands-store/test/storeStatus.ts b/store/plugin-commands-store/test/storeStatus.ts index 6684510f39d..abc82f4d084 100644 --- a/store/plugin-commands-store/test/storeStatus.ts +++ b/store/plugin-commands-store/test/storeStatus.ts @@ -41,7 +41,7 @@ test('CLI fails when store status finds modified packages', async () => { storeDir, userConfig: {}, dlxCacheMaxAge: 0, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, ['status']) } catch (_err: any) { // eslint-disable-line err = _err @@ -94,6 +94,6 @@ test('CLI does not fail when store status does not find modified packages', asyn storeDir, userConfig: {}, dlxCacheMaxAge: 0, - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, }, ['status']) }) diff --git a/store/plugin-commands-store/tsconfig.json b/store/plugin-commands-store/tsconfig.json index 8e1bfe6efae..4bcc2164030 100644 --- a/store/plugin-commands-store/tsconfig.json +++ b/store/plugin-commands-store/tsconfig.json @@ -36,6 +36,9 @@ { "path": "../../lockfile/utils" }, + { + "path": "../../packages/constants" + }, { "path": "../../packages/dependency-path" }, diff --git a/store/server/test/index.ts b/store/server/test/index.ts index c146f97667f..f5878e7e053 100644 --- a/store/server/test/index.ts +++ b/store/server/test/index.ts @@ -164,7 +164,13 @@ test('server upload', async () => { const storeCtrl = await connectStoreController({ remotePrefix, concurrency: 100 }) const fakeEngine = 'client-engine' - const filesIndexFile = path.join(storeDir, 'test.example.com/fake-pkg/1.0.0.json') + const filesIndexFile = path.join(storeDir, 'fake-pkg@1.0.0.json') + + fs.writeFileSync(filesIndexFile, JSON.stringify({ + name: 'fake-pkg', + version: '1.0.0', + files: {}, + }), 'utf8') await storeCtrl.upload(path.join(__dirname, '__fixtures__/side-effect-fake-dir'), { sideEffectsCacheKey: fakeEngine, @@ -172,7 +178,7 @@ test('server upload', async () => { }) const cacheIntegrity = loadJsonFile.sync(filesIndexFile) // eslint-disable-line @typescript-eslint/no-explicit-any - expect(Object.keys(cacheIntegrity?.['sideEffects'][fakeEngine]).sort()).toStrictEqual(['side-effect.js', 'side-effect.txt']) + expect(Object.keys(cacheIntegrity?.['sideEffects'][fakeEngine].added).sort()).toStrictEqual(['side-effect.js', 'side-effect.txt']) await server.close() await storeCtrl.close() diff --git a/store/store-path/package.json b/store/store-path/package.json index 63d1e18af1b..85c1380f7e7 100644 --- a/store/store-path/package.json +++ b/store/store-path/package.json @@ -30,6 +30,7 @@ }, "homepage": "https://github.com/pnpm/pnpm/blob/main/store/store-path#readme", "dependencies": { + "@pnpm/constants": "workspace:*", "@pnpm/error": "workspace:*", "@zkochan/rimraf": "catalog:", "can-link": "catalog:", diff --git a/store/store-path/src/index.ts b/store/store-path/src/index.ts index 179647b215e..e6993502a13 100644 --- a/store/store-path/src/index.ts +++ b/store/store-path/src/index.ts @@ -1,4 +1,5 @@ import { promises as fs } from 'fs' +import { STORE_VERSION } from '@pnpm/constants' import { PnpmError } from '@pnpm/error' import rimraf from '@zkochan/rimraf' import canLink from 'can-link' @@ -9,8 +10,6 @@ import pathTemp from 'path-temp' import rootLinkTarget from 'root-link-target' import touch from 'touch' -const STORE_VERSION = 'v3' - export function getStorePath ( { pkgRoot, diff --git a/store/store-path/test/index.ts b/store/store-path/test/index.ts index 14259175ee2..8209df6cc36 100644 --- a/store/store-path/test/index.ts +++ b/store/store-path/test/index.ts @@ -1,3 +1,4 @@ +import { STORE_VERSION } from '@pnpm/constants' import { getStorePath } from '@pnpm/store-path' import isWindows from 'is-windows' @@ -10,21 +11,21 @@ skipOnWindows('when a link can be created to the homedir', async () => { expect(await getStorePath({ pkgRoot: '/can-link-to-homedir', pnpmHomeDir: '/local/share/pnpm', - })).toBe('/local/share/pnpm/store/v3') + })).toBe(`/local/share/pnpm/store/${STORE_VERSION}`) }) skipOnWindows('a link can be created to the root of the drive', async () => { expect(await getStorePath({ pkgRoot: '/src/workspace/project', pnpmHomeDir: '/local/share/pnpm', - })).toBe('/.pnpm-store/v3') + })).toBe(`/.pnpm-store/${STORE_VERSION}`) }) skipOnWindows('a link can be created to the a subdir in the root of the drive', async () => { expect(await getStorePath({ pkgRoot: '/mnt/project', pnpmHomeDir: '/local/share/pnpm', - })).toBe('/mnt/.pnpm-store/v3') + })).toBe(`/mnt/.pnpm-store/${STORE_VERSION}`) }) test('fail when pnpm home directory is not defined', async () => { diff --git a/store/store-path/tsconfig.json b/store/store-path/tsconfig.json index 019cba19e73..5eeb7b4a57c 100644 --- a/store/store-path/tsconfig.json +++ b/store/store-path/tsconfig.json @@ -9,6 +9,9 @@ "../../__typings__/**/*.d.ts" ], "references": [ + { + "path": "../../packages/constants" + }, { "path": "../../packages/error" } diff --git a/tools/plugin-commands-self-updater/test/selfUpdate.test.ts b/tools/plugin-commands-self-updater/test/selfUpdate.test.ts index 7460bdd1edb..d3d5dfd0135 100644 --- a/tools/plugin-commands-self-updater/test/selfUpdate.test.ts +++ b/tools/plugin-commands-self-updater/test/selfUpdate.test.ts @@ -53,7 +53,7 @@ function prepareOptions (dir: string) { pnpmfile: '', rawConfig: {}, cacheDir: path.join(dir, '.cache'), - virtualStoreDirMaxLength: 120, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, dir: process.cwd(), managePackageManagerVersions: false, } diff --git a/worker/src/index.ts b/worker/src/index.ts index ec4ddaaa877..0bddcc6252d 100644 --- a/worker/src/index.ts +++ b/worker/src/index.ts @@ -54,7 +54,7 @@ interface AddFilesResult { requiresBuild: boolean } -type AddFilesFromDirOptions = Pick +type AddFilesFromDirOptions = Pick export async function addFilesFromDir (opts: AddFilesFromDirOptions): Promise { if (!workerPool) { @@ -72,7 +72,7 @@ export async function addFilesFromDir (opts: AddFilesFromDirOptions): Promise & { +type AddFilesFromTarballOptions = Pick & { url: string } @@ -145,7 +145,7 @@ export async function addFilesFromTarball (opts: AddFilesFromTarballOptions): Pr localWorker.postMessage({ type: 'extract', buffer: opts.buffer, - cafsDir: opts.cafsDir, + storeDir: opts.storeDir, integrity: opts.integrity, filesIndexFile: opts.filesIndexFile, readManifest: opts.readManifest, @@ -155,7 +155,7 @@ export async function addFilesFromTarball (opts: AddFilesFromTarballOptions): Pr } export async function readPkgFromCafs ( - cafsDir: string, + storeDir: string, verifyStoreIntegrity: boolean, filesIndexFile: string, readManifest?: boolean @@ -175,7 +175,7 @@ export async function readPkgFromCafs ( }) localWorker.postMessage({ type: 'readPkgFromCafs', - cafsDir, + storeDir, filesIndexFile, readManifest, verifyStoreIntegrity, @@ -250,3 +250,24 @@ export async function hardLinkDir (src: string, destDirs: string[]): Promise { + if (!workerPool) { + workerPool = createTarballWorkerPool() + } + const localWorker = await workerPool.checkoutWorkerAsync(true) + return new Promise((resolve, reject) => { + localWorker.once('message', ({ status, error }) => { + workerPool!.checkinWorker(localWorker) + if (status === 'error') { + reject(new PnpmError(error.code ?? 'INIT_CAFS_FAILED', error.message as string)) + return + } + resolve() + }) + localWorker.postMessage({ + type: 'init-store', + storeDir, + }) + }) +} diff --git a/worker/src/types.ts b/worker/src/types.ts index 0bcd5ba43c4..43a0a875139 100644 --- a/worker/src/types.ts +++ b/worker/src/types.ts @@ -5,10 +5,15 @@ export interface PkgNameVersion { version?: string } +export interface InitStoreMessage { + type: 'init-store' + storeDir: string +} + export interface TarballExtractMessage { type: 'extract' buffer: Buffer - cafsDir: string + storeDir: string integrity?: string filesIndexFile: string readManifest?: boolean @@ -39,7 +44,7 @@ export interface SymlinkAllModulesMessage { export interface AddDirToStoreMessage { type: 'add-dir' - cafsDir: string + storeDir: string dir: string filesIndexFile: string sideEffectsCacheKey?: string @@ -50,7 +55,7 @@ export interface AddDirToStoreMessage { export interface ReadPkgFromCafsMessage { type: 'readPkgFromCafs' - cafsDir: string + storeDir: string filesIndexFile: string readManifest: boolean verifyStoreIntegrity: boolean diff --git a/worker/src/worker.ts b/worker/src/worker.ts index 15daef9dc5b..010cd2ac01a 100644 --- a/worker/src/worker.ts +++ b/worker/src/worker.ts @@ -1,7 +1,7 @@ import path from 'path' import fs from 'fs' import gfs from '@pnpm/graceful-fs' -import { type Cafs } from '@pnpm/cafs-types' +import { type Cafs, type PackageFiles, type SideEffects, type SideEffectsDiff } from '@pnpm/cafs-types' import { createCafsStore } from '@pnpm/create-cafs-store' import * as crypto from '@pnpm/crypto.polyfill' import { pkgRequiresBuild } from '@pnpm/exec.pkg-requires-build' @@ -10,9 +10,7 @@ import { type CafsFunctions, checkPkgFilesIntegrity, createCafs, - type PackageFileInfo, type PackageFilesIndex, - type SideEffects, type FilesIndex, optimisticRenameOverwrite, readManifestFromStore, @@ -29,6 +27,7 @@ import { type SymlinkAllModulesMessage, type TarballExtractMessage, type HardLinkDirMessage, + type InitStoreMessage, } from './types' const INTEGRITY_REGEX: RegExp = /^([^-]+)-([A-Za-z0-9+/=]+)$/ @@ -40,7 +39,15 @@ const cafsStoreCache = new Map() const cafsLocker = new Map() async function handleMessage ( - message: TarballExtractMessage | LinkPkgMessage | AddDirToStoreMessage | ReadPkgFromCafsMessage | SymlinkAllModulesMessage | HardLinkDirMessage | false + message: + | TarballExtractMessage + | LinkPkgMessage + | AddDirToStoreMessage + | ReadPkgFromCafsMessage + | SymlinkAllModulesMessage + | HardLinkDirMessage + | InitStoreMessage + | false ): Promise { if (message === false) { parentPort!.off('message', handleMessage) @@ -60,8 +67,12 @@ async function handleMessage ( parentPort!.postMessage(addFilesFromDir(message)) break } + case 'init-store': { + parentPort!.postMessage(initStore(message)) + break + } case 'readPkgFromCafs': { - let { cafsDir, filesIndexFile, readManifest, verifyStoreIntegrity } = message + let { storeDir, filesIndexFile, readManifest, verifyStoreIntegrity } = message let pkgFilesIndex: PackageFilesIndex | undefined try { pkgFilesIndex = loadJsonFile(filesIndexFile) @@ -83,11 +94,11 @@ async function handleMessage ( readManifest = true } if (verifyStoreIntegrity) { - verifyResult = checkPkgFilesIntegrity(cafsDir, pkgFilesIndex, readManifest) + verifyResult = checkPkgFilesIntegrity(storeDir, pkgFilesIndex, readManifest) } else { verifyResult = { passed: true, - manifest: readManifest ? readManifestFromStore(cafsDir, pkgFilesIndex) : undefined, + manifest: readManifest ? readManifestFromStore(storeDir, pkgFilesIndex) : undefined, } } const requiresBuild = pkgFilesIndex.requiresBuild ?? pkgRequiresBuild(verifyResult.manifest, pkgFilesIndex.files) @@ -123,7 +134,7 @@ async function handleMessage ( } } -function addTarballToStore ({ buffer, cafsDir, integrity, filesIndexFile }: TarballExtractMessage) { +function addTarballToStore ({ buffer, storeDir, integrity, filesIndexFile }: TarballExtractMessage) { if (integrity) { const [, algo, integrityHash] = integrity.match(INTEGRITY_REGEX)! // Compensate for the possibility of non-uniform Base64 padding @@ -142,10 +153,10 @@ function addTarballToStore ({ buffer, cafsDir, integrity, filesIndexFile }: Tarb } } } - if (!cafsCache.has(cafsDir)) { - cafsCache.set(cafsDir, createCafs(cafsDir)) + if (!cafsCache.has(storeDir)) { + cafsCache.set(storeDir, createCafs(storeDir)) } - const cafs = cafsCache.get(cafsDir)! + const cafs = cafsCache.get(storeDir)! const { filesIndex, manifest } = cafs.addFilesFromTarball(buffer, true) const { filesIntegrity, filesMap } = processFilesIndex(filesIndex) const requiresBuild = writeFilesIndexFile(filesIndexFile, { manifest: manifest ?? {}, files: filesIntegrity }) @@ -161,11 +172,31 @@ interface AddFilesFromDirResult { } } -function addFilesFromDir ({ dir, cafsDir, filesIndexFile, sideEffectsCacheKey, files }: AddDirToStoreMessage): AddFilesFromDirResult { - if (!cafsCache.has(cafsDir)) { - cafsCache.set(cafsDir, createCafs(cafsDir)) +function initStore ({ storeDir }: InitStoreMessage): { status: string } { + fs.mkdirSync(storeDir, { recursive: true }) + try { + const hexChars = '0123456789abcdef'.split('') + for (const subDir of ['files', 'index']) { + const subDirPath = path.join(storeDir, subDir) + fs.mkdirSync(subDirPath) + for (const hex1 of hexChars) { + for (const hex2 of hexChars) { + fs.mkdirSync(path.join(subDirPath, `${hex1}${hex2}`)) + } + } + } + } catch { + // If a parallel process has already started creating the directories in the store, + // then we just stop. } - const cafs = cafsCache.get(cafsDir)! + return { status: 'success' } +} + +function addFilesFromDir ({ dir, storeDir, filesIndexFile, sideEffectsCacheKey, files }: AddDirToStoreMessage): AddFilesFromDirResult { + if (!cafsCache.has(storeDir)) { + cafsCache.set(storeDir, createCafs(storeDir)) + } + const cafs = cafsCache.get(storeDir)! const { filesIndex, manifest } = cafs.addFilesFromDir(dir, { files, readManifest: true, @@ -177,10 +208,18 @@ function addFilesFromDir ({ dir, cafsDir, filesIndexFile, sideEffectsCacheKey, f try { filesIndex = loadJsonFile(filesIndexFile) } catch { - filesIndex = { name: manifest?.name, version: manifest?.version, files: filesIntegrity } + // If there is no existing index file, then we cannot store the side effects. + return { + status: 'success', + value: { + filesIndex: filesMap, + manifest, + requiresBuild: pkgRequiresBuild(manifest, filesIntegrity), + }, + } } filesIndex.sideEffects = filesIndex.sideEffects ?? {} - filesIndex.sideEffects[sideEffectsCacheKey] = filesIntegrity + filesIndex.sideEffects[sideEffectsCacheKey] = calculateDiff(filesIndex.files, filesIntegrity) if (filesIndex.requiresBuild == null) { requiresBuild = pkgRequiresBuild(manifest, filesIntegrity) } else { @@ -193,13 +232,37 @@ function addFilesFromDir ({ dir, cafsDir, filesIndexFile, sideEffectsCacheKey, f return { status: 'success', value: { filesIndex: filesMap, manifest, requiresBuild } } } +function calculateDiff (baseFiles: PackageFiles, sideEffectsFiles: PackageFiles): SideEffectsDiff { + const deleted: string[] = [] + const added: PackageFiles = {} + for (const file of new Set([...Object.keys(baseFiles), ...Object.keys(sideEffectsFiles)])) { + if (!sideEffectsFiles[file]) { + deleted.push(file) + } else if ( + !baseFiles[file] || + baseFiles[file].integrity !== sideEffectsFiles[file].integrity || + baseFiles[file].mode !== sideEffectsFiles[file].mode + ) { + added[file] = sideEffectsFiles[file] + } + } + const diff: SideEffectsDiff = {} + if (deleted.length > 0) { + diff.deleted = deleted + } + if (Object.keys(added).length > 0) { + diff.added = added + } + return diff +} + interface ProcessFilesIndexResult { - filesIntegrity: Record + filesIntegrity: PackageFiles filesMap: Record } function processFilesIndex (filesIndex: FilesIndex): ProcessFilesIndexResult { - const filesIntegrity: Record = {} + const filesIntegrity: PackageFiles = {} const filesMap: Record = {} for (const [k, { checkedAt, filePath, integrity, mode, size }] of Object.entries(filesIndex)) { filesIntegrity[k] = { @@ -263,7 +326,7 @@ function writeFilesIndexFile ( filesIndexFile: string, { manifest, files, sideEffects }: { manifest: Partial - files: Record + files: PackageFiles sideEffects?: SideEffects } ): boolean {