diff --git a/.pnp.cjs b/.pnp.cjs index d6e95d86513e..35e5e40f9746 100755 --- a/.pnp.cjs +++ b/.pnp.cjs @@ -4451,6 +4451,24 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD", }] ]], + ["@chevrotain/types", [ + ["npm:9.1.0", { + "packageLocation": "./.yarn/cache/@chevrotain-types-npm-9.1.0-80ac254cc2-5f26ff26aa.zip/node_modules/@chevrotain/types/", + "packageDependencies": [ + ["@chevrotain/types", "npm:9.1.0"] + ], + "linkType": "HARD", + }] + ]], + ["@chevrotain/utils", [ + ["npm:9.1.0", { + "packageLocation": "./.yarn/cache/@chevrotain-utils-npm-9.1.0-5e5d6d7acc-ca78c97c7c.zip/node_modules/@chevrotain/utils/", + "packageDependencies": [ + ["@chevrotain/utils", "npm:9.1.0"] + ], + "linkType": "HARD", + }] + ]], ["@cnakazawa/watch", [ ["npm:1.0.3", { "packageLocation": "./.yarn/cache/@cnakazawa-watch-npm-1.0.3-e2afda3405-c11ca927d9.zip/node_modules/@cnakazawa/watch/", @@ -10330,6 +10348,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["stream-to-promise", "npm:2.2.0"], ["strip-ansi", "npm:6.0.0"], ["tar", "npm:6.0.5"], + ["tinylogic", "npm:1.0.3"], ["treeify", "npm:1.1.0"], ["tslib", "npm:1.13.0"], ["tunnel", "npm:0.0.6"] @@ -18200,6 +18219,18 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD", }] ]], + ["chevrotain", [ + ["npm:9.1.0", { + "packageLocation": "./.yarn/cache/chevrotain-npm-9.1.0-9280f9d77f-632d0d7c69.zip/node_modules/chevrotain/", + "packageDependencies": [ + ["chevrotain", "npm:9.1.0"], + ["@chevrotain/types", "npm:9.1.0"], + ["@chevrotain/utils", "npm:9.1.0"], + ["regexp-to-ast", "npm:0.5.0"] + ], + "linkType": "HARD", + }] + ]], ["chokidar", [ ["npm:2.1.8", { "packageLocation": "./.yarn/cache/chokidar-npm-2.1.8-32fdcd020e-0c43e89cbf.zip/node_modules/chokidar/", @@ -35004,6 +35035,15 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD", }] ]], + ["regexp-to-ast", [ + ["npm:0.5.0", { + "packageLocation": "./.yarn/cache/regexp-to-ast-npm-0.5.0-1e96b9f3a0-72e32f2a12.zip/node_modules/regexp-to-ast/", + "packageDependencies": [ + ["regexp-to-ast", "npm:0.5.0"] + ], + "linkType": "HARD", + }] + ]], ["regexp.prototype.flags", [ ["npm:1.3.1", { "packageLocation": "./.yarn/cache/regexp.prototype.flags-npm-1.3.1-f0c34f894f-343595db5a.zip/node_modules/regexp.prototype.flags/", @@ -38157,6 +38197,16 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD", }] ]], + ["tinylogic", [ + ["npm:1.0.3", { + "packageLocation": "./.yarn/cache/tinylogic-npm-1.0.3-bd596a96c4-fdf7fcc170.zip/node_modules/tinylogic/", + "packageDependencies": [ + ["tinylogic", "npm:1.0.3"], + ["chevrotain", "npm:9.1.0"] + ], + "linkType": "HARD", + }] + ]], ["tmp", [ ["npm:0.0.29", { "packageLocation": "./.yarn/cache/tmp-npm-0.0.29-33768985a5-6caab5b666.zip/node_modules/tmp/", diff --git a/.yarn/cache/@chevrotain-types-npm-9.1.0-80ac254cc2-5f26ff26aa.zip b/.yarn/cache/@chevrotain-types-npm-9.1.0-80ac254cc2-5f26ff26aa.zip new file mode 100644 index 000000000000..9ce956cf8eac Binary files /dev/null and b/.yarn/cache/@chevrotain-types-npm-9.1.0-80ac254cc2-5f26ff26aa.zip differ diff --git a/.yarn/cache/@chevrotain-utils-npm-9.1.0-5e5d6d7acc-ca78c97c7c.zip b/.yarn/cache/@chevrotain-utils-npm-9.1.0-5e5d6d7acc-ca78c97c7c.zip new file mode 100644 index 000000000000..699df33ab5fc Binary files /dev/null and b/.yarn/cache/@chevrotain-utils-npm-9.1.0-5e5d6d7acc-ca78c97c7c.zip differ diff --git a/.yarn/cache/chevrotain-npm-9.1.0-9280f9d77f-632d0d7c69.zip b/.yarn/cache/chevrotain-npm-9.1.0-9280f9d77f-632d0d7c69.zip new file mode 100644 index 000000000000..25fc152c556f Binary files /dev/null and b/.yarn/cache/chevrotain-npm-9.1.0-9280f9d77f-632d0d7c69.zip differ diff --git a/.yarn/cache/fsevents-patch-3340e2eb10-edbd0fd80b.zip b/.yarn/cache/fsevents-patch-3340e2eb10-8.zip similarity index 100% rename from .yarn/cache/fsevents-patch-3340e2eb10-edbd0fd80b.zip rename to .yarn/cache/fsevents-patch-3340e2eb10-8.zip diff --git a/.yarn/cache/fsevents-patch-3fa09df81b-94a10c4d03.zip b/.yarn/cache/fsevents-patch-3fa09df81b-8.zip similarity index 100% rename from .yarn/cache/fsevents-patch-3fa09df81b-94a10c4d03.zip rename to .yarn/cache/fsevents-patch-3fa09df81b-8.zip diff --git a/.yarn/cache/regexp-to-ast-npm-0.5.0-1e96b9f3a0-72e32f2a12.zip b/.yarn/cache/regexp-to-ast-npm-0.5.0-1e96b9f3a0-72e32f2a12.zip new file mode 100644 index 000000000000..15fdfab61b88 Binary files /dev/null and b/.yarn/cache/regexp-to-ast-npm-0.5.0-1e96b9f3a0-72e32f2a12.zip differ diff --git a/.yarn/cache/tinylogic-npm-1.0.3-bd596a96c4-fdf7fcc170.zip b/.yarn/cache/tinylogic-npm-1.0.3-bd596a96c4-fdf7fcc170.zip new file mode 100644 index 000000000000..28b97ce38778 Binary files /dev/null and b/.yarn/cache/tinylogic-npm-1.0.3-bd596a96c4-fdf7fcc170.zip differ diff --git a/.yarn/versions/771de10e.yml b/.yarn/versions/771de10e.yml new file mode 100644 index 000000000000..0e553c73de99 --- /dev/null +++ b/.yarn/versions/771de10e.yml @@ -0,0 +1,34 @@ +releases: + "@yarnpkg/builder": minor + "@yarnpkg/cli": minor + "@yarnpkg/core": minor + "@yarnpkg/doctor": minor + "@yarnpkg/esbuild-plugin-pnp": minor + "@yarnpkg/nm": minor + "@yarnpkg/plugin-compat": minor + "@yarnpkg/plugin-constraints": minor + "@yarnpkg/plugin-dlx": minor + "@yarnpkg/plugin-essentials": minor + "@yarnpkg/plugin-exec": minor + "@yarnpkg/plugin-file": minor + "@yarnpkg/plugin-git": minor + "@yarnpkg/plugin-github": minor + "@yarnpkg/plugin-http": minor + "@yarnpkg/plugin-init": minor + "@yarnpkg/plugin-interactive-tools": minor + "@yarnpkg/plugin-link": minor + "@yarnpkg/plugin-nm": minor + "@yarnpkg/plugin-npm": minor + "@yarnpkg/plugin-npm-cli": minor + "@yarnpkg/plugin-pack": minor + "@yarnpkg/plugin-patch": minor + "@yarnpkg/plugin-pnp": minor + "@yarnpkg/plugin-pnpm": minor + "@yarnpkg/plugin-stage": minor + "@yarnpkg/plugin-typescript": minor + "@yarnpkg/plugin-version": minor + "@yarnpkg/plugin-workspace-tools": minor + "@yarnpkg/pnp": minor + "@yarnpkg/pnpify": minor + "@yarnpkg/sdks": minor + vscode-zipfs: minor diff --git a/.yarnrc.yml b/.yarnrc.yml index 4efbf59a3aab..1231855ebb12 100644 --- a/.yarnrc.yml +++ b/.yarnrc.yml @@ -6,6 +6,10 @@ enableGlobalCache: false immutablePatterns: - .pnp.* +supportedArchitectures: + os: [darwin, linux, win32] + cpu: [x64, arm64] + initScope: yarnpkg npmPublishAccess: public diff --git a/packages/acceptance-tests/pkg-tests-core/sources/utils/tests.ts b/packages/acceptance-tests/pkg-tests-core/sources/utils/tests.ts index 990e451ed4d9..37ed2edec0f6 100644 --- a/packages/acceptance-tests/pkg-tests-core/sources/utils/tests.ts +++ b/packages/acceptance-tests/pkg-tests-core/sources/utils/tests.ts @@ -34,7 +34,62 @@ export type PackageRunDriver = ( opts: RunDriverOptions, ) => Promise; +export enum RequestType { + Login = `login`, + PackageInfo = `packageInfo`, + PackageTarball = `packageTarball`, + Whoami = `whoami`, + Repository = `repository`, + Publish = `publish`, +} + +export type Request = { + type: RequestType.Login; + username: string, +} | { + type: RequestType.PackageInfo; + scope?: string; + localName: string; +} | { + type: RequestType.PackageTarball; + scope?: string; + localName: string; + version?: string; +} | { + type: RequestType.Whoami; + login: Login +} | { + type: RequestType.Repository; +} | { + type: RequestType.Publish; + scope?: string; + localName: string; +}; + +export interface Login { + username: string; + password: string; + requiresOtp: boolean; + otp?: string; + npmAuthToken: string; +} + let whitelist = new Map(); +let recording: Array | null = null; + +export const startRegistryRecording = async ( + fn: () => Promise, +) => { + const currentRecording: Array = []; + recording = currentRecording; + + try { + await fn(); + return currentRecording; + } finally { + recording = null; + } +}; export const setPackageWhitelist = async ( packages: Map>, @@ -184,38 +239,6 @@ export const startPackageServer = ({type}: { type: keyof typeof packageServerUrl if (serverUrl !== null) return Promise.resolve(serverUrl); - enum RequestType { - Login = `login`, - PackageInfo = `packageInfo`, - PackageTarball = `packageTarball`, - Whoami = `whoami`, - Repository = `repository`, - Publish = `publish`, - } - - type Request = { - type: RequestType.Login; - username: string, - } | { - type: RequestType.PackageInfo; - scope?: string; - localName: string; - } | { - type: RequestType.PackageTarball; - scope?: string; - localName: string; - version?: string; - } | { - type: RequestType.Whoami; - login: Login - } | { - type: RequestType.Repository; - } | { - type: RequestType.Publish; - scope?: string; - localName: string; - }; - const processors: {[requestType in RequestType]: (parsedRequest: Request, request: IncomingMessage, response: ServerResponse) => Promise} = { async [RequestType.PackageInfo](parsedRequest, _, response) { if (parsedRequest.type !== RequestType.PackageInfo) @@ -472,14 +495,6 @@ export const startPackageServer = ({type}: { type: keyof typeof packageServerUrl } }; - interface Login { - username: string; - password: string; - requiresOtp: boolean; - otp?: string; - npmAuthToken: string; - } - const validLogins: Record = { testUser: { username: `testUser`, @@ -518,6 +533,9 @@ export const startPackageServer = ({type}: { type: keyof typeof packageServerUrl return; } + if (recording !== null) + recording.push(parsedRequest); + const {authorization} = req.headers; if (authorization != null) { const auth = validAuthorizations.get(authorization); diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/native-1.0.0/index.js b/packages/acceptance-tests/pkg-tests-fixtures/packages/native-1.0.0/index.js new file mode 100644 index 000000000000..a6bf8f586524 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/native-1.0.0/index.js @@ -0,0 +1,10 @@ +/* @flow */ + +module.exports = require(`./package.json`); + +for (const key of [`dependencies`, `devDependencies`, `peerDependencies`]) { + for (const dep of Object.keys(module.exports[key] || {})) { + // $FlowFixMe The whole point of this file is to be dynamic + module.exports[key][dep] = require(dep); + } +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/native-1.0.0/package.json b/packages/acceptance-tests/pkg-tests-fixtures/packages/native-1.0.0/package.json new file mode 100644 index 000000000000..dfc6a25f6b0e --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/native-1.0.0/package.json @@ -0,0 +1,9 @@ +{ + "name": "native", + "version": "1.0.0", + "dependencies": { + "native-bar-x64": "1.0.0", + "native-foo-x64": "1.0.0", + "native-foo-x86": "1.0.0" + } +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/native-bar-x64-1.0.0/index.js b/packages/acceptance-tests/pkg-tests-fixtures/packages/native-bar-x64-1.0.0/index.js new file mode 100644 index 000000000000..a6bf8f586524 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/native-bar-x64-1.0.0/index.js @@ -0,0 +1,10 @@ +/* @flow */ + +module.exports = require(`./package.json`); + +for (const key of [`dependencies`, `devDependencies`, `peerDependencies`]) { + for (const dep of Object.keys(module.exports[key] || {})) { + // $FlowFixMe The whole point of this file is to be dynamic + module.exports[key][dep] = require(dep); + } +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/native-bar-x64-1.0.0/package.json b/packages/acceptance-tests/pkg-tests-fixtures/packages/native-bar-x64-1.0.0/package.json new file mode 100644 index 000000000000..744f5fc6577b --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/native-bar-x64-1.0.0/package.json @@ -0,0 +1,6 @@ +{ + "name": "native-bar-x64", + "version": "1.0.0", + "os": ["bar"], + "cpu": ["x64"] +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/native-foo-x64-1.0.0/index.js b/packages/acceptance-tests/pkg-tests-fixtures/packages/native-foo-x64-1.0.0/index.js new file mode 100644 index 000000000000..a6bf8f586524 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/native-foo-x64-1.0.0/index.js @@ -0,0 +1,10 @@ +/* @flow */ + +module.exports = require(`./package.json`); + +for (const key of [`dependencies`, `devDependencies`, `peerDependencies`]) { + for (const dep of Object.keys(module.exports[key] || {})) { + // $FlowFixMe The whole point of this file is to be dynamic + module.exports[key][dep] = require(dep); + } +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/native-foo-x64-1.0.0/package.json b/packages/acceptance-tests/pkg-tests-fixtures/packages/native-foo-x64-1.0.0/package.json new file mode 100644 index 000000000000..2aa952069aa9 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/native-foo-x64-1.0.0/package.json @@ -0,0 +1,6 @@ +{ + "name": "native-foo-x64", + "version": "1.0.0", + "os": ["foo"], + "cpu": ["x64"] +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/native-foo-x86-1.0.0/index.js b/packages/acceptance-tests/pkg-tests-fixtures/packages/native-foo-x86-1.0.0/index.js new file mode 100644 index 000000000000..a6bf8f586524 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/native-foo-x86-1.0.0/index.js @@ -0,0 +1,10 @@ +/* @flow */ + +module.exports = require(`./package.json`); + +for (const key of [`dependencies`, `devDependencies`, `peerDependencies`]) { + for (const dep of Object.keys(module.exports[key] || {})) { + // $FlowFixMe The whole point of this file is to be dynamic + module.exports[key][dep] = require(dep); + } +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/native-foo-x86-1.0.0/package.json b/packages/acceptance-tests/pkg-tests-fixtures/packages/native-foo-x86-1.0.0/package.json new file mode 100644 index 000000000000..daabad2da215 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/native-foo-x86-1.0.0/package.json @@ -0,0 +1,6 @@ +{ + "name": "native-foo-x86", + "version": "1.0.0", + "os": ["foo"], + "cpu": ["x86"] +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/optional-native-1.0.0/index.js b/packages/acceptance-tests/pkg-tests-fixtures/packages/optional-native-1.0.0/index.js new file mode 100644 index 000000000000..a6bf8f586524 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/optional-native-1.0.0/index.js @@ -0,0 +1,10 @@ +/* @flow */ + +module.exports = require(`./package.json`); + +for (const key of [`dependencies`, `devDependencies`, `peerDependencies`]) { + for (const dep of Object.keys(module.exports[key] || {})) { + // $FlowFixMe The whole point of this file is to be dynamic + module.exports[key][dep] = require(dep); + } +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/optional-native-1.0.0/package.json b/packages/acceptance-tests/pkg-tests-fixtures/packages/optional-native-1.0.0/package.json new file mode 100644 index 000000000000..1b5947a67d70 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/optional-native-1.0.0/package.json @@ -0,0 +1,9 @@ +{ + "name": "optional-native", + "version": "1.0.0", + "optionalDependencies": { + "native-bar-x64": "1.0.0", + "native-foo-x64": "1.0.0", + "native-foo-x86": "1.0.0" + } +} diff --git a/packages/acceptance-tests/pkg-tests-specs/sources/features/__snapshots__/mergeConflictResolution.test.js.snap b/packages/acceptance-tests/pkg-tests-specs/sources/features/__snapshots__/mergeConflictResolution.test.js.snap index fdc197d3cdfe..6b12bd060363 100644 --- a/packages/acceptance-tests/pkg-tests-specs/sources/features/__snapshots__/mergeConflictResolution.test.js.snap +++ b/packages/acceptance-tests/pkg-tests-specs/sources/features/__snapshots__/mergeConflictResolution.test.js.snap @@ -5,7 +5,7 @@ exports[`Features Merge Conflict Resolution it should properly fix merge conflic # Manual changes might be lost - proceed with caution! __metadata: - version: 4 + version: 5 cacheKey: 0 \\"no-deps@npm:*\\": @@ -54,7 +54,7 @@ exports[`Features Merge Conflict Resolution it should properly fix merge conflic # Manual changes might be lost - proceed with caution! __metadata: - version: 4 + version: 5 cacheKey: 0 <<<<<<< HEAD diff --git a/packages/acceptance-tests/pkg-tests-specs/sources/features/__snapshots__/prunedNativeDeps.test.ts.snap b/packages/acceptance-tests/pkg-tests-specs/sources/features/__snapshots__/prunedNativeDeps.test.ts.snap new file mode 100644 index 000000000000..7bc0a9baf0e3 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-specs/sources/features/__snapshots__/prunedNativeDeps.test.ts.snap @@ -0,0 +1,58 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Features Pruned native deps should resolve all dependencies, regardless of the system 1`] = ` +"# This file is generated by running \\"yarn install\\" inside your project. +# Manual changes might be lost - proceed with caution! + +__metadata: + version: 5 + cacheKey: 0 + +\\"native-bar-x64@npm:1.0.0\\": + version: 1.0.0 + resolution: \\"native-bar-x64@npm:1.0.0\\" + conditions: os=bar & cpu=x64 + languageName: node + linkType: hard + +\\"native-foo-x64@npm:1.0.0\\": + version: 1.0.0 + resolution: \\"native-foo-x64@npm:1.0.0\\" + conditions: os=foo & cpu=x64 + languageName: node + linkType: hard + +\\"native-foo-x86@npm:1.0.0\\": + version: 1.0.0 + resolution: \\"native-foo-x86@npm:1.0.0\\" + conditions: os=foo & cpu=x86 + languageName: node + linkType: hard + +\\"optional-native@npm:1.0.0\\": + version: 1.0.0 + resolution: \\"optional-native@npm:1.0.0\\" + dependencies: + native-bar-x64: 1.0.0 + native-foo-x64: 1.0.0 + native-foo-x86: 1.0.0 + dependenciesMeta: + native-bar-x64: + optional: true + native-foo-x64: + optional: true + native-foo-x86: + optional: true + checksum: 0e662ccf2f901c37aa95cc3e8bee87315782ca27582521644c86e1ee44367a31a0201049bad23109ca9783efc9ac70bc7dae8619c31f2b6472830b7cf1670f0d + languageName: node + linkType: hard + +\\"root-workspace-0b6124@workspace:.\\": + version: 0.0.0-use.local + resolution: \\"root-workspace-0b6124@workspace:.\\" + dependencies: + optional-native: 1.0.0 + languageName: unknown + linkType: soft +" +`; diff --git a/packages/acceptance-tests/pkg-tests-specs/sources/features/prunedNativeDeps.test.ts b/packages/acceptance-tests/pkg-tests-specs/sources/features/prunedNativeDeps.test.ts new file mode 100644 index 000000000000..c47341fc59c9 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-specs/sources/features/prunedNativeDeps.test.ts @@ -0,0 +1,290 @@ +import {Filename, PortablePath, ppath, xfs} from '@yarnpkg/fslib'; +import {RequestType, startRegistryRecording} from 'pkg-tests-core/sources/utils/tests'; + +export {}; + +describe(`Features`, () => { + describe(`Pruned native deps`, () => { + it(`should resolve all dependencies, regardless of the system`, makeTemporaryEnv({ + dependencies: { + [`optional-native`]: `1.0.0`, + }, + }, async ({path, run, source}) => { + await xfs.writeJsonPromise(ppath.join(path, Filename.rc), { + supportedArchitectures: { + cpu: [`foo`], + os: [`x64`], + }, + }); + + await run(`install`); + + await expect(xfs.readFilePromise(ppath.join(path, Filename.lockfile), `utf8`)).resolves.toMatchSnapshot(); + })); + + it(`shouldn't fetch packages that it won't need`, makeTemporaryEnv({ + dependencies: { + [`optional-native`]: `1.0.0`, + }, + }, async ({path, run, source}) => { + await xfs.writeJsonPromise(ppath.join(path, Filename.rc), { + supportedArchitectures: { + os: [`foo`], + cpu: [`x64`], + }, + }); + + const recording = await startRegistryRecording(async () => { + await run(`install`); + }); + + const tarballRequests = recording.filter(request => { + return request.type === RequestType.PackageTarball; + }).sort((a, b) => { + const aJson = JSON.stringify(a); + const bJson = JSON.stringify(b); + return aJson < bJson ? -1 : aJson > bJson ? 1 : 0; + }); + + expect(tarballRequests).toEqual([{ + type: RequestType.PackageTarball, + localName: `native-foo-x64`, + version: `1.0.0`, + }, { + type: RequestType.PackageTarball, + localName: `optional-native`, + version: `1.0.0`, + }]); + })); + + it(`should overfetch if requested to do so`, makeTemporaryEnv({ + dependencies: { + [`optional-native`]: `1.0.0`, + }, + }, async ({path, run, source}) => { + await xfs.writeJsonPromise(ppath.join(path, Filename.rc), { + supportedArchitectures: { + os: [`foo`], + cpu: [`x64`, `x86`], + }, + }); + + const recording = await startRegistryRecording(async () => { + await run(`install`); + }); + + const tarballRequests = recording.filter(request => { + return request.type === RequestType.PackageTarball; + }).sort((a, b) => { + const aJson = JSON.stringify(a); + const bJson = JSON.stringify(b); + return aJson < bJson ? -1 : aJson > bJson ? 1 : 0; + }); + + expect(tarballRequests).toEqual([{ + type: RequestType.PackageTarball, + localName: `native-foo-x64`, + version: `1.0.0`, + }, { + type: RequestType.PackageTarball, + localName: `native-foo-x86`, + version: `1.0.0`, + }, { + type: RequestType.PackageTarball, + localName: `optional-native`, + version: `1.0.0`, + }]); + })); + + it(`should produce a stable lockfile, regardless of the architecture`, makeTemporaryEnv({ + dependencies: { + [`optional-native`]: `1.0.0`, + }, + }, async ({path, run, source}) => { + await xfs.writeJsonPromise(ppath.join(path, Filename.rc), { + supportedArchitectures: { + os: [`foo`], + cpu: [`x64`], + }, + }); + + await run(`install`); + const lockfile64 = await xfs.readFilePromise(ppath.join(path, Filename.lockfile), `utf8`); + + await xfs.writeJsonPromise(ppath.join(path, Filename.rc), { + supportedArchitectures: { + os: [`foo`], + cpu: [`x86`], + }, + }); + + await run(`install`); + const lockfile86 = await xfs.readFilePromise(ppath.join(path, Filename.lockfile), `utf8`); + + expect(lockfile86).toEqual(lockfile64); + })); + + it(`should produce a stable PnP hook, regardless of the architecture`, makeTemporaryEnv({ + dependencies: { + [`optional-native`]: `1.0.0`, + }, + }, async ({path, run, source}) => { + await xfs.writeJsonPromise(ppath.join(path, Filename.rc), { + supportedArchitectures: { + os: [`foo`], + cpu: [`x64`], + }, + }); + + await run(`install`); + const hook64 = await xfs.readFilePromise(ppath.join(path, Filename.pnpCjs), `utf8`); + + await xfs.writeJsonPromise(ppath.join(path, Filename.rc), { + supportedArchitectures: { + os: [`foo`], + cpu: [`x86`], + }, + }); + + await run(`install`); + const hook86 = await xfs.readFilePromise(ppath.join(path, Filename.pnpCjs), `utf8`); + + expect(hook86).toEqual(hook64); + })); + + it(`shouldn't break when using --check-cache with native packages`, makeTemporaryEnv({ + dependencies: { + [`optional-native`]: `1.0.0`, + }, + }, async ({path, run, source}) => { + await xfs.writeJsonPromise(ppath.join(path, Filename.rc), { + supportedArchitectures: { + os: [`foo`], + cpu: [`x64`], + }, + }); + + await run(`install`); + + const cacheFolder = ppath.join(path, `.yarn/cache` as PortablePath); + const cacheListing = await xfs.readdirPromise(cacheFolder); + const nativeFile = cacheListing.find(entry => entry.startsWith(`native-foo-x64-`)); + + // Sanity check + expect(nativeFile).toBeDefined(); + + await run(`install`, `--check-cache`); + })); + + it(`should detect packages being tampered when using --check-cache`, makeTemporaryEnv({ + dependencies: { + [`optional-native`]: `1.0.0`, + }, + }, async ({path, run, source}) => { + await xfs.writeJsonPromise(ppath.join(path, Filename.rc), { + supportedArchitectures: { + os: [`foo`], + cpu: [`x64`], + }, + }); + + await run(`install`); + + const cacheFolder = ppath.join(path, `.yarn/cache` as PortablePath); + const cacheListing = await xfs.readdirPromise(cacheFolder); + const nativeFile = cacheListing.find(entry => entry.startsWith(`native-foo-x64-`)); + + // Sanity check + expect(nativeFile).toBeDefined(); + + await xfs.appendFilePromise(ppath.join(cacheFolder, nativeFile as Filename), Buffer.from([0])); + + await expect(async () => { + await run(`install`, `--check-cache`); + }).rejects.toThrow(); + })); + + it(`should also validate other architectures than the current one if necessary when using --check-cache`, makeTemporaryEnv({ + dependencies: { + [`optional-native`]: `1.0.0`, + }, + }, async ({path, run, source}) => { + await xfs.writeJsonPromise(ppath.join(path, Filename.rc), { + supportedArchitectures: { + os: [`foo`], + cpu: [`x64`, `x86`], + }, + }); + + await run(`install`); + + const cacheFolder = ppath.join(path, `.yarn/cache` as PortablePath); + const cacheListing = await xfs.readdirPromise(cacheFolder); + const nativeFile = cacheListing.find(entry => entry.startsWith(`native-foo-x64-`)); + + // Sanity check + expect(nativeFile).toBeDefined(); + + await xfs.appendFilePromise(ppath.join(cacheFolder, nativeFile as Filename), Buffer.from([0])); + + await xfs.writeJsonPromise(ppath.join(path, Filename.rc), { + supportedArchitectures: { + os: [`foo`], + cpu: [`x86`], + }, + }); + + await expect(async () => { + await run(`install`, `--check-cache`); + }).rejects.toThrow(); + })); + + it(`should only fetch other architectures when using --check-cache if they are already in the cache`, makeTemporaryEnv({ + dependencies: { + [`optional-native`]: `1.0.0`, + }, + }, async ({path, run, source}) => { + await xfs.writeJsonPromise(ppath.join(path, Filename.rc), { + supportedArchitectures: { + os: [`foo`], + cpu: [`x64`, `x86`], + }, + }); + + await run(`install`); + + await xfs.writeJsonPromise(ppath.join(path, Filename.rc), { + supportedArchitectures: { + os: [`foo`], + cpu: [`x64`], + }, + }); + + const recording = await startRegistryRecording(async () => { + await run(`install`, `--check-cache`); + }); + + const tarballRequests = recording.filter(request => { + return request.type === RequestType.PackageTarball; + }).sort((a, b) => { + const aJson = JSON.stringify(a); + const bJson = JSON.stringify(b); + return aJson < bJson ? -1 : aJson > bJson ? 1 : 0; + }); + + expect(tarballRequests).toEqual([{ + type: RequestType.PackageTarball, + localName: `native-foo-x64`, + version: `1.0.0`, + }, { + type: RequestType.PackageTarball, + localName: `native-foo-x86`, + version: `1.0.0`, + }, { + type: RequestType.PackageTarball, + localName: `optional-native`, + version: `1.0.0`, + }]); + })); + }); +}); diff --git a/packages/acceptance-tests/pkg-tests-specs/sources/node-modules.test.ts b/packages/acceptance-tests/pkg-tests-specs/sources/node-modules.test.ts index 015f54fab9b3..e414a66bcb3a 100644 --- a/packages/acceptance-tests/pkg-tests-specs/sources/node-modules.test.ts +++ b/packages/acceptance-tests/pkg-tests-specs/sources/node-modules.test.ts @@ -285,7 +285,7 @@ describe(`Node_Modules`, () => { const stdout = (await run(`install`)).stdout; expect(stdout).not.toContain(`Shall not be run`); - expect(stdout).toMatch(new RegExp(`dep@file:./dep.*The platform ${process.platform} is incompatible with this module, link skipped.`)); + expect(stdout).toMatch(new RegExp(`dep@file:./dep.*The ${process.platform}-${process.arch} architecture is incompatible with this module, link skipped.`)); await expect(source(`require('dep')`)).rejects.toMatchObject({ externalException: { diff --git a/packages/acceptance-tests/pkg-tests-specs/sources/pnp.test.js b/packages/acceptance-tests/pkg-tests-specs/sources/pnp.test.js index e24386b2ae1e..afaecfb8117d 100644 --- a/packages/acceptance-tests/pkg-tests-specs/sources/pnp.test.js +++ b/packages/acceptance-tests/pkg-tests-specs/sources/pnp.test.js @@ -1652,7 +1652,7 @@ describe(`Plug'n'Play`, () => { const stdout = (await run(`install`)).stdout; expect(stdout).not.toContain(`Shall not be run`); - expect(stdout).toMatch(new RegExp(`dep@file:./dep.*The platform ${process.platform} is incompatible with this module, build skipped.`)); + expect(stdout).toMatch(new RegExp(`dep@file:./dep.*The ${process.platform}-${process.arch} architecture is incompatible with this module, build skipped.`)); await expect(source(`require('dep')`)).resolves.toMatchObject({ name: `dep`, diff --git a/packages/gatsby/content/advanced/error-codes.md b/packages/gatsby/content/advanced/error-codes.md index f583f79d694c..261f8dd9a206 100644 --- a/packages/gatsby/content/advanced/error-codes.md +++ b/packages/gatsby/content/advanced/error-codes.md @@ -358,3 +358,13 @@ workspace(WorkspaceCwd), workspace_field(WorkspaceCwd, 'name', _). ``` For more information about the parameters that must be instantiated when calling the predicate reported by the error message, consult the [dedicated page](/features/constraints#query-predicate) from our documentation. + +## YN0076 - `INCOMPATIBLE_ARCHITECTURE` + +A package is specified in its manifest (through the [`os`](/configuration/manifest#os) / [`cpu`](/configuration/manifest#cpu) fields) as being incompatible with the system architecture. Its postinstall scripts will not run on this system. + +## YN0077 - `GHOST_ARCHITECTURE` + +Some native packages may be excluded from the install if they signal they don't support the systems the project is intended for. This detection is typically based on your current system parameters, but it can be configured using the [`supportedArchitectures` config option](/configuration/yarnrc#supportedArchitectures). If your os or cpu are missing from this list, Yarn will skip the packages and raise a warning. + +Note that all fields from `supportedArchitectures` default to `current`, which is a dynamic value depending on your local parameters. For instance, if you wish to support "my current os, whatever it is, plus linux", you can set `supportedArchitectures.os` to `["current", "linux"]`. diff --git a/packages/gatsby/static/configuration/manifest.json b/packages/gatsby/static/configuration/manifest.json index aa99b8e8d5ad..2a93614fd805 100644 --- a/packages/gatsby/static/configuration/manifest.json +++ b/packages/gatsby/static/configuration/manifest.json @@ -46,6 +46,22 @@ "type": "string", "examples": ["MIT"] }, + "os": { + "description": "A value compared during install with `process.platform`. If the values don't match, the package won't see its postinstall scripts run (if listed in `dependencies`) or won't be installed at all (if listed in `optionalDependencies`).", + "type": "array", + "items": { + "type": "string" + }, + "_exampleItems": ["linux", "darwin", "win32"] + }, + "cpu": { + "description": "A value compared during install with `process.arch`. If the values don't match, the package won't see its postinstall scripts run (if listed in `dependencies`) or won't be installed at all (if listed in `optionalDependencies`).", + "type": "array", + "items": { + "type": "string" + }, + "_exampleItems": ["x64", "ia32", "arm64"] + }, "main": { "description": "The path that will be used to resolve the qualified path to use when accessing the package by its name. This field can be modified at publish-time through the use of the `publishConfig.main` field.", "type": "string", @@ -114,7 +130,7 @@ "_exampleKeys": ["webpack"] }, "optionalDependencies": { - "description": "Similar to the `dependencies` field, except that these entries will not be required to build properly should they have any build script. Note that such dependencies must still be resolvable and fetchable (otherwise we couldn't store it in the lockfile, which could lead to non-reproducible installs) - only the build step is optional.\n\n**This field is usually not what you're looking for**, unless you depend on the `fsevents` package. If you need a package to be required only when a specific feature is used then use an optional peer dependency. Your users will have to satisfy it should they use the feature, but it won't cause the build errors to be silently swallowed when the feature is needed.", + "description": "Similar to the `dependencies` field, except that these entries will not be required to build properly should they have any build script. Note that such dependencies must always be resolvable (otherwise we couldn't store it in the lockfile, which could lead to non-reproducible installs), but those which list `cpu`/`os` fields will not be fetched unless they match the current system architecture.\n\n**This field is usually not what you're looking for**, unless you depend on the `fsevents` package. If you need a package to be required only when a specific feature is used then use an optional peer dependency. Your users will have to satisfy it should they use the feature, but it won't cause the build errors to be silently swallowed when the feature is needed.", "type": "object", "patternProperties": { "^(?:@([^/]+?)/)?([^/]+?)$": { @@ -173,7 +189,7 @@ "examples": [false] }, "optional": { - "description": "If true, the build isn't required to succeed for the install to be considered a success. It's what the `optionalDependencies` field compiles down to.\n\n**This settings will be applied even when found within a nested manifest**, but the highest requirement in the dependency tree will prevail.", + "description": "If true, the build isn't required to succeed for the install to be considered a success, and the dependency may be skipped if its `os` and `cpu` fields don't match the current system architecture. It's what the `optionalDependencies` field compiles down to.\n\n**This settings will be applied even when found within a nested manifest**, but the highest requirement in the dependency tree will prevail.", "type": "boolean", "examples": [false] }, diff --git a/packages/gatsby/static/configuration/yarnrc.json b/packages/gatsby/static/configuration/yarnrc.json index e7efaafbe2b7..36e419ade889 100644 --- a/packages/gatsby/static/configuration/yarnrc.json +++ b/packages/gatsby/static/configuration/yarnrc.json @@ -618,6 +618,32 @@ "pattern": "^[^<>:;,?\"*|/]+$", "default": ".yarnrc.yml" }, + "supportedArchitectures": { + "_package": "@yarnpkg/core", + "description": "Defines which for systems should Yarn install packages.", + "type": "object", + "properties": { + "os": { + "description": "The list of operating systems to cover.", + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "_exampleItems": ["current", "darwin", "linux", "win32"] + }, + "cpu": { + "description": "The list of CPUs to cover.", + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "_exampleItems": ["current", "x86", "ia32"] + } + }, + "_exampleKeys": ["os", "cpu"] + }, "telemetryInterval": { "_package": "@yarnpkg/core", "description": "This setting defines the minimal amount of time between two telemetry uploads, in days. By default we only send one request per week, making it impossible for us to track your usage with a lower granularity.", diff --git a/packages/plugin-essentials/sources/commands/info.ts b/packages/plugin-essentials/sources/commands/info.ts index d650677500c8..8a6d780e9ccf 100644 --- a/packages/plugin-essentials/sources/commands/info.ts +++ b/packages/plugin-essentials/sources/commands/info.ts @@ -255,8 +255,13 @@ export default class InfoCommand extends BaseCommand { if (!extra.has(`cache`)) return; + const cacheOptions = { + mockedPackages: project.disabledLocators, + unstablePackages: project.conditionalLocators, + }; + const checksum = project.storedChecksums.get(pkg.locatorHash) ?? null; - const cachePath = cache.getLocatorPath(pkg, checksum); + const cachePath = cache.getLocatorPath(pkg, checksum, cacheOptions); let stat; if (cachePath !== null) { diff --git a/packages/plugin-exec/sources/ExecFetcher.ts b/packages/plugin-exec/sources/ExecFetcher.ts index 0d859880aebc..b6a7a2a31920 100644 --- a/packages/plugin-exec/sources/ExecFetcher.ts +++ b/packages/plugin-exec/sources/ExecFetcher.ts @@ -51,8 +51,7 @@ export class ExecFetcher implements Fetcher { onHit: () => opts.report.reportCacheHit(locator), onMiss: () => opts.report.reportCacheMiss(locator), loader: () => this.fetchFromDisk(locator, opts), - skipIntegrityCheck: opts.skipIntegrityCheck, - }); + }, opts.cacheOptions); return { packageFs, diff --git a/packages/plugin-exec/sources/ExecResolver.ts b/packages/plugin-exec/sources/ExecResolver.ts index ca6d6f8590ea..ea01a0df3995 100644 --- a/packages/plugin-exec/sources/ExecResolver.ts +++ b/packages/plugin-exec/sources/ExecResolver.ts @@ -82,6 +82,8 @@ export class ExecResolver implements Resolver { languageName: manifest.languageName || opts.project.configuration.get(`defaultLanguageName`), linkType: LinkType.HARD, + conditions: manifest.getConditions(), + dependencies: manifest.dependencies, peerDependencies: manifest.peerDependencies, diff --git a/packages/plugin-file/sources/FileFetcher.ts b/packages/plugin-file/sources/FileFetcher.ts index 91e9cd9c0e43..cf8b5c09f4d8 100644 --- a/packages/plugin-file/sources/FileFetcher.ts +++ b/packages/plugin-file/sources/FileFetcher.ts @@ -33,8 +33,7 @@ export class FileFetcher implements Fetcher { onHit: () => opts.report.reportCacheHit(locator), onMiss: () => opts.report.reportCacheMiss(locator, `${structUtils.prettyLocator(opts.project.configuration, locator)} can't be found in the cache and will be fetched from the disk`), loader: () => this.fetchFromDisk(locator, opts), - skipIntegrityCheck: opts.skipIntegrityCheck, - }); + }, opts.cacheOptions); return { packageFs, diff --git a/packages/plugin-file/sources/FileResolver.ts b/packages/plugin-file/sources/FileResolver.ts index a98a0a8cfa78..088032af65ee 100644 --- a/packages/plugin-file/sources/FileResolver.ts +++ b/packages/plugin-file/sources/FileResolver.ts @@ -94,6 +94,8 @@ export class FileResolver implements Resolver { languageName: manifest.languageName || opts.project.configuration.get(`defaultLanguageName`), linkType: LinkType.HARD, + conditions: manifest.getConditions(), + dependencies: manifest.dependencies, peerDependencies: manifest.peerDependencies, diff --git a/packages/plugin-file/sources/TarballFileFetcher.ts b/packages/plugin-file/sources/TarballFileFetcher.ts index d452d3d1c66d..4f1dbe02be45 100644 --- a/packages/plugin-file/sources/TarballFileFetcher.ts +++ b/packages/plugin-file/sources/TarballFileFetcher.ts @@ -27,8 +27,7 @@ export class TarballFileFetcher implements Fetcher { onHit: () => opts.report.reportCacheHit(locator), onMiss: () => opts.report.reportCacheMiss(locator, `${structUtils.prettyLocator(opts.project.configuration, locator)} can't be found in the cache and will be fetched from the disk`), loader: () => this.fetchFromDisk(locator, opts), - skipIntegrityCheck: opts.skipIntegrityCheck, - }); + }, opts.cacheOptions); return { packageFs, diff --git a/packages/plugin-file/sources/TarballFileResolver.ts b/packages/plugin-file/sources/TarballFileResolver.ts index 214eb0f31742..c7609ff494fd 100644 --- a/packages/plugin-file/sources/TarballFileResolver.ts +++ b/packages/plugin-file/sources/TarballFileResolver.ts @@ -78,6 +78,8 @@ export class TarballFileResolver implements Resolver { languageName: manifest.languageName || opts.project.configuration.get(`defaultLanguageName`), linkType: LinkType.HARD, + conditions: manifest.getConditions(), + dependencies: manifest.dependencies, peerDependencies: manifest.peerDependencies, diff --git a/packages/plugin-git/sources/GitFetcher.ts b/packages/plugin-git/sources/GitFetcher.ts index 67f5d3e4694e..db2ee47a40dc 100644 --- a/packages/plugin-git/sources/GitFetcher.ts +++ b/packages/plugin-git/sources/GitFetcher.ts @@ -31,8 +31,7 @@ export class GitFetcher implements Fetcher { onHit: () => opts.report.reportCacheHit(locator), onMiss: () => opts.report.reportCacheMiss(locator, `${structUtils.prettyLocator(opts.project.configuration, locator)} can't be found in the cache and will be fetched from the remote repository`), loader: () => this.cloneFromRemote(normalizedLocator, nextOpts), - skipIntegrityCheck: opts.skipIntegrityCheck, - }); + }, opts.cacheOptions); return { packageFs, diff --git a/packages/plugin-git/sources/GitResolver.ts b/packages/plugin-git/sources/GitResolver.ts index b54422b6219a..fe625ea91f20 100644 --- a/packages/plugin-git/sources/GitResolver.ts +++ b/packages/plugin-git/sources/GitResolver.ts @@ -55,6 +55,8 @@ export class GitResolver implements Resolver { languageName: manifest.languageName || opts.project.configuration.get(`defaultLanguageName`), linkType: LinkType.HARD, + conditions: manifest.getConditions(), + dependencies: manifest.dependencies, peerDependencies: manifest.peerDependencies, diff --git a/packages/plugin-github/sources/GithubFetcher.ts b/packages/plugin-github/sources/GithubFetcher.ts index 78bc5f2c4ac1..086c344ed15a 100644 --- a/packages/plugin-github/sources/GithubFetcher.ts +++ b/packages/plugin-github/sources/GithubFetcher.ts @@ -25,8 +25,7 @@ export class GithubFetcher implements Fetcher { onHit: () => opts.report.reportCacheHit(locator), onMiss: () => opts.report.reportCacheMiss(locator, `${structUtils.prettyLocator(opts.project.configuration, locator)} can't be found in the cache and will be fetched from GitHub`), loader: () => this.fetchFromNetwork(locator, opts), - skipIntegrityCheck: opts.skipIntegrityCheck, - }); + }, opts.cacheOptions); return { packageFs, diff --git a/packages/plugin-http/sources/TarballHttpFetcher.ts b/packages/plugin-http/sources/TarballHttpFetcher.ts index e73aa8ee295e..96856240cc20 100644 --- a/packages/plugin-http/sources/TarballHttpFetcher.ts +++ b/packages/plugin-http/sources/TarballHttpFetcher.ts @@ -26,8 +26,7 @@ export class TarballHttpFetcher implements Fetcher { onHit: () => opts.report.reportCacheHit(locator), onMiss: () => opts.report.reportCacheMiss(locator, `${structUtils.prettyLocator(opts.project.configuration, locator)} can't be found in the cache and will be fetched from the remote server`), loader: () => this.fetchFromNetwork(locator, opts), - skipIntegrityCheck: opts.skipIntegrityCheck, - }); + }, opts.cacheOptions); return { packageFs, diff --git a/packages/plugin-http/sources/TarballHttpResolver.ts b/packages/plugin-http/sources/TarballHttpResolver.ts index 78b78db9d13b..bd3e8877fc42 100644 --- a/packages/plugin-http/sources/TarballHttpResolver.ts +++ b/packages/plugin-http/sources/TarballHttpResolver.ts @@ -64,6 +64,8 @@ export class TarballHttpResolver implements Resolver { languageName: manifest.languageName || opts.project.configuration.get(`defaultLanguageName`), linkType: LinkType.HARD, + conditions: manifest.getConditions(), + dependencies: manifest.dependencies, peerDependencies: manifest.peerDependencies, diff --git a/packages/plugin-link/sources/LinkResolver.ts b/packages/plugin-link/sources/LinkResolver.ts index 526e64005c37..3f78b6394bfe 100644 --- a/packages/plugin-link/sources/LinkResolver.ts +++ b/packages/plugin-link/sources/LinkResolver.ts @@ -63,6 +63,8 @@ export class LinkResolver implements Resolver { languageName: manifest.languageName || opts.project.configuration.get(`defaultLanguageName`), linkType: LinkType.SOFT, + conditions: manifest.getConditions(), + dependencies: new Map([...manifest.dependencies]), peerDependencies: manifest.peerDependencies, diff --git a/packages/plugin-link/sources/RawLinkResolver.ts b/packages/plugin-link/sources/RawLinkResolver.ts index 20b89e0fa40b..429aeeee0a7a 100644 --- a/packages/plugin-link/sources/RawLinkResolver.ts +++ b/packages/plugin-link/sources/RawLinkResolver.ts @@ -54,6 +54,8 @@ export class RawLinkResolver implements Resolver { languageName: opts.project.configuration.get(`defaultLanguageName`), linkType: LinkType.SOFT, + conditions: null, + dependencies: new Map(), peerDependencies: new Map(), diff --git a/packages/plugin-nm/sources/NodeModulesLinker.ts b/packages/plugin-nm/sources/NodeModulesLinker.ts index 96274e3df686..7bce9239aa66 100644 --- a/packages/plugin-nm/sources/NodeModulesLinker.ts +++ b/packages/plugin-nm/sources/NodeModulesLinker.ts @@ -138,7 +138,7 @@ class NodeModulesInstaller implements Installer { } // We don't link the package at all if it's for an unsupported platform - if (!jsInstallUtils.checkAndReportManifestCompatibility(pkg, customPackageData, `link`, {configuration: this.opts.project.configuration, report: this.opts.report})) + if (!jsInstallUtils.checkAndReportManifestCompatibility(pkg, `link`, {configuration: this.opts.project.configuration, report: this.opts.report})) return {packageLocation: null, buildDirective: null}; const packageDependencies = new Map(); @@ -367,8 +367,6 @@ async function extractCustomPackageData(pkg: Package, fetchResult: FetchResult) return { manifest: { bin: manifest.bin, - os: manifest.os, - cpu: manifest.cpu, scripts: manifest.scripts, }, misc: { diff --git a/packages/plugin-npm/sources/NpmHttpFetcher.ts b/packages/plugin-npm/sources/NpmHttpFetcher.ts index 11acd6ac1dab..c2c5596718f9 100644 --- a/packages/plugin-npm/sources/NpmHttpFetcher.ts +++ b/packages/plugin-npm/sources/NpmHttpFetcher.ts @@ -32,8 +32,7 @@ export class NpmHttpFetcher implements Fetcher { onHit: () => opts.report.reportCacheHit(locator), onMiss: () => opts.report.reportCacheMiss(locator, `${structUtils.prettyLocator(opts.project.configuration, locator)} can't be found in the cache and will be fetched from the remote server`), loader: () => this.fetchFromNetwork(locator, opts), - skipIntegrityCheck: opts.skipIntegrityCheck, - }); + }, opts.cacheOptions); return { packageFs, diff --git a/packages/plugin-npm/sources/NpmSemverFetcher.ts b/packages/plugin-npm/sources/NpmSemverFetcher.ts index 73c1c45de0f4..946044bfa153 100644 --- a/packages/plugin-npm/sources/NpmSemverFetcher.ts +++ b/packages/plugin-npm/sources/NpmSemverFetcher.ts @@ -34,8 +34,7 @@ export class NpmSemverFetcher implements Fetcher { onHit: () => opts.report.reportCacheHit(locator), onMiss: () => opts.report.reportCacheMiss(locator, `${structUtils.prettyLocator(opts.project.configuration, locator)} can't be found in the cache and will be fetched from the remote registry`), loader: () => this.fetchFromNetwork(locator, opts), - skipIntegrityCheck: opts.skipIntegrityCheck, - }); + }, opts.cacheOptions); return { packageFs, diff --git a/packages/plugin-npm/sources/NpmSemverResolver.ts b/packages/plugin-npm/sources/NpmSemverResolver.ts index 9146fecb7bac..80964d169edd 100644 --- a/packages/plugin-npm/sources/NpmSemverResolver.ts +++ b/packages/plugin-npm/sources/NpmSemverResolver.ts @@ -158,6 +158,8 @@ export class NpmSemverResolver implements Resolver { languageName: `node`, linkType: LinkType.HARD, + conditions: manifest.getConditions(), + dependencies: manifest.dependencies, peerDependencies: manifest.peerDependencies, diff --git a/packages/plugin-patch/sources/PatchFetcher.ts b/packages/plugin-patch/sources/PatchFetcher.ts index dea17c1ee758..6db150d5920f 100644 --- a/packages/plugin-patch/sources/PatchFetcher.ts +++ b/packages/plugin-patch/sources/PatchFetcher.ts @@ -27,8 +27,7 @@ export class PatchFetcher implements Fetcher { onHit: () => opts.report.reportCacheHit(locator), onMiss: () => opts.report.reportCacheMiss(locator, `${structUtils.prettyLocator(opts.project.configuration, locator)} can't be found in the cache and will be fetched from the disk`), loader: () => this.patchPackage(locator, opts), - skipIntegrityCheck: opts.skipIntegrityCheck, - }); + }, opts.cacheOptions); return { packageFs, diff --git a/packages/plugin-pnp/sources/PnpLinker.ts b/packages/plugin-pnp/sources/PnpLinker.ts index e71cda8a93bb..2623e32cd2d4 100644 --- a/packages/plugin-pnp/sources/PnpLinker.ts +++ b/packages/plugin-pnp/sources/PnpLinker.ts @@ -1,7 +1,7 @@ import {miscUtils, structUtils, formatUtils, Descriptor, LocatorHash} from '@yarnpkg/core'; import {FetchResult, Locator, Package} from '@yarnpkg/core'; import {Linker, LinkOptions, MinimalLinkOptions, Manifest, MessageName, DependencyMeta, LinkType, Installer} from '@yarnpkg/core'; -import {CwdFS, PortablePath, VirtualFS, npath, ppath, xfs, Filename} from '@yarnpkg/fslib'; +import {AliasFS, CwdFS, PortablePath, VirtualFS, npath, ppath, xfs, Filename} from '@yarnpkg/fslib'; import {generateInlinedScript, generateSplitScript, PackageRegistry, PnpApi, PnpSettings} from '@yarnpkg/pnp'; import {UsageError} from 'clipanion'; @@ -381,6 +381,9 @@ export class PnpInstaller implements Installer { if (FORCED_UNPLUG_PACKAGES.has(pkg.identHash)) return true; + if (pkg.conditions !== null) + return true; + if (customPackageData.manifest.preferUnplugged !== null) return customPackageData.manifest.preferUnplugged; @@ -392,6 +395,9 @@ export class PnpInstaller implements Installer { private async unplugPackage(locator: Locator, fetchResult: FetchResult) { const unplugPath = pnpUtils.getUnpluggedPath(locator, {configuration: this.opts.project.configuration}); + if (this.opts.project.disabledLocators.has(locator.locatorHash)) + return new AliasFS(unplugPath, {baseFs: fetchResult.packageFs, pathUtils: ppath}); + this.unpluggedPaths.add(unplugPath); const readyFile = ppath.join(unplugPath, fetchResult.prefixPath, `.ready` as Filename); @@ -462,8 +468,6 @@ async function extractCustomPackageData(fetchResult: FetchResult) { return { manifest: { - os: manifest.os, - cpu: manifest.cpu, scripts: manifest.scripts, preferUnplugged: manifest.preferUnplugged, }, diff --git a/packages/plugin-pnp/sources/jsInstallUtils.ts b/packages/plugin-pnp/sources/jsInstallUtils.ts index 6775cbc8e222..51050b86c228 100644 --- a/packages/plugin-pnp/sources/jsInstallUtils.ts +++ b/packages/plugin-pnp/sources/jsInstallUtils.ts @@ -1,25 +1,16 @@ import {BuildDirective, BuildType, Configuration, DependencyMeta, FetchResult, LinkType, Manifest, MessageName, Package, Report, structUtils} from '@yarnpkg/core'; import {Filename, ppath} from '@yarnpkg/fslib'; -export type ManifestCompatibilityDataRequirements = { - manifest: Pick, -}; - -export function checkAndReportManifestCompatibility(pkg: Package, requirements: ManifestCompatibilityDataRequirements, label: string, {configuration, report}: {configuration: Configuration, report?: Report | null}) { - if (!Manifest.isManifestFieldCompatible(requirements.manifest.os, process.platform)) { - report?.reportWarningOnce(MessageName.INCOMPATIBLE_OS, `${structUtils.prettyLocator(configuration, pkg)} The platform ${process.platform} is incompatible with this module, ${label} skipped.`); - return false; - } - - if (!Manifest.isManifestFieldCompatible(requirements.manifest.cpu, process.arch)) { - report?.reportWarningOnce(MessageName.INCOMPATIBLE_CPU, `${structUtils.prettyLocator(configuration, pkg)} The CPU architecture ${process.arch} is incompatible with this module, ${label} skipped.`); +export function checkAndReportManifestCompatibility(pkg: Package, label: string, {configuration, report}: {configuration: Configuration, report?: Report | null}) { + if (!structUtils.isPackageCompatible(pkg, {os: [process.platform], cpu: [process.arch]})) { + report?.reportWarningOnce(MessageName.INCOMPATIBLE_ARCHITECTURE, `${structUtils.prettyLocator(configuration, pkg)} The ${process.platform}-${process.arch} architecture is incompatible with this module, ${label} skipped.`); return false; } return true; } -export type ExtractBuildScriptDataRequirements = ManifestCompatibilityDataRequirements & { +export type ExtractBuildScriptDataRequirements = { manifest: Pick, misc: { hasBindingGyp: boolean, @@ -55,7 +46,7 @@ export function extractBuildScripts(pkg: Package, requirements: ExtractBuildScri return []; } - const isManifestCompatible = checkAndReportManifestCompatibility(pkg, requirements, `build`, {configuration, report}); + const isManifestCompatible = checkAndReportManifestCompatibility(pkg, `build`, {configuration, report}); if (!isManifestCompatible) return []; diff --git a/packages/yarnpkg-core/package.json b/packages/yarnpkg-core/package.json index d28a81abb1d5..048e40935196 100644 --- a/packages/yarnpkg-core/package.json +++ b/packages/yarnpkg-core/package.json @@ -34,6 +34,7 @@ "stream-to-promise": "^2.2.0", "strip-ansi": "^6.0.0", "tar": "^6.0.5", + "tinylogic": "^1.0.3", "treeify": "^1.1.0", "tslib": "^1.13.0", "tunnel": "^0.0.6" diff --git a/packages/yarnpkg-core/sources/Cache.ts b/packages/yarnpkg-core/sources/Cache.ts index 4177cacd307e..539839ff160a 100644 --- a/packages/yarnpkg-core/sources/Cache.ts +++ b/packages/yarnpkg-core/sources/Cache.ts @@ -14,8 +14,12 @@ import {LocatorHash, Locator} from './ const CACHE_VERSION = 8; -export type FetchFromCacheOptions = { - checksums: Map, +export type CacheOptions = { + mockedPackages?: Set; + unstablePackages?: Set; + + mirrorWriteOnly?: boolean; + skipIntegrityCheck?: boolean; }; export class Cache { @@ -39,7 +43,11 @@ export class Cache { public readonly cacheKey: string; - private mutexes: Map> = new Map(); + private mutexes: Map> = new Map(); /** * To ensure different instances of `Cache` doesn't end up copying to the same @@ -101,10 +109,12 @@ export class Cache { return `${structUtils.slugifyLocator(locator)}-${significantChecksum}.zip` as Filename; } - getLocatorPath(locator: Locator, expectedChecksum: string | null) { + getLocatorPath(locator: Locator, expectedChecksum: string | null, opts: CacheOptions = {}) { // If there is no mirror, then the local cache *is* the mirror, in which - // case we use the versioned filename pattern. - if (this.mirrorCwd === null) + // case we use the versioned filename pattern. Same if the package is + // unstable, meaning it may be there or not depending on the environment, + // so we can't rely on its checksum to get a stable location. + if (this.mirrorCwd === null || opts.unstablePackages?.has(locator.locatorHash)) return ppath.resolve(this.cwd, this.getVersionFilename(locator)); // If we don't yet know the checksum, discard the path resolution for now @@ -148,16 +158,42 @@ export class Cache { } } - async fetchPackageFromCache(locator: Locator, expectedChecksum: string | null, {onHit, onMiss, loader, skipIntegrityCheck}: {onHit?: () => void, onMiss?: () => void, loader?: () => Promise, skipIntegrityCheck?: boolean}): Promise<[FakeFS, () => void, string]> { + async fetchPackageFromCache(locator: Locator, expectedChecksum: string | null, {onHit, onMiss, loader}: {onHit?: () => void, onMiss?: () => void, loader?: () => Promise}, opts: CacheOptions = {}): Promise<[FakeFS, () => void, string | null]> { const mirrorPath = this.getLocatorMirrorPath(locator); const baseFs = new NodeFS(); + // Conditional packages may not be fetched if they're intended for a + // different architecture than the current one. To avoid having to be + // careful about those packages everywhere, we instead change their + // content to that of an empty in-memory package. + // + // This memory representation will be wrapped into an AliasFS to make + // it seem like it actually exist on the disk, at the location of the + // cache the package would fill if it was normally fetched. + const makeMockPackage = () => { + const zipFs = new ZipFS(null, {libzip}); + + const rootPackageDir = ppath.join(PortablePath.root, structUtils.getIdentVendorPath(locator)); + zipFs.mkdirSync(rootPackageDir, {recursive: true}); + zipFs.writeJsonSync(ppath.join(rootPackageDir, Filename.manifest), { + name: structUtils.stringifyIdent(locator), + mocked: true, + }); + + return zipFs; + }; + const validateFile = async (path: PortablePath, refetchPath: PortablePath | null = null) => { - const actualChecksum = (!skipIntegrityCheck || !expectedChecksum) ? `${this.cacheKey}/${await hashUtils.checksumFile(path)}` : expectedChecksum; + const actualChecksum = (!opts.skipIntegrityCheck || !expectedChecksum) + ? `${this.cacheKey}/${await hashUtils.checksumFile(path)}` + : expectedChecksum; if (refetchPath !== null) { - const previousChecksum = (!skipIntegrityCheck || !expectedChecksum) ? `${this.cacheKey}/${await hashUtils.checksumFile(refetchPath)}` : expectedChecksum; + const previousChecksum = (!opts.skipIntegrityCheck || !expectedChecksum) + ? `${this.cacheKey}/${await hashUtils.checksumFile(refetchPath)}` + : expectedChecksum; + if (actualChecksum !== previousChecksum) { throw new ReportError(MessageName.CACHE_CHECKSUM_MISMATCH, `The remote archive doesn't match the local checksum - has the local cache been corrupted?`); } @@ -220,6 +256,7 @@ export class Cache { const loadPackage = async () => { if (!loader) throw new Error(`Cache entry required but missing for ${structUtils.prettyLocator(this.configuration, locator)}`); + if (this.immutable) throw new ReportError(MessageName.IMMUTABLE_CACHE, `Cache entry required but missing for ${structUtils.prettyLocator(this.configuration, locator)}`); @@ -228,32 +265,41 @@ export class Cache { // Do this before moving the file so that we don't pollute the cache with corrupted archives const checksum = await validateFile(packagePath); - const cachePath = this.getLocatorPath(locator, checksum); + const cachePath = this.getLocatorPath(locator, checksum, opts); if (!cachePath) throw new Error(`Assertion failed: Expected the cache path to be available`); - await Promise.all([ - // Copy the package into the mirror - (async () => { - if (packageSource !== `mirror` && mirrorPath !== null) { - const mirrorPathTemp = `${mirrorPath}${this.cacheId}` as PortablePath; - await xfs.copyFilePromise(packagePath, mirrorPathTemp, fs.constants.COPYFILE_FICLONE); - await xfs.chmodPromise(mirrorPathTemp, 0o644); - // Doing a rename is important to ensure the cache is atomic - await xfs.renamePromise(mirrorPathTemp, mirrorPath); - } - })(), - // Copy the package into the cache - (async () => { + const copyProcess: Array<() => Promise> = []; + + // Copy the package into the mirror + if (packageSource !== `mirror` && mirrorPath !== null) { + copyProcess.push(async () => { + const mirrorPathTemp = `${mirrorPath}${this.cacheId}` as PortablePath; + await xfs.copyFilePromise(packagePath, mirrorPathTemp, fs.constants.COPYFILE_FICLONE); + await xfs.chmodPromise(mirrorPathTemp, 0o644); + // Doing a rename is important to ensure the cache is atomic + await xfs.renamePromise(mirrorPathTemp, mirrorPath); + }); + } + + // Copy the package into the cache + if (!opts.mirrorWriteOnly || mirrorPath === null) { + copyProcess.push(async () => { const cachePathTemp = `${cachePath}${this.cacheId}` as PortablePath; await xfs.copyFilePromise(packagePath, cachePathTemp, fs.constants.COPYFILE_FICLONE); await xfs.chmodPromise(cachePathTemp, 0o644); // Doing a rename is important to ensure the cache is atomic await xfs.renamePromise(cachePathTemp, cachePath); - })(), - ]); + }); + } + + const finalPath = opts.mirrorWriteOnly + ? mirrorPath ?? cachePath + : cachePath; + + await Promise.all(copyProcess.map(copy => copy())); - return [cachePath, checksum] as const; + return [false, finalPath, checksum] as const; }; const loadPackageThroughMutex = async () => { @@ -261,30 +307,34 @@ export class Cache { // We don't yet know whether the cache path can be computed yet, since that // depends on whether the cache is actually the mirror or not, and whether // the checksum is known or not. - const tentativeCachePath = this.getLocatorPath(locator, expectedChecksum); + const tentativeCachePath = this.getLocatorPath(locator, expectedChecksum, opts); - const cacheExists = tentativeCachePath !== null + const cacheFileExists = tentativeCachePath !== null ? await baseFs.existsPromise(tentativeCachePath) : false; - const action = cacheExists + const shouldMock = !!opts.mockedPackages?.has(locator.locatorHash) && (!this.check || !cacheFileExists); + const isCacheHit = shouldMock || cacheFileExists; + + const action = isCacheHit ? onHit : onMiss; if (action) action(); - if (!cacheExists) { + if (!isCacheHit) { return loadPackage(); } else { let checksum: string | null = null; const cachePath = tentativeCachePath!; - if (this.check) - checksum = await validateFileAgainstRemote(cachePath); - else - checksum = await validateFile(cachePath); - return [cachePath, checksum] as const; + if (!shouldMock) + checksum = this.check + ? await validateFileAgainstRemote(cachePath) + : await validateFile(cachePath); + + return [shouldMock, cachePath, checksum] as const; } }; @@ -301,15 +351,20 @@ export class Cache { for (let mutex; (mutex = this.mutexes.get(locator.locatorHash));) await mutex; - const [cachePath, checksum] = await loadPackageThroughMutex(); + const [shouldMock, cachePath, checksum] = await loadPackageThroughMutex(); this.markedFiles.add(cachePath); - let zipFs: ZipFS | null = null; + let zipFs: ZipFS | undefined; const libzip = await getLibzipPromise(); - const lazyFs: LazyFS = new LazyFS(() => miscUtils.prettifySyncErrors(() => { - return zipFs = new ZipFS(cachePath, {baseFs, libzip, readOnly: true}); + + const zipFsBuilder = shouldMock + ? () => makeMockPackage() + : () => new ZipFS(cachePath, {baseFs, libzip, readOnly: true}); + + const lazyFs = new LazyFS(() => miscUtils.prettifySyncErrors(() => { + return zipFs = zipFsBuilder(); }, message => { return `Failed to open the cache entry for ${structUtils.prettyLocator(this.configuration, locator)}: ${message}`; }), ppath); @@ -319,12 +374,15 @@ export class Cache { const aliasFs = new AliasFS(cachePath, {baseFs: lazyFs, pathUtils: ppath}); const releaseFs = () => { - if (zipFs !== null) { - zipFs.discardAndClose(); - } + zipFs?.discardAndClose(); }; - return [aliasFs, releaseFs, checksum]; + // We hide the checksum if the package presence is conditional, because it becomes unreliable + const exposedChecksum = !opts.unstablePackages?.has(locator.locatorHash) + ? checksum + : null; + + return [aliasFs, releaseFs, exposedChecksum]; } } diff --git a/packages/yarnpkg-core/sources/Configuration.ts b/packages/yarnpkg-core/sources/Configuration.ts index af266e25b9f5..3d84842cc2eb 100644 --- a/packages/yarnpkg-core/sources/Configuration.ts +++ b/packages/yarnpkg-core/sources/Configuration.ts @@ -76,6 +76,11 @@ export enum SettingsType { MAP = `MAP`, } +export type SupportedArchitectures = { + os: Array | null; + cpu: Array | null; +}; + export type FormatType = formatUtils.Type; export const FormatType = formatUtils.Type; @@ -284,6 +289,26 @@ export const coreDefinitions: {[coreSettingName: string]: SettingsDefinition} = type: SettingsType.BOOLEAN, default: true, }, + supportedArchitectures: { + description: `Architectures that Yarn will fetch and inject into the resolver`, + type: SettingsType.SHAPE, + properties: { + os: { + description: `Array of supported process.platform strings, or null to target them all`, + type: SettingsType.STRING, + isArray: true, + isNullable: true, + default: [`current`], + }, + cpu: { + description: `Array of supported process.arch strings, or null to target them all`, + type: SettingsType.STRING, + isArray: true, + isNullable: true, + default: [`current`], + }, + }, + }, // Settings related to network access enableMirror: { @@ -518,6 +543,7 @@ export interface ConfigurationValueMap { defaultLanguageName: string; defaultProtocol: string; enableTransparentWorkspaces: boolean; + supportedArchitectures: miscUtils.ToMapValue; enableMirror: boolean; enableNetwork: boolean; @@ -1419,6 +1445,20 @@ export class Configuration { return linkers; } + getSupportedArchitectures() { + const supportedArchitectures = this.get(`supportedArchitectures`); + + let os = supportedArchitectures.get(`os`); + if (os !== null) + os = os.map(value => value === `current` ? process.platform : value); + + let cpu = supportedArchitectures.get(`cpu`); + if (cpu !== null) + cpu = cpu.map(value => value === `current` ? process.arch : value); + + return {os, cpu}; + } + async refreshPackageExtensions() { this.packageExtensions = new Map(); const packageExtensions = this.packageExtensions; diff --git a/packages/yarnpkg-core/sources/Fetcher.ts b/packages/yarnpkg-core/sources/Fetcher.ts index 9c8ab83170d2..6963c736ce68 100644 --- a/packages/yarnpkg-core/sources/Fetcher.ts +++ b/packages/yarnpkg-core/sources/Fetcher.ts @@ -1,6 +1,6 @@ import {FakeFS, PortablePath} from '@yarnpkg/fslib'; -import {Cache} from './Cache'; +import {Cache, CacheOptions} from './Cache'; import {Project} from './Project'; import {Report} from './Report'; import {LocatorHash, Locator} from './types'; @@ -12,6 +12,7 @@ export type MinimalFetchOptions = { export type FetchOptions = MinimalFetchOptions & { cache: Cache, + cacheOptions?: CacheOptions, checksums: Map, report: Report, skipIntegrityCheck?: boolean @@ -41,7 +42,7 @@ export type FetchResult = { /** * The checksum for the fetch result. */ - checksum?: string, + checksum?: string | null, /** * If true, the package location won't be considered for package lookups (so diff --git a/packages/yarnpkg-core/sources/Manifest.ts b/packages/yarnpkg-core/sources/Manifest.ts index 2893bd04e358..e28cfae955f5 100644 --- a/packages/yarnpkg-core/sources/Manifest.ts +++ b/packages/yarnpkg-core/sources/Manifest.ts @@ -658,6 +658,17 @@ export class Manifest { return false; } + getConditions() { + const fields: Array = []; + + if (this.os && this.os.length > 0) + fields.push(toConditionLine(`os`, this.os)); + if (this.cpu && this.cpu.length > 0) + fields.push(toConditionLine(`cpu`, this.cpu)); + + return fields.length > 0 ? fields.join(` & `) : null; + } + isCompatibleWithOS(os: string): boolean { return Manifest.isManifestFieldCompatible(this.os, os); } @@ -976,3 +987,22 @@ function tryParseOptionalBoolean(value: unknown, {yamlCompatibilityMode}: {yamlC return null; } + +function toConditionToken(name: string, raw: string) { + const index = raw.search(/[^!]/); + if (index === -1) + return `invalid`; + + const prefix = index % 2 === 0 ? `` : `!`; + const value = raw.slice(index); + + return `${prefix}${name}=${value}`; +} + +function toConditionLine(name: string, rawTokens: Array) { + if (rawTokens.length === 1) { + return toConditionToken(name, rawTokens[0]); + } else { + return `(${rawTokens.map(raw => toConditionToken(name, raw)).join(` | `)})`; + } +} diff --git a/packages/yarnpkg-core/sources/MessageName.ts b/packages/yarnpkg-core/sources/MessageName.ts index 72f1206b1cb0..82bd9c8cda6c 100644 --- a/packages/yarnpkg-core/sources/MessageName.ts +++ b/packages/yarnpkg-core/sources/MessageName.ts @@ -78,6 +78,8 @@ export enum MessageName { UPDATE_LOCKFILE_ONLY_SKIP_LINK = 73, NM_HARDLINKS_MODE_DOWNGRADED = 74, PROLOG_INSTANTIATION_ERROR = 75, + INCOMPATIBLE_ARCHITECTURE = 76, + GHOST_ARCHITECTURE = 77, } export function stringifyMessageName(name: MessageName | number): string { diff --git a/packages/yarnpkg-core/sources/Project.ts b/packages/yarnpkg-core/sources/Project.ts index 9509c4049111..23eba13248c5 100644 --- a/packages/yarnpkg-core/sources/Project.ts +++ b/packages/yarnpkg-core/sources/Project.ts @@ -12,7 +12,7 @@ import v8 from 'v8 import zlib from 'zlib'; import {Cache} from './Cache'; -import {Configuration} from './Configuration'; +import {Configuration, FormatType} from './Configuration'; import {Fetcher} from './Fetcher'; import {Installer, BuildDirective, BuildType, InstallStatus} from './Installer'; import {LegacyMigrationResolver} from './LegacyMigrationResolver'; @@ -41,7 +41,7 @@ import {IdentHash, DescriptorHash, LocatorHash, PackageExtensionStatus} from './ // When upgraded, the lockfile entries have to be resolved again (but the specific // versions are still pinned, no worry). Bump it when you change the fields within // the Package type; no more no less. -const LOCKFILE_VERSION = 4; +const LOCKFILE_VERSION = 5; // Same thing but must be bumped when the members of the Project class changes (we // don't recommend our users to check-in this file, so it's fine to bump it even @@ -137,6 +137,8 @@ const INSTALL_STATE_FIELDS = { restoreResolutions: [ `accessibleLocators`, + `conditionalLocators`, + `disabledLocators`, `optionalBuilds`, `storedDescriptors`, `storedResolutions`, @@ -194,6 +196,8 @@ export class Project { public storedBuildState: Map = new Map(); public accessibleLocators: Set = new Set(); + public conditionalLocators: Set = new Set(); + public disabledLocators: Set = new Set(); public originalPackages: Map = new Map(); public optionalBuilds: Set = new Set(); @@ -317,6 +321,8 @@ export class Project { const languageName = manifest.languageName || defaultLanguageName; const linkType = data.linkType.toUpperCase() as LinkType; + const conditions = data.conditions ?? null; + const dependencies = manifest.dependencies; const peerDependencies = manifest.peerDependencies; @@ -333,7 +339,7 @@ export class Project { this.storedChecksums.set(locator.locatorHash, checksum); } - const pkg: Package = {...locator, version, languageName, linkType, dependencies, peerDependencies, dependenciesMeta, peerDependenciesMeta, bin}; + const pkg: Package = {...locator, version, languageName, linkType, conditions, dependencies, peerDependencies, dependenciesMeta, peerDependenciesMeta, bin}; this.originalPackages.set(pkg.locatorHash, pkg); for (const entry of key.split(MULTIPLE_KEYS_REGEXP)) { @@ -676,7 +682,7 @@ export class Project { const resolveOptions: ResolveOptions = opts.lockfileOnly ? {project: this, report: opts.report, resolver} - : {project: this, report: opts.report, resolver, fetchOptions: {project: this, cache: opts.cache, checksums: this.storedChecksums, report: opts.report, fetcher}}; + : {project: this, report: opts.report, resolver, fetchOptions: {project: this, cache: opts.cache, checksums: this.storedChecksums, report: opts.report, fetcher, cacheOptions: {mirrorWriteOnly: true}}}; const allDescriptors = new Map(); const allPackages = new Map(); @@ -688,6 +694,7 @@ export class Project { const descriptorResolutionPromises = new Map>(); const dependencyResolutionLocator = this.topLevelWorkspace.anchoredLocator; + const allResolutionDependencyPackages = new Set(); const resolutionQueue: Array> = []; @@ -756,7 +763,11 @@ export class Project { const resolutionDependencies = resolver.getResolutionDependencies(descriptor, resolveOptions); const resolvedDependencies = new Map(await Promise.all(resolutionDependencies.map(async dependency => { const bound = resolver.bindDescriptor(dependency, dependencyResolutionLocator, resolveOptions); - return [dependency.descriptorHash, await scheduleDescriptorResolution(bound)] as const; + + const resolvedPackage = await scheduleDescriptorResolution(bound); + allResolutionDependencyPackages.add(resolvedPackage.locatorHash); + + return [dependency.descriptorHash, resolvedPackage] as const; }))); const candidateResolutions = await miscUtils.prettifyAsyncErrors(async () => { @@ -822,6 +833,9 @@ export class Project { allPackages, }); + for (const locatorHash of allResolutionDependencyPackages) + optionalBuilds.delete(locatorHash); + // All descriptors still referenced within the volatileDescriptors set are // descriptors that aren't depended upon by anything in the dependency tree. @@ -830,6 +844,37 @@ export class Project { allResolutions.delete(descriptorHash); } + const supportedArchitectures = this.configuration.getSupportedArchitectures(); + + const conditionalLocators = new Set(); + const disabledLocators = new Set(); + + for (const pkg of allPackages.values()) { + if (pkg.conditions === null) + continue; + + if (!optionalBuilds.has(pkg.locatorHash)) + continue; + + if (!structUtils.isPackageCompatible(pkg, supportedArchitectures)) { + if (structUtils.isPackageCompatible(pkg, {os: [process.platform], cpu: [process.arch]})) { + opts.report.reportWarningOnce(MessageName.GHOST_ARCHITECTURE, `${ + structUtils.prettyLocator(this.configuration, pkg) + }: Your current architecture (${ + process.platform + }-${ + process.arch + }) is supported by this package, but is missing from the ${ + formatUtils.pretty(this.configuration, `supportedArchitectures`, FormatType.SETTING) + } setting`); + } + + disabledLocators.add(pkg.locatorHash); + } + + conditionalLocators.add(pkg.locatorHash); + } + // Everything is done, we can now update our internal resolutions to // reference the new ones @@ -838,6 +883,8 @@ export class Project { this.storedPackages = allPackages; this.accessibleLocators = accessibleLocators; + this.conditionalLocators = conditionalLocators; + this.disabledLocators = disabledLocators; this.originalPackages = originalPackages; this.optionalBuilds = optionalBuilds; this.peerRequirements = peerRequirements; @@ -849,8 +896,13 @@ export class Project { } async fetchEverything({cache, report, fetcher: userFetcher, mode}: InstallOptions) { + const cacheOptions = { + mockedPackages: this.disabledLocators, + unstablePackages: this.conditionalLocators, + }; + const fetcher = userFetcher || this.configuration.makeFetcher(); - const fetcherOptions = {checksums: this.storedChecksums, project: this, cache, fetcher, report}; + const fetcherOptions = {checksums: this.storedChecksums, project: this, cache, fetcher, report, cacheOptions}; let locatorHashes = Array.from( new Set( @@ -898,7 +950,7 @@ export class Project { return; } - if (fetchResult.checksum) + if (fetchResult.checksum != null) this.storedChecksums.set(pkg.locatorHash, fetchResult.checksum); else this.storedChecksums.delete(pkg.locatorHash); @@ -917,8 +969,14 @@ export class Project { } async linkEverything({cache, report, fetcher: optFetcher, mode}: InstallOptions) { + const cacheOptions = { + mockedPackages: this.disabledLocators, + unstablePackages: this.conditionalLocators, + skipIntegrityCheck: true, + }; + const fetcher = optFetcher || this.configuration.makeFetcher(); - const fetcherOptions = {checksums: this.storedChecksums, project: this, cache, fetcher, report, skipIntegrityCheck: true}; + const fetcherOptions = {checksums: this.storedChecksums, project: this, cache, fetcher, report, cacheOptions}; const linkers = this.configuration.getLinkers(); const linkerOptions = {project: this, report}; @@ -1540,6 +1598,7 @@ export class Project { optimizedLockfile.__metadata = { version: LOCKFILE_VERSION, + cacheKey: undefined, }; for (const [locatorHash, descriptorHashes] of reverseLookup.entries()) { @@ -1586,7 +1645,7 @@ export class Project { if (typeof checksum !== `undefined`) { const cacheKeyIndex = checksum.indexOf(`/`); if (cacheKeyIndex === -1) - throw new Error(`Assertion failed: Expecte the checksum to reference its cache key`); + throw new Error(`Assertion failed: Expected the checksum to reference its cache key`); const cacheKey = checksum.slice(0, cacheKeyIndex); const hash = checksum.slice(cacheKeyIndex + 1); @@ -1610,6 +1669,8 @@ export class Project { resolution: structUtils.stringifyLocator(pkg), checksum: entryChecksum, + + conditions: pkg.conditions || undefined, }; } diff --git a/packages/yarnpkg-core/sources/WorkspaceResolver.ts b/packages/yarnpkg-core/sources/WorkspaceResolver.ts index 47a6dfdf34f0..2c36891140e7 100644 --- a/packages/yarnpkg-core/sources/WorkspaceResolver.ts +++ b/packages/yarnpkg-core/sources/WorkspaceResolver.ts @@ -58,6 +58,8 @@ export class WorkspaceResolver implements Resolver { languageName: `unknown`, linkType: LinkType.SOFT, + conditions: null, + dependencies: new Map([...workspace.manifest.dependencies, ...workspace.manifest.devDependencies]), peerDependencies: new Map([...workspace.manifest.peerDependencies]), diff --git a/packages/yarnpkg-core/sources/structUtils.ts b/packages/yarnpkg-core/sources/structUtils.ts index fd35b4bd25bd..df13e5b8c1be 100644 --- a/packages/yarnpkg-core/sources/structUtils.ts +++ b/packages/yarnpkg-core/sources/structUtils.ts @@ -1,6 +1,7 @@ import {PortablePath, toFilename} from '@yarnpkg/fslib'; import querystring from 'querystring'; import semver from 'semver'; +import {makeParser} from 'tinylogic'; import {Configuration} from './Configuration'; import {Workspace} from './Workspace'; @@ -14,6 +15,9 @@ import {Ident, Descriptor, Locator, Package} from './types'; const VIRTUAL_PROTOCOL = `virtual:`; const VIRTUAL_ABBREVIATE = 5; +const conditionRegex = /(os|cpu)=([a-z0-9_-]+)/; +const conditionParser = makeParser(conditionRegex); + /** * Creates a package ident. * @@ -120,6 +124,8 @@ export function renamePackage(pkg: Package, locator: Locator): Package { languageName: pkg.languageName, linkType: pkg.linkType, + conditions: pkg.conditions, + dependencies: new Map(pkg.dependencies), peerDependencies: new Map(pkg.peerDependencies), @@ -801,3 +807,18 @@ export function prettyDependent(configuration: Configuration, locator: Locator, export function getIdentVendorPath(ident: Ident) { return `node_modules/${stringifyIdent(ident)}` as PortablePath; } + +/** + * Returns whether the given package is compatible with the specified environment. + */ +export function isPackageCompatible(pkg: Package, architectures: {os: Array | null, cpu: Array | null}) { + if (!pkg.conditions) + return true; + + return conditionParser(pkg.conditions, specifier => { + const [, name, value] = specifier.match(conditionRegex)!; + const supported = architectures[name as keyof typeof architectures]; + + return supported ? supported.includes(value) : true; + }); +} diff --git a/packages/yarnpkg-core/sources/types.ts b/packages/yarnpkg-core/sources/types.ts index 41072c9db305..d59e8c469da8 100644 --- a/packages/yarnpkg-core/sources/types.ts +++ b/packages/yarnpkg-core/sources/types.ts @@ -133,6 +133,12 @@ export interface Package extends Locator { */ linkType: LinkType, + /** + * A set of constraints indicating whether the package supports the host + * environment. + */ + conditions: string | null, + /** * A map of the package's dependencies. There's no distinction between prod * dependencies and dev dependencies, because those have already been merged diff --git a/packages/yarnpkg-core/tests/TestPlugin.ts b/packages/yarnpkg-core/tests/TestPlugin.ts index 1581b3581cdc..d44653b129f5 100644 --- a/packages/yarnpkg-core/tests/TestPlugin.ts +++ b/packages/yarnpkg-core/tests/TestPlugin.ts @@ -46,6 +46,8 @@ export class UnboundDescriptorResolver implements Resolver { languageName: `node`, linkType: LinkType.HARD, + conditions: null, + dependencies: new Map(), peerDependencies: new Map(), @@ -104,6 +106,8 @@ export class ResolutionDependencyResolver implements Resolver { languageName: `node`, linkType: LinkType.HARD, + conditions: null, + dependencies: new Map(), peerDependencies: new Map(), diff --git a/yarn.lock b/yarn.lock index 0ebabb62ecb6..704bd12110df 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # Manual changes might be lost - proceed with caution! __metadata: - version: 4 + version: 5 cacheKey: 8 "@actions/core@npm:^1.2.6": @@ -1729,6 +1729,20 @@ __metadata: languageName: node linkType: hard +"@chevrotain/types@npm:^9.1.0": + version: 9.1.0 + resolution: "@chevrotain/types@npm:9.1.0" + checksum: 5f26ff26aa345bc823b39ebe48f038db0998c80d13fa4b937961d68523a259ac86ec693bc1ad3f3cfa9ef83e5ffb6d94337960c3a1ee7cb82e3014adb4f5bf30 + languageName: node + linkType: hard + +"@chevrotain/utils@npm:^9.1.0": + version: 9.1.0 + resolution: "@chevrotain/utils@npm:9.1.0" + checksum: ca78c97c7c3e444431d0fafa348f0c955998cd86bc0d4bbdeaae3ff5abba8d416d69d5a4163e86cac962a392f1c325cb4a97b8b05722527da62e9b7635025e02 + languageName: node + linkType: hard + "@cnakazawa/watch@npm:^1.0.3": version: 1.0.3 resolution: "@cnakazawa/watch@npm:1.0.3" @@ -5371,6 +5385,7 @@ __metadata: stream-to-promise: ^2.2.0 strip-ansi: ^6.0.0 tar: ^6.0.5 + tinylogic: ^1.0.3 treeify: ^1.1.0 tslib: ^1.13.0 tunnel: ^0.0.6 @@ -8295,6 +8310,17 @@ __metadata: languageName: node linkType: hard +"chevrotain@npm:^9.1.0": + version: 9.1.0 + resolution: "chevrotain@npm:9.1.0" + dependencies: + "@chevrotain/types": ^9.1.0 + "@chevrotain/utils": ^9.1.0 + regexp-to-ast: 0.5.0 + checksum: 632d0d7c69081e3cc3a08c071cb738c46499a05f1a513b7f9101f7a9b5570d6ee62cac5ba506659a85bf9e71e1029c462dbb7bd9fe1bfe019b6c1879ca29c525 + languageName: node + linkType: hard + "chokidar@npm:^2.1.8": version: 2.1.8 resolution: "chokidar@npm:2.1.8" @@ -12505,6 +12531,7 @@ fsevents@^1.2.7: nan: ^2.9.2 node-pre-gyp: ^0.10.0 checksum: f291742931e9fe14f1b1346c98a36be561e9aa769a66424ceed8713c3fd35f5fee7cdd1c07608afa18811e06cbdfd92b897bb82dc994a4060e013bbeecb3fd7c + conditions: os=darwin languageName: node linkType: hard @@ -12514,6 +12541,7 @@ fsevents@^1.2.7: dependencies: node-gyp: latest checksum: 97ade64e75091afee5265e6956cb72ba34db7819b4c3e94c431d4be2b19b8bb7a2d4116da417950c3425f17c8fe693d25e20212cac583ac1521ad066b77ae31f + conditions: os=darwin languageName: node linkType: hard @@ -12523,7 +12551,7 @@ fsevents@^1.2.7: dependencies: nan: ^2.9.2 node-pre-gyp: ^0.10.0 - checksum: 94a10c4d030ef662e75ea916ab326901e71e502250f1b8ab7d1902872cbbd576fcf93b2228e7e46bc02f33be0c6be9d2bf7f68cb913e20ecfe23cae178f003d0 + conditions: os=darwin languageName: node linkType: hard @@ -12532,7 +12560,7 @@ fsevents@^1.2.7: resolution: "fsevents@patch:fsevents@npm%3A2.3.2#~builtin::version=2.3.2&hash=18f3a7" dependencies: node-gyp: latest - checksum: edbd0fd80be379c14409605f77e52fdc78a119e17f875e8b90a220c3e5b29e54a1477c21d91fd30b957ea4866406dc3ff87b61432d2840ff8866b309e5866140 + conditions: os=darwin languageName: node linkType: hard @@ -22207,6 +22235,13 @@ fsevents@^1.2.7: languageName: node linkType: hard +"regexp-to-ast@npm:0.5.0": + version: 0.5.0 + resolution: "regexp-to-ast@npm:0.5.0" + checksum: 72e32f2a1217bb22398ac30867ddd43e16943b6b569dd4eb472de47494c7a39e34f47ee3e92ad4cbf92308f98997da366fe094a0e58eb6b93eab0ee956fff86d + languageName: node + linkType: hard + "regexp.prototype.flags@npm:^1.2.0, regexp.prototype.flags@npm:^1.3.1": version: 1.3.1 resolution: "regexp.prototype.flags@npm:1.3.1" @@ -25030,6 +25065,15 @@ resolve@^2.0.0-next.3: languageName: node linkType: hard +"tinylogic@npm:^1.0.3": + version: 1.0.3 + resolution: "tinylogic@npm:1.0.3" + dependencies: + chevrotain: ^9.1.0 + checksum: fdf7fcc170050889b210fd035b1eb2ac81a68d1324010a427eeee53ac49613ecaa3fbd33b41adb1264dfb02b4d500b3f442da1db3ffc53834c654345c1658afa + languageName: node + linkType: hard + "tmp@npm:0.0.29": version: 0.0.29 resolution: "tmp@npm:0.0.29"