diff --git a/crates/turborepo-lockfiles/fixtures/yarn4-direct-and-indirect.lock b/crates/turborepo-lockfiles/fixtures/yarn4-direct-and-indirect.lock new file mode 100644 index 0000000000000..d5637b6f38dac --- /dev/null +++ b/crates/turborepo-lockfiles/fixtures/yarn4-direct-and-indirect.lock @@ -0,0 +1,78 @@ +# This file is generated by running "yarn install" inside your project. +# Manual changes might be lost - proceed with caution! + +__metadata: + version: 8 + cacheKey: 10 + +"a@workspace:packages/a": + version: 0.0.0-use.local + resolution: "a@workspace:packages/a" + dependencies: + is-odd: "patch:is-odd@npm%3A3.0.1#~/.yarn/patches/is-odd-npm-3.0.1-93c3c3f41b.patch" + languageName: unknown + linkType: soft + +"b@workspace:packages/b": + version: 0.0.0-use.local + resolution: "b@workspace:packages/b" + dependencies: + is-even: "npm:^1.0.0" + languageName: unknown + linkType: soft + +"is-even@npm:^1.0.0": + version: 1.0.0 + resolution: "is-even@npm:1.0.0" + dependencies: + is-odd: "npm:^0.1.2" + checksum: 0267545d7cb6724aee249e88942cf22f6263aa006cd9bf83c2ddbb2a1d47280e8c4d72b2d50e38bd3575df717c993904b44153cc1772a55dabca250ca40cc4f7 + languageName: node + linkType: hard + +"is-number@npm:^3.0.0": + version: 3.0.0 + resolution: "is-number@npm:3.0.0" + checksum: 0c62bf8e9d72c4dd203a74d8cfc751c746e75513380fef420cda8237e619a988ee43e678ddb23c87ac24d91ac0fe9f22e4ffb1301a50310c697e9d73ca3994e9 + languageName: node + linkType: hard + +"is-number@npm:^6.0.0": + version: 6.0.0 + resolution: "is-number@npm:6.0.0" + checksum: 8668b49747649ee0878e0a6d9e35e1c95bab58c5dd1a2b698df34989512ec553cfd090fa7af247d590afe4b12ec996e735b9a670c2cf0efbaefe7fb2c2457615 + languageName: node + linkType: hard + +"is-odd@npm:3.0.1": + version: 3.0.1 + resolution: "is-odd@npm:3.0.1" + dependencies: + is-number: "npm:^6.0.0" + checksum: c9d35c336c0c0ada0bfaf1f4564f354a222c4ffb9c3b42fac353767c9b8f0af844d3ddf16fbf7b12d6ecf57ee4d2fbeb9e456e8c9d68a78bb44e91bb43fdfd56 + languageName: node + linkType: hard + +"is-odd@npm:^0.1.2": + version: 0.1.2 + resolution: "is-odd@npm:0.1.2" + dependencies: + is-number: "npm:^3.0.0" + checksum: 146069d7622c991c75c17ca63bccf5470cd730c24082874e53e797a10ff38a896197d6ce34ad137a2f422dcc614b10ff24d31fe93dcdb29f0cb758f2d924f477 + languageName: node + linkType: hard + +"is-odd@patch:is-odd@npm%3A3.0.1#~/.yarn/patches/is-odd-npm-3.0.1-93c3c3f41b.patch": + version: 3.0.1 + resolution: "is-odd@patch:is-odd@npm%3A3.0.1#~/.yarn/patches/is-odd-npm-3.0.1-93c3c3f41b.patch::version=3.0.1&hash=496de7" + dependencies: + is-number: "npm:^6.0.0" + checksum: 8da62f4b41d59d4c15e90b7dc619b1312668421b982908c3792324bb0622128c1b8d484b96ccd15c05eab50dcdef64c169686c2dba92b925914c4c85ce4300f9 + languageName: node + linkType: hard + +"small-yarn4@workspace:.": + version: 0.0.0-use.local + resolution: "small-yarn4@workspace:." + languageName: unknown + linkType: soft diff --git a/crates/turborepo-lockfiles/fixtures/yarn4-patch.lock b/crates/turborepo-lockfiles/fixtures/yarn4-patch.lock new file mode 100644 index 0000000000000..b6542f802f3d1 --- /dev/null +++ b/crates/turborepo-lockfiles/fixtures/yarn4-patch.lock @@ -0,0 +1,60 @@ +# This file is generated by running "yarn install" inside your project. +# Manual changes might be lost - proceed with caution! + +__metadata: + version: 8 + cacheKey: 10c0 + +"a@workspace:packages/a": + version: 0.0.0-use.local + resolution: "a@workspace:packages/a" + dependencies: + c: "workspace:*" + languageName: unknown + linkType: soft + +"b@workspace:packages/b": + version: 0.0.0-use.local + resolution: "b@workspace:packages/b" + dependencies: + c: "workspace:*" + is-odd: "patch:is-odd@npm%3A3.0.1#~/.yarn/patches/is-odd-npm-3.0.1-93c3c3f41b.patch" + languageName: unknown + linkType: soft + +"c@workspace:*, c@workspace:packages/c": + version: 0.0.0-use.local + resolution: "c@workspace:packages/c" + languageName: unknown + linkType: soft + +"is-number@npm:^6.0.0": + version: 6.0.0 + resolution: "is-number@npm:6.0.0" + checksum: 5da4c68401529675c575878d2760d66f18eaef4b014858577f6003daf66488d7fe4eae684b1e8574e3fa1bb447c6c6c56b8491d2b4b3239da2d32e5f6f218008 + languageName: node + linkType: hard + +"is-odd@npm:3.0.1": + version: 3.0.1 + resolution: "is-odd@npm:3.0.1" + dependencies: + is-number: "npm:^6.0.0" + checksum: 89ee2e353c5a3f3bd400c79db1c307a5b3506198ee8169d521e533a9b1d8a08fc95f21a919c084e98845b4286d7ffe309778da03744dfe66c3c1763ab1a030c6 + languageName: node + linkType: hard + +"is-odd@patch:is-odd@npm%3A3.0.1#~/.yarn/patches/is-odd-npm-3.0.1-93c3c3f41b.patch": + version: 3.0.1 + resolution: "is-odd@patch:is-odd@npm%3A3.0.1#~/.yarn/patches/is-odd-npm-3.0.1-93c3c3f41b.patch::version=3.0.1&hash=9b90ad" + dependencies: + is-number: "npm:^6.0.0" + checksum: 4cd944e688e02e147969d6c1784bad1156f6084edbbd4d688f6a37b5fc764671aa99679494fc0bfaf623919bea2779e724fffc31c6ee0432b7c91f174526e5fe + languageName: node + linkType: hard + +"yarn4-patch@workspace:.": + version: 0.0.0-use.local + resolution: "yarn4-patch@workspace:." + languageName: unknown + linkType: soft diff --git a/crates/turborepo-lockfiles/src/berry/identifiers.rs b/crates/turborepo-lockfiles/src/berry/identifiers.rs index e15a8d235e073..2eaed5aa75dec 100644 --- a/crates/turborepo-lockfiles/src/berry/identifiers.rs +++ b/crates/turborepo-lockfiles/src/berry/identifiers.rs @@ -25,7 +25,7 @@ fn multikey() -> &'static Regex { fn builtin() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| Regex::new(r"^builtin<([^>]+)>$").unwrap()) + RE.get_or_init(|| Regex::new(r"^(?:optional!)?builtin<([^>]+)>$").unwrap()) } #[derive(Debug, Error)] @@ -243,7 +243,10 @@ impl<'a> Locator<'a> { .and_then(|caps| caps.get(2)) .map(|m| { let s = m.as_str(); - s.strip_prefix("./").unwrap_or(s) + s.strip_prefix("./") + // Yarn 4 uses ~ to indicate the yarn root + .or_else(|| s.strip_prefix("~/")) + .unwrap_or(s) }) } diff --git a/crates/turborepo-lockfiles/src/berry/mod.rs b/crates/turborepo-lockfiles/src/berry/mod.rs index 835e73307a096..e8462a3785f3e 100644 --- a/crates/turborepo-lockfiles/src/berry/mod.rs +++ b/crates/turborepo-lockfiles/src/berry/mod.rs @@ -39,6 +39,8 @@ pub enum Error { descriptor: Descriptor<'static>, other: String, }, + #[error("unable to parse as patch reference: {0}")] + InvalidPatchReference(String), } // We depend on BTree iteration being sorted for correct serialization @@ -181,6 +183,14 @@ impl BerryLockfile { } possible_extensions.remove(&descriptor); } + + // For Yarn 4, remove any patch sources that are accounted for by a patch + if let Some(Locator { ident, reference }) = locator.patched_locator() { + possible_extensions.remove(&Descriptor { + ident, + range: reference, + }); + } } self.extensions.extend( @@ -249,7 +259,7 @@ impl BerryLockfile { /// Produces a new lockfile containing only the given workspaces and /// packages - pub fn subgraph( + fn subgraph( &self, workspace_packages: &[String], packages: &[String], @@ -306,6 +316,26 @@ impl BerryLockfile { if let Some(patch_locator) = self.patches.get(&locator) { patches.insert(locator.as_owned(), patch_locator.clone()); } + + // Yarn 4 allows workspaces to depend directly on patched dependencies instead + // of using resolutions. This results in the patched dependency appearing in the + // closure instead of the original. + if locator.patch_file().is_some() { + if let Some((original, _)) = + self.patches.iter().find(|(_, patch)| patch == &&locator) + { + patches.insert(original.as_owned(), locator.as_owned()); + // We include the patched dependency resolution + let Locator { ident, reference } = original.as_owned(); + resolutions.insert( + Descriptor { + ident, + range: reference, + }, + original.as_owned(), + ); + } + } } for patch in patches.values() { @@ -461,103 +491,8 @@ impl Lockfile for BerryLockfile { workspace_packages: &[String], packages: &[String], ) -> Result, crate::Error> { - let reverse_lookup = self.locator_to_descriptors(); - - let mut resolutions = Map::new(); - let mut patches = Map::new(); - - // Include all workspace packages and their references - for (locator, package) in &self.locator_package { - if workspace_packages - .iter() - .map(|s| s.as_str()) - .chain(iter::once(".")) - .any(|path| locator.is_workspace_path(path)) - { - // We need to track all of the descriptors coming out the workspace - for (name, range) in package.dependencies.iter().flatten() { - let dependency = self.resolve_dependency(locator, name, range.as_ref())?; - let dep_locator = self - .resolutions - .get(&dependency) - .ok_or_else(|| Error::MissingLocator(dependency.clone().into_owned()))?; - resolutions.insert(dependency, dep_locator.clone()); - } - - // Included workspaces will always have their locator listed as a descriptor. - // All other descriptors should show up in the other workspace package - // dependencies. - resolutions.insert(Descriptor::from(locator.clone()), locator.clone()); - } - } - - for key in packages { - // The error mapping is required to help massage the error types - let locator = Locator::try_from(key.as_str()).map_err(Error::from)?; - - let package = self - .locator_package - .get(&locator) - .cloned() - .ok_or_else(|| Error::MissingPackageForLocator(locator.as_owned()))?; - - for (name, range) in package.dependencies.iter().flatten() { - let dependency = self.resolve_dependency(&locator, name, range.as_ref())?; - let dep_locator = self - .resolutions - .get(&dependency) - .ok_or_else(|| Error::MissingLocator(dependency.clone().into_owned()))?; - resolutions.insert(dependency, dep_locator.clone()); - } - - // If the package has an associated patch we include it in the subgraph - if let Some(patch_locator) = self.patches.get(&locator) { - patches.insert(locator.as_owned(), patch_locator.clone()); - } - } - - for patch in patches.values() { - let patch_descriptors = reverse_lookup - .get(patch) - .unwrap_or_else(|| panic!("Unable to find {patch} in reverse lookup")); - - // For each patch descriptor we extract the primary descriptor that each patch - // descriptor targets and check if that descriptor is present in the - // pruned map and add it if it is present - for patch_descriptor in patch_descriptors { - let version = patch_descriptor.primary_version().unwrap(); - let primary_descriptor = Descriptor { - ident: patch_descriptor.ident.clone(), - range: version.into(), - }; - - if resolutions.contains_key(&primary_descriptor) { - resolutions.insert((*patch_descriptor).clone(), patch.clone()); - } - } - } - - // Add any descriptors used by package extensions - for descriptor in &self.extensions { - let locator = self - .resolutions - .get(descriptor) - .ok_or_else(|| Error::MissingLocator(descriptor.to_owned()))?; - resolutions.insert(descriptor.clone(), locator.clone()); - } - - Ok(Box::new(Self { - data: self.data.clone(), - resolutions, - patches, - // We clone the following structures without any alterations and - // rely on resolutions being correctly pruned. - locator_package: self.locator_package.clone(), - resolver: self.resolver.clone(), - extensions: self.extensions.clone(), - overrides: self.overrides.clone(), - workspace_path_to_locator: self.workspace_path_to_locator.clone(), - })) + let subgraph = self.subgraph(workspace_packages, packages)?; + Ok(Box::new(subgraph)) } fn encode(&self) -> Result, crate::Error> { @@ -756,7 +691,7 @@ mod test { assert_eq!( &lockfile.extensions, - &(["@babel/types@npm:^7.8.3", "lodash@npm:4.17.21"] + &(["@babel/types@npm:^7.8.3"] .iter() .map(|s| Descriptor::try_from(*s).unwrap()) .collect::>()) @@ -1048,4 +983,124 @@ mod test { .unwrap(); assert_eq!(c_pkg.key, "c@workspace:pkgs/c"); } + + #[test] + fn test_yarn4_patches_direct_dependency() { + let data = + LockfileData::from_bytes(include_bytes!("../../fixtures/yarn4-patch.lock")).unwrap(); + let lockfile = BerryLockfile::new(data, None).unwrap(); + + let is_odd_locator = lockfile + .resolve_package( + "packages/b", + "is-odd", + "patch:is-odd@npm%3A3.0.1#~/.yarn/patches/is-odd-npm-3.0.1-93c3c3f41b.patch", + ) + .unwrap() + .unwrap(); + + let expected_key = "is-odd@patch:is-odd@npm%3A3.0.1#~/.yarn/patches/is-odd-npm-3.0.\ + 1-93c3c3f41b.patch::version=3.0.1&hash=9b90ad"; + + assert_eq!( + is_odd_locator, + crate::Package { + key: expected_key.into(), + version: "3.0.1".into(), + } + ); + + let deps = crate::transitive_closure( + &lockfile, + "packages/b", + vec![( + "is-odd".to_string(), + "patch:is-odd@npm%3A3.0.1#~/.yarn/patches/is-odd-npm-3.0.1-93c3c3f41b.patch" + .to_string(), + )] + .into_iter() + .collect(), + ) + .unwrap(); + + assert_eq!( + deps, + vec![ + crate::Package { + key: expected_key.into(), + version: "3.0.1".into() + }, + crate::Package { + key: "is-number@npm:6.0.0".into(), + version: "6.0.0".into() + } + ] + .into_iter() + .collect() + ); + + let subgraph = lockfile + .subgraph( + &["packages/b".into(), "packages/c".into()], + &[expected_key.into(), "is-number@npm:6.0.0".into()], + ) + .unwrap(); + + let sublockfile = subgraph.lockfile().unwrap(); + + // Should contain both patched dependency and original + assert!(sublockfile.packages.contains_key("is-odd@npm:3.0.1")); + assert!(sublockfile.packages.contains_key( + "is-odd@patch:is-odd@npm%3A3.0.1#~/.yarn/patches/is-odd-npm-3.0.1-93c3c3f41b.patch" + )); + + let patches = + vec![ + RelativeUnixPathBuf::new(".yarn/patches/is-odd-npm-3.0.1-93c3c3f41b.patch") + .unwrap(), + ]; + assert_eq!(lockfile.patches().unwrap(), patches); + assert_eq!(subgraph.patches().unwrap(), patches); + } + + #[test] + fn test_yarn4_patches_direct_and_indirect_dependency() { + let data = LockfileData::from_bytes(include_bytes!( + "../../fixtures/yarn4-direct-and-indirect.lock" + )) + .unwrap(); + let lockfile = BerryLockfile::new(data, None).unwrap(); + + let b_closure = lockfile + .subgraph( + &["packages/b".into()], + &[ + "is-even@npm:1.0.0".into(), + "is-odd@npm:0.1.2".into(), + "is-number@npm:3.0.0".into(), + ], + ) + .unwrap(); + + assert_eq!(b_closure.patches().unwrap(), vec![]); + + let mut locators = b_closure + .lockfile() + .unwrap() + .packages + .values() + .map(|package| package.resolution.clone()) + .collect::>(); + locators.sort(); + assert_eq!( + locators, + vec![ + "b@workspace:packages/b".to_string(), + "is-even@npm:1.0.0".to_string(), + "is-number@npm:3.0.0".to_string(), + "is-odd@npm:0.1.2".to_string(), + "small-yarn4@workspace:.".to_string(), + ] + ); + } }