From f6aeecdaadd333d9dd222dc6a5c262974772f74a Mon Sep 17 00:00:00 2001 From: Nathan Hammond Date: Tue, 12 Sep 2023 09:56:47 +0800 Subject: [PATCH 01/11] Capture incremental state. --- .../turborepo-lib/src/package_manager/bun.rs | 63 ++++ .../turborepo-lib/src/package_manager/mod.rs | 33 +- crates/turborepo-lib/src/shim.rs | 1 + crates/turborepo-lockfiles/src/bun/de.rs | 317 ++++++++++++++++++ crates/turborepo-lockfiles/src/bun/mod.rs | 173 ++++++++++ crates/turborepo-lockfiles/src/bun/ser.rs | 215 ++++++++++++ crates/turborepo-lockfiles/src/lib.rs | 2 + 7 files changed, 798 insertions(+), 6 deletions(-) create mode 100644 crates/turborepo-lib/src/package_manager/bun.rs create mode 100644 crates/turborepo-lockfiles/src/bun/de.rs create mode 100644 crates/turborepo-lockfiles/src/bun/mod.rs create mode 100644 crates/turborepo-lockfiles/src/bun/ser.rs diff --git a/crates/turborepo-lib/src/package_manager/bun.rs b/crates/turborepo-lib/src/package_manager/bun.rs new file mode 100644 index 0000000000000..4ed64cb207d46 --- /dev/null +++ b/crates/turborepo-lib/src/package_manager/bun.rs @@ -0,0 +1,63 @@ +use turbopath::AbsoluteSystemPath; + +use crate::package_manager::{Error, PackageManager}; + +pub const LOCKFILE: &str = "bun.lockb"; + +pub struct BunDetector<'a> { + repo_root: &'a AbsoluteSystemPath, + found: bool, +} + +impl<'a> BunDetector<'a> { + pub fn new(repo_root: &'a AbsoluteSystemPath) -> Self { + Self { + repo_root, + found: false, + } + } +} + +impl<'a> Iterator for BunDetector<'a> { + type Item = Result; + + fn next(&mut self) -> Option { + if self.found { + return None; + } + + self.found = true; + let package_json = self.repo_root.join_component(LOCKFILE); + + if package_json.exists() { + Some(Ok(PackageManager::Bun)) + } else { + None + } + } +} + +#[cfg(test)] +mod tests { + use std::fs::File; + + use anyhow::Result; + use tempfile::tempdir; + use turbopath::AbsoluteSystemPathBuf; + + use super::LOCKFILE; + use crate::package_manager::PackageManager; + + #[test] + fn test_detect_bun() -> Result<()> { + let repo_root = tempdir()?; + let repo_root_path = AbsoluteSystemPathBuf::try_from(repo_root.path())?; + + let lockfile_path = repo_root.path().join(LOCKFILE); + File::create(lockfile_path)?; + let package_manager = PackageManager::detect_package_manager(&repo_root_path)?; + assert_eq!(package_manager, PackageManager::Bun); + + Ok(()) + } +} diff --git a/crates/turborepo-lib/src/package_manager/mod.rs b/crates/turborepo-lib/src/package_manager/mod.rs index d9f65e7a1db96..87ef8bf9cbc33 100644 --- a/crates/turborepo-lib/src/package_manager/mod.rs +++ b/crates/turborepo-lib/src/package_manager/mod.rs @@ -1,3 +1,4 @@ +mod bun; mod npm; mod pnpm; mod yarn; @@ -21,7 +22,7 @@ use wax::{Any, Glob, Pattern}; use crate::{ package_json::PackageJson, - package_manager::{npm::NpmDetector, pnpm::PnpmDetector, yarn::YarnDetector}, + package_manager::{bun::BunDetector, npm::NpmDetector, pnpm::PnpmDetector, yarn::YarnDetector}, }; #[derive(Debug, Deserialize)] @@ -67,6 +68,7 @@ pub enum PackageManager { Pnpm, Pnpm6, Yarn, + Bun, } impl Display for PackageManager { @@ -79,6 +81,7 @@ impl Display for PackageManager { PackageManager::Pnpm => write!(f, "pnpm"), PackageManager::Pnpm6 => write!(f, "pnpm6"), PackageManager::Yarn => write!(f, "yarn"), + PackageManager::Bun => write!(f, "bun"), } } } @@ -218,6 +221,10 @@ impl Display for MissingWorkspaceError { "package.json: no workspaces found. Turborepo requires npm workspaces to be \ defined in the root package.json" } + PackageManager::Bun => { + "package.json: no workspaces found. Turborepo requires bun workspaces to be \ + defined in the root package.json" + } }; write!(f, "{}", err) } @@ -277,7 +284,7 @@ pub enum Error { } static PACKAGE_MANAGER_PATTERN: Lazy = - lazy_regex!(r"(?Pnpm|pnpm|yarn)@(?P\d+\.\d+\.\d+(-.+)?)"); + lazy_regex!(r"(?Pbun|npm|pnpm|yarn)@(?P\d+\.\d+\.\d+(-.+)?)"); impl PackageManager { /// Returns the set of globs for the workspace. @@ -304,6 +311,7 @@ impl PackageManager { ["**/node_modules/**", "**/bower_components/**"].as_slice() } PackageManager::Npm => ["**/node_modules/**"].as_slice(), + PackageManager::Bun => ["**/node_modules", "**/.git"].as_slice(), PackageManager::Berry => ["**/node_modules", "**/.git", "**/.yarn"].as_slice(), PackageManager::Yarn => [].as_slice(), // yarn does its own handling above }; @@ -325,7 +333,10 @@ impl PackageManager { pnpm_workspace.packages } } - PackageManager::Berry | PackageManager::Npm | PackageManager::Yarn => { + PackageManager::Berry + | PackageManager::Npm + | PackageManager::Yarn + | PackageManager::Bun => { let package_json_text = fs::read_to_string(root_path.join_component("package.json"))?; let package_json: PackageJsonWorkspaces = serde_json::from_str(&package_json_text)?; @@ -374,6 +385,7 @@ impl PackageManager { let version = version.parse()?; let manager = match manager { "npm" => Some(PackageManager::Npm), + "bun" => Some(PackageManager::Bun), "yarn" => Some(YarnDetector::detect_berry_or_yarn(&version)?), "pnpm" => Some(PnpmDetector::detect_pnpm6_or_pnpm(&version)?), _ => None, @@ -386,6 +398,7 @@ impl PackageManager { let mut detected_package_managers = PnpmDetector::new(repo_root) .chain(NpmDetector::new(repo_root)) .chain(YarnDetector::new(repo_root)) + .chain(BunDetector::new(repo_root)) .collect::, Error>>()?; match detected_package_managers.len() { @@ -433,6 +446,7 @@ impl PackageManager { pub fn lockfile_name(&self) -> &'static str { match self { PackageManager::Npm => npm::LOCKFILE, + PackageManager::Bun => bun::LOCKFILE, PackageManager::Pnpm | PackageManager::Pnpm6 => pnpm::LOCKFILE, PackageManager::Yarn | PackageManager::Berry => yarn::LOCKFILE, } @@ -441,7 +455,10 @@ impl PackageManager { pub fn workspace_configuration_path(&self) -> Option<&'static str> { match self { PackageManager::Pnpm | PackageManager::Pnpm6 => Some("pnpm-workspace.yaml"), - PackageManager::Npm | PackageManager::Berry | PackageManager::Yarn => None, + PackageManager::Npm + | PackageManager::Berry + | PackageManager::Yarn + | PackageManager::Bun => None, } } @@ -467,6 +484,9 @@ impl PackageManager { PackageManager::Yarn => { Box::new(turborepo_lockfiles::Yarn1Lockfile::from_bytes(contents)?) } + PackageManager::Bun => { + Box::new(turborepo_lockfiles::Yarn1Lockfile::from_bytes(contents)?) + } PackageManager::Berry => Box::new(turborepo_lockfiles::BerryLockfile::load( contents, Some(turborepo_lockfiles::BerryManifest::with_resolutions( @@ -490,8 +510,8 @@ impl PackageManager { PackageManager::Pnpm6 | PackageManager::Pnpm => { pnpm::prune_patches(package_json, patches) } - PackageManager::Yarn | PackageManager::Npm => { - unreachable!("npm and yarn 1 don't have a concept of patches") + PackageManager::Yarn | PackageManager::Npm | PackageManager::Bun => { + unreachable!("bun, npm, and yarn 1 don't have a concept of patches") } } } @@ -588,6 +608,7 @@ mod tests { let expected: &[&str] = match mgr { PackageManager::Npm => &["**/node_modules/**"], PackageManager::Berry => &["**/node_modules", "**/.git", "**/.yarn"], + PackageManager::Bun => &["**/node_modules", "**/.git"], PackageManager::Yarn => &["apps/*/node_modules/**", "packages/*/node_modules/**"], PackageManager::Pnpm | PackageManager::Pnpm6 => &[ "**/node_modules/**", diff --git a/crates/turborepo-lib/src/shim.rs b/crates/turborepo-lib/src/shim.rs index c541ba8b97022..6e045fa120f75 100644 --- a/crates/turborepo-lib/src/shim.rs +++ b/crates/turborepo-lib/src/shim.rs @@ -299,6 +299,7 @@ pub struct LocalTurboState { impl LocalTurboState { // Hoisted strategy: + // - `bun install` // - `npm install` // - `yarn` // - `yarn install --flat` diff --git a/crates/turborepo-lockfiles/src/bun/de.rs b/crates/turborepo-lockfiles/src/bun/de.rs new file mode 100644 index 0000000000000..b0da7d08e3089 --- /dev/null +++ b/crates/turborepo-lockfiles/src/bun/de.rs @@ -0,0 +1,317 @@ +use std::sync::OnceLock; + +use nom::{ + branch::alt, + bytes::complete::{escaped_transform, is_not, tag, take_till}, + character::complete::{anychar, char as nom_char, crlf, newline, none_of, satisfy, space1}, + combinator::{all_consuming, map, not, opt, peek, recognize, value}, + multi::{count, many0, many1}, + sequence::{delimited, pair, preceded, separated_pair, terminated, tuple}, + IResult, +}; +use regex::Regex; +use serde_json::Value; + +// regex for trimming spaces from start and end +fn pseudostring_replace() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r"^ *| *$").unwrap()) +} + +pub fn parse_syml(input: &str) -> Result { + match all_consuming(property_statements(0))(input) { + Ok((_, value)) => Ok(value), + Err(e) => Err(super::Error::SymlParse(e.to_string())), + } +} + +// Array and map types +fn item_statements(level: usize) -> impl Fn(&str) -> IResult<&str, Value> { + move |i: &str| map(many0(item_statement(level)), Value::Array)(i) +} + +fn item_statement(level: usize) -> impl Fn(&str) -> IResult<&str, Value> { + move |i: &str| { + let (i, _) = indent(level)(i)?; + let (i, _) = nom_char('-')(i)?; + let (i, _) = blankspace(i)?; + expression(level)(i) + } +} + +fn property_statements(level: usize) -> impl Fn(&str) -> IResult<&str, Value> { + move |i: &str| { + let (i, properties) = many0(property_statement(level))(i)?; + let mut map = serde_json::Map::new(); + for (key, value) in properties.into_iter().flatten() { + map.insert(key, value); + } + Ok((i, Value::Object(map))) + } +} + +fn property_statement(level: usize) -> impl Fn(&str) -> IResult<&str, Vec<(String, Value)>> { + move |i: &str| { + alt(( + value( + vec![], + tuple(( + opt(blankspace), + opt(pair(nom_char('#'), many1(pair(not(eol), anychar)))), + many1(eol_any), + )), + ), + map( + preceded( + indent(level), + separated_pair(name, wrapped_colon, expression(level)), + ), + |entry| vec![entry], + ), + // legacy names + map( + preceded( + indent(level), + separated_pair(legacy_name, wrapped_colon, expression(level)), + ), + |entry| vec![entry], + ), + // legacy prop without colon + map( + preceded( + indent(level), + separated_pair( + legacy_name, + blankspace, + terminated(legacy_literal, many1(eol_any)), + ), + ), + |entry| vec![entry], + ), + multikey_property_statement(level), + ))(i) + } +} + +fn multikey_property_statement( + level: usize, +) -> impl Fn(&str) -> IResult<&str, Vec<(String, Value)>> { + move |i: &str| { + let (i, ()) = indent(level)(i)?; + let (i, property) = legacy_name(i)?; + let (i, others) = many1(preceded( + delimited(opt(blankspace), nom_char(','), opt(blankspace)), + legacy_name, + ))(i)?; + let (i, _) = wrapped_colon(i)?; + let (i, value) = expression(level)(i)?; + + Ok(( + i, + std::iter::once(property) + .chain(others) + .map(|key| (key, value.clone())) + .collect(), + )) + } +} + +fn wrapped_colon(i: &str) -> IResult<&str, char> { + delimited(opt(blankspace), nom_char(':'), opt(blankspace))(i) +} + +fn expression(level: usize) -> impl Fn(&str) -> IResult<&str, Value> { + move |i: &str| { + alt(( + preceded( + tuple(( + peek(tuple((eol, indent(level + 1), nom_char('-'), blankspace))), + eol_any, + )), + item_statements(level + 1), + ), + preceded(eol, property_statements(level + 1)), + terminated(literal, many1(eol_any)), + ))(i) + } +} + +fn indent(level: usize) -> impl Fn(&str) -> IResult<&str, ()> { + move |i: &str| { + let (i, _) = count(nom_char(' '), level * 2)(i)?; + Ok((i, ())) + } +} + +// Simple types + +fn name(i: &str) -> IResult<&str, String> { + alt((string, pseudostring))(i) +} + +fn legacy_name(i: &str) -> IResult<&str, String> { + alt(( + string, + map(recognize(many1(pseudostring_legacy)), |s| s.to_string()), + ))(i) +} + +fn literal(i: &str) -> IResult<&str, Value> { + alt(( + value(Value::Null, null), + map(boolean, Value::Bool), + map(string, Value::String), + map(pseudostring, Value::String), + ))(i) +} + +fn legacy_literal(i: &str) -> IResult<&str, Value> { + alt(( + value(Value::Null, null), + map(string, Value::String), + map(pseudostring_legacy, Value::String), + ))(i) +} + +fn pseudostring(i: &str) -> IResult<&str, String> { + let (i, pseudo) = recognize(pseudostring_inner)(i)?; + Ok(( + i, + pseudostring_replace().replace_all(pseudo, "").into_owned(), + )) +} + +fn pseudostring_inner(i: &str) -> IResult<&str, ()> { + let (i, _) = none_of("\r\n\t ?:,][{}#&*!|>'\"%@`-")(i)?; + let (i, _) = many0(tuple((opt(blankspace), none_of("\r\n\t ,][{}:#\"'"))))(i)?; + Ok((i, ())) +} + +fn pseudostring_legacy(i: &str) -> IResult<&str, String> { + let (i, pseudo) = recognize(pseudostring_legacy_inner)(i)?; + let replaced = pseudostring_replace().replace_all(pseudo, ""); + Ok((i, replaced.to_string())) +} + +fn pseudostring_legacy_inner(i: &str) -> IResult<&str, ()> { + let (i, _) = opt(tag("--"))(i)?; + let (i, _) = satisfy(|c| c.is_ascii_alphanumeric() || c == '/')(i)?; + let (i, _) = take_till(|c| "\r\n\t :,".contains(c))(i)?; + Ok((i, ())) +} + +// String parsing + +fn null(i: &str) -> IResult<&str, &str> { + tag("null")(i) +} + +fn boolean(i: &str) -> IResult<&str, bool> { + alt((value(true, tag("true")), value(false, tag("false"))))(i) +} + +fn string(i: &str) -> IResult<&str, String> { + alt((empty_string, delimited(tag("\""), syml_chars, tag("\""))))(i) +} + +fn empty_string(i: &str) -> IResult<&str, String> { + let (i, _) = tag(r#""""#)(i)?; + Ok((i, "".to_string())) +} + +fn syml_chars(i: &str) -> IResult<&str, String> { + // The SYML grammar provided by Yarn2+ includes escape sequences that weren't + // supported by the yarn1 parser. We diverge from the Yarn2+ provided + // grammar to match the actual parser used by yarn1. + escaped_transform( + is_not("\"\\"), + '\\', + alt(( + value("\"", tag("\"")), + value("\\", tag("\\")), + value("/", tag("/")), + value("\n", tag("n")), + value("\r", tag("r")), + value("\t", tag("t")), + )), + )(i) +} + +// Spaces +fn blankspace(i: &str) -> IResult<&str, &str> { + space1(i) +} + +fn eol_any(i: &str) -> IResult<&str, &str> { + recognize(tuple((eol, many0(tuple((opt(blankspace), eol))))))(i) +} + +fn eol(i: &str) -> IResult<&str, &str> { + alt((crlf, value("\n", newline), value("\r", nom_char('\r'))))(i) +} + +#[cfg(test)] +mod test { + use serde_json::json; + use test_case::test_case; + + use super::*; + + #[test_case("null", Value::Null ; "null")] + #[test_case("false", Value::Bool(false) ; "literal false")] + #[test_case("true", Value::Bool(true) ; "literal true")] + #[test_case("\"\"", Value::String("".into()) ; "empty string literal")] + #[test_case("\"foo\"", Value::String("foo".into()) ; "quoted string literal")] + #[test_case("foo", Value::String("foo".into()) ; "unquoted string literal")] + fn test_literal(input: &str, expected: Value) { + let (_, actual) = literal(input).unwrap(); + assert_eq!(actual, expected); + } + + #[test_case("name: foo", "name" ; "basic")] + #[test_case("technically a name: foo", "technically a name" ; "multiword name")] + fn test_name(input: &str, expected: &str) { + let (_, actual) = name(input).unwrap(); + assert_eq!(actual, expected); + } + + #[test_case("foo@1:", "foo@1" ; "name with colon terminator")] + #[test_case("\"foo@1\":", "foo@1" ; "qutoed name with colon terminator")] + #[test_case("name foo", "name" ; "name without colon terminator")] + fn test_legacy_name(input: &str, expected: &str) { + let (_, actual) = legacy_name(input).unwrap(); + assert_eq!(actual, expected); + } + + #[test_case("null\n", Value::Null ; "null")] + #[test_case("\"foo\"\n", json!("foo") ; "basic string")] + #[test_case("\n name: foo\n", json!({ "name": "foo" }) ; "basic object")] + fn test_expression(input: &str, expected: Value) { + let (_, actual) = expression(0)(input).unwrap(); + assert_eq!(actual, expected); + } + + #[test_case("# a comment\n", vec![] ; "comment")] + #[test_case("foo: null\n", vec![("foo".into(), Value::Null)] ; "single property")] + #[test_case("name foo\n", vec![("name".into(), json!("foo"))] ; "legacy property")] + fn test_property_statement(input: &str, expected: Vec<(String, Value)>) { + let (_, actual) = property_statement(0)(input).unwrap(); + assert_eq!(actual, expected); + } + + #[test_case("name: foo\n", json!({"name": "foo"}) ; "single property object")] + #[test_case("\"name\": foo\n", json!({"name": "foo"}) ; "single quoted property object")] + #[test_case("name foo\n", json!({"name": "foo"}) ; "single property without colon object")] + #[test_case("# comment\nname: foo\n", json!({"name": "foo"}) ; "comment doesn't affect object")] + #[test_case("name foo\nversion \"1.2.3\"\n", json!({"name": "foo", "version": "1.2.3"}) ; "multi-property object")] + #[test_case("foo:\n version \"1.2.3\"\n", json!({"foo": {"version": "1.2.3"}}) ; "nested object")] + #[test_case("foo, bar, baz:\n version \"1.2.3\"\n", json!({ + "foo": {"version": "1.2.3"}, + "bar": {"version": "1.2.3"}, + "baz": {"version": "1.2.3"}, + }) ; "multi-key object")] + fn test_property_statements(input: &str, expected: Value) { + let (_, actual) = property_statements(0)(input).unwrap(); + assert_eq!(actual, expected); + } +} diff --git a/crates/turborepo-lockfiles/src/bun/mod.rs b/crates/turborepo-lockfiles/src/bun/mod.rs new file mode 100644 index 0000000000000..e26c65d99f9c6 --- /dev/null +++ b/crates/turborepo-lockfiles/src/bun/mod.rs @@ -0,0 +1,173 @@ +use std::str::FromStr; + +use serde::Deserialize; + +use crate::Lockfile; + +mod de; +mod ser; + +type Map = std::collections::BTreeMap; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("unable to parse: {0}")] + SymlParse(String), + #[error("unable to convert to structured syml: {0}")] + SymlStructure(#[from] serde_json::Error), + #[error("unexpected non-utf8 yarn.lock")] + NonUTF8(#[from] std::str::Utf8Error), +} + +pub struct Yarn1Lockfile { + inner: Map, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Entry { + name: Option, + version: String, + uid: Option, + resolved: Option, + integrity: Option, + registry: Option, + dependencies: Option>, + optional_dependencies: Option>, +} + +impl Yarn1Lockfile { + pub fn from_bytes(input: &[u8]) -> Result { + let input = std::str::from_utf8(input).map_err(Error::from)?; + Self::from_str(input) + } +} + +impl FromStr for Yarn1Lockfile { + type Err = super::Error; + + fn from_str(s: &str) -> Result { + let value = de::parse_syml(s)?; + let inner = serde_json::from_value(value)?; + Ok(Self { inner }) + } +} + +impl Lockfile for Yarn1Lockfile { + fn resolve_package( + &self, + _workspace_path: &str, + name: &str, + version: &str, + ) -> Result, crate::Error> { + for key in possible_keys(name, version) { + if let Some(entry) = self.inner.get(&key) { + return Ok(Some(crate::Package { + key, + version: entry.version.clone(), + })); + } + } + + Ok(None) + } + + fn all_dependencies( + &self, + key: &str, + ) -> Result>, crate::Error> { + let Some(entry) = self.inner.get(key) else { + return Ok(None); + }; + + let all_deps: std::collections::HashMap<_, _> = entry.dependency_entries().collect(); + Ok(match all_deps.is_empty() { + false => Some(all_deps), + true => None, + }) + } + + fn subgraph( + &self, + _workspace_packages: &[String], + packages: &[String], + ) -> Result, super::Error> { + let mut inner = Map::new(); + + for (key, entry) in packages.iter().filter_map(|key| { + let entry = self.inner.get(key)?; + Some((key, entry)) + }) { + inner.insert(key.clone(), entry.clone()); + } + + Ok(Box::new(Self { inner })) + } + + fn encode(&self) -> Result, crate::Error> { + Ok(self.to_string().into_bytes()) + } + + fn global_change_key(&self) -> Vec { + vec![b'y', b'a', b'r', b'n', 0] + } +} + +pub fn yarn_subgraph(contents: &[u8], packages: &[String]) -> Result, crate::Error> { + let lockfile = Yarn1Lockfile::from_bytes(contents)?; + let pruned_lockfile = lockfile.subgraph(&[], packages)?; + pruned_lockfile.encode() +} + +impl Entry { + fn dependency_entries(&self) -> impl Iterator + '_ { + self.dependencies + .iter() + .flatten() + .chain(self.optional_dependencies.iter().flatten()) + .map(|(k, v)| (k.clone(), v.clone())) + } +} + +const PROTOCOLS: &[&str] = ["", "npm:", "file:", "workspace:", "yarn:"].as_slice(); + +fn possible_keys<'a>(name: &'a str, version: &'a str) -> impl Iterator + 'a { + PROTOCOLS + .iter() + .copied() + .map(move |protocol| format!("{name}@{protocol}{version}")) +} + +#[cfg(test)] +mod test { + use pretty_assertions::assert_eq; + use test_case::test_case; + + use super::*; + + const MINIMAL: &str = include_str!("../../fixtures/yarn1.lock"); + const FULL: &str = include_str!("../../fixtures/yarn1full.lock"); + + #[test_case(MINIMAL ; "minimal lockfile")] + #[test_case(FULL ; "full lockfile")] + fn test_roundtrip(input: &str) { + let lockfile = Yarn1Lockfile::from_str(input).unwrap(); + assert_eq!(input, lockfile.to_string()); + } + + #[test] + fn test_key_splitting() { + let lockfile = Yarn1Lockfile::from_str(FULL).unwrap(); + for key in [ + "@babel/types@^7.18.10", + "@babel/types@^7.18.6", + "@babel/types@^7.19.0", + ] { + assert!( + lockfile.inner.contains_key(key), + "missing {} in lockfile", + key + ); + } + } +} diff --git a/crates/turborepo-lockfiles/src/bun/ser.rs b/crates/turborepo-lockfiles/src/bun/ser.rs new file mode 100644 index 0000000000000..57f6be81fe4ac --- /dev/null +++ b/crates/turborepo-lockfiles/src/bun/ser.rs @@ -0,0 +1,215 @@ +use std::{ + borrow::Cow, + collections::{HashMap, HashSet}, + fmt, +}; + +use super::{Entry, Yarn1Lockfile}; + +const INDENT: &str = " "; + +impl Yarn1Lockfile { + fn reverse_lookup(&self) -> HashMap<&Entry, HashSet<&str>> { + let mut reverse_lookup = HashMap::new(); + for (key, value) in self.inner.iter() { + let keys: &mut HashSet<&str> = reverse_lookup.entry(value).or_default(); + keys.insert(key); + } + reverse_lookup + } +} + +impl fmt::Display for Yarn1Lockfile { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str( + "# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.\n# yarn lockfile \ + v1\n\n", + )?; + let reverse_lookup = self.reverse_lookup(); + let mut added_keys: HashSet<&str> = HashSet::with_capacity(self.inner.len()); + for (key, entry) in self.inner.iter() { + if added_keys.contains(key.as_str()) { + continue; + } + + let all_keys = reverse_lookup + .get(entry) + .expect("entry in lockfile should appear as a key in reverse lookup"); + added_keys.extend(all_keys); + let mut keys = all_keys.iter().copied().collect::>(); + // Keys must be sorted before they get wrapped + keys.sort(); + + let wrapped_keys = keys.into_iter().map(maybe_wrap).collect::>(); + let key_line = wrapped_keys.join(", "); + + f.write_fmt(format_args!("\n{}:\n{}\n", key_line, entry))?; + } + Ok(()) + } +} + +impl fmt::Display for Entry { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut leading = LeadingNewline::new(); + if let Some(name) = &self.name { + f.write_fmt(format_args!( + "{}{INDENT}name {}", + leading.leading(), + maybe_wrap(name) + ))?; + } + f.write_fmt(format_args!( + "{}{INDENT}version {}", + leading.leading(), + maybe_wrap(&self.version) + ))?; + if let Some(uid) = &self.uid { + f.write_fmt(format_args!( + "{}{INDENT}uid {}", + leading.leading(), + maybe_wrap(uid) + ))?; + } + if let Some(resolved) = &self.resolved { + f.write_fmt(format_args!( + "{}{INDENT}resolved {}", + leading.leading(), + maybe_wrap(resolved) + ))?; + } + if let Some(integrity) = &self.integrity { + f.write_fmt(format_args!( + "{}{INDENT}integrity {}", + leading.leading(), + maybe_wrap(integrity) + ))?; + } + if let Some(registry) = &self.registry { + f.write_fmt(format_args!( + "{}{INDENT}integrity {}", + leading.leading(), + maybe_wrap(registry) + ))?; + } + // encode deps and optional deps + if let Some(deps) = &self.dependencies { + f.write_fmt(format_args!("{}{INDENT}dependencies:", leading.leading()))?; + encode_map(deps.iter().map(|(k, v)| (k.as_ref(), v.as_ref())), f)?; + } + if let Some(optional_deps) = &self.optional_dependencies { + f.write_fmt(format_args!( + "{}{INDENT}optionalDependencies:", + leading.leading() + ))?; + encode_map( + optional_deps.iter().map(|(k, v)| (k.as_ref(), v.as_ref())), + f, + )?; + } + Ok(()) + } +} + +#[derive(Debug, Clone, Copy)] +enum LeadingNewline { + First, + Rest, +} + +impl LeadingNewline { + fn new() -> Self { + Self::First + } + + fn leading(&mut self) -> &'static str { + let res = match self { + LeadingNewline::First => "", + LeadingNewline::Rest => "\n", + }; + *self = Self::Rest; + res + } +} + +fn encode_map<'a, I: Iterator>( + entries: I, + f: &mut fmt::Formatter<'_>, +) -> fmt::Result { + let mut wrapped_entries = entries + .map(|(k, v)| (maybe_wrap(k), maybe_wrap(v))) + .collect::>(); + wrapped_entries.sort_unstable_by(|(k1, _), (k2, _)| k1.cmp(k2)); + // we sort the via wrapped keys + // then we write each line with the value wrapped as well + for (key, value) in wrapped_entries { + f.write_fmt(format_args!("\n{INDENT}{INDENT}{key} {value}"))?; + } + + Ok(()) +} + +fn maybe_wrap(s: &str) -> Cow { + match should_wrap_key(s) { + // yarn uses JSON.stringify to escape strings + // we approximate this behavior using serde_json + true => serde_json::to_string(s) + .expect("failed at encoding string as json") + .into(), + false => s.into(), + } +} + +// Determines if we need to wrap a key +fn should_wrap_key(s: &str) -> bool { + // Wrap if it starts with a syml keyword + s.starts_with("true") || + s.starts_with("false") || + // Wrap if it doesn't start with a-zA-Z + s.chars().next().map_or(false, |c| !c.is_ascii_alphabetic()) || + // Wrap if it contains any unwanted chars + s.chars().any(|c| matches!( + c, + ' ' | ':' | '\t' | '\r' | '\u{000B}' | '\u{000C}' | '\n' | '\\' | '"' | ',' | '[' | ']' + )) +} + +#[cfg(test)] +mod test { + use pretty_assertions::assert_eq; + + use super::*; + + #[test] + fn test_should_wrap() { + assert!(should_wrap_key("jsx-ast-utils@^2.4.1 || ^3.0.0")) + } + + #[test] + fn test_basic_serialization() { + let entry = Entry { + version: "12.2.5".into(), + resolved: Some("https://registry.yarnpkg.com/next/-/next-12.2.5.tgz#14fb5975e8841fad09553b8ef41fe1393602b717".into()), + integrity: Some("sha512-tBdjqX5XC/oFs/6gxrZhjmiq90YWizUYU6qOWAfat7zJwrwapJ+BYgX2PmiacunXMaRpeVT4vz5MSPSLgNkrpA==".into()), + dependencies: Some(vec![ + ("@next/env".into(), "12.2.5".into()), + ("caniuse-lite".into(), "^1.0.30001332".into()), + ("postcss".into(), "8.4.14".into()), + ].into_iter().collect()), + optional_dependencies: Some(vec![("@next/swc-win32-x64-msvc".into(), "12.2.5".into())].into_iter().collect()), + ..Default::default() + }; + assert_eq!( + entry.to_string(), + r#" version "12.2.5" + resolved "https://registry.yarnpkg.com/next/-/next-12.2.5.tgz#14fb5975e8841fad09553b8ef41fe1393602b717" + integrity sha512-tBdjqX5XC/oFs/6gxrZhjmiq90YWizUYU6qOWAfat7zJwrwapJ+BYgX2PmiacunXMaRpeVT4vz5MSPSLgNkrpA== + dependencies: + "@next/env" "12.2.5" + caniuse-lite "^1.0.30001332" + postcss "8.4.14" + optionalDependencies: + "@next/swc-win32-x64-msvc" "12.2.5""# + ); + } +} diff --git a/crates/turborepo-lockfiles/src/lib.rs b/crates/turborepo-lockfiles/src/lib.rs index 606e77c1717e2..7cc410669d22b 100644 --- a/crates/turborepo-lockfiles/src/lib.rs +++ b/crates/turborepo-lockfiles/src/lib.rs @@ -1,6 +1,7 @@ #![deny(clippy::all)] mod berry; +mod bun; mod error; mod npm; mod pnpm; @@ -9,6 +10,7 @@ mod yarn1; use std::collections::{HashMap, HashSet}; pub use berry::{Error as BerryError, *}; +pub use bun::{bun_subgraph, BunLockfile}; pub use error::Error; pub use npm::*; pub use pnpm::{pnpm_global_change, pnpm_subgraph, PnpmLockfile}; From 50afda02c396c4ebc22f7d1db529b4be47e31c07 Mon Sep 17 00:00:00 2001 From: Nathan Hammond Date: Tue, 12 Sep 2023 10:34:25 +0800 Subject: [PATCH 02/11] compiles --- cli/internal/ffi/ffi.go | 2 + cli/internal/ffi/proto/messages.pb.go | 25 ++++++-- cli/internal/lockfile/bun_lockfile.go | 38 +++++++++++++ cli/internal/lockfile/lockfile.go | 3 + cli/internal/packagemanager/bun.go | 57 +++++++++++++++++++ cli/internal/packagemanager/packagemanager.go | 3 + crates/turborepo-ffi/messages.proto | 1 + crates/turborepo-ffi/src/lockfile.rs | 24 +++++++- crates/turborepo-lockfiles/src/bun/mod.rs | 16 +++--- crates/turborepo-lockfiles/src/bun/ser.rs | 6 +- crates/turborepo-lockfiles/src/error.rs | 2 + 11 files changed, 160 insertions(+), 17 deletions(-) create mode 100644 cli/internal/lockfile/bun_lockfile.go create mode 100644 cli/internal/packagemanager/bun.go diff --git a/cli/internal/ffi/ffi.go b/cli/internal/ffi/ffi.go index 6b478bee94a10..641d6ed79880c 100644 --- a/cli/internal/ffi/ffi.go +++ b/cli/internal/ffi/ffi.go @@ -220,6 +220,8 @@ func toPackageManager(packageManager string) ffi_proto.PackageManager { return ffi_proto.PackageManager_PNPM case "yarn": return ffi_proto.PackageManager_YARN + case "bun": + return ffi_proto.PackageManager_BUN default: panic(fmt.Sprintf("Invalid package manager string: %s", packageManager)) } diff --git a/cli/internal/ffi/proto/messages.pb.go b/cli/internal/ffi/proto/messages.pb.go index 9dac0a04fba62..989c237f4f08c 100644 --- a/cli/internal/ffi/proto/messages.pb.go +++ b/cli/internal/ffi/proto/messages.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 -// protoc v3.21.12 +// protoc-gen-go v1.31.0 +// protoc v4.23.4 // source: turborepo-ffi/messages.proto package proto @@ -27,6 +27,7 @@ const ( PackageManager_BERRY PackageManager = 1 PackageManager_PNPM PackageManager = 2 PackageManager_YARN PackageManager = 3 + PackageManager_BUN PackageManager = 4 ) // Enum value maps for PackageManager. @@ -36,12 +37,14 @@ var ( 1: "BERRY", 2: "PNPM", 3: "YARN", + 4: "BUN", } PackageManager_value = map[string]int32{ "NPM": 0, "BERRY": 1, "PNPM": 2, "YARN": 3, + "BUN": 4, } ) @@ -196,6 +199,7 @@ type GlobResp struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *GlobResp_Files // *GlobResp_Error Response isGlobResp_Response `protobuf_oneof:"response"` @@ -394,6 +398,7 @@ type ChangedFilesResp struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *ChangedFilesResp_Files // *ChangedFilesResp_Error Response isChangedFilesResp_Response `protobuf_oneof:"response"` @@ -584,6 +589,7 @@ type PreviousContentResp struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *PreviousContentResp_Content // *PreviousContentResp_Error Response isPreviousContentResp_Response `protobuf_oneof:"response"` @@ -884,6 +890,7 @@ type TransitiveDepsResponse struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *TransitiveDepsResponse_Dependencies // *TransitiveDepsResponse_Error Response isTransitiveDepsResponse_Response `protobuf_oneof:"response"` @@ -1200,6 +1207,7 @@ type SubgraphResponse struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *SubgraphResponse_Contents // *SubgraphResponse_Error Response isSubgraphResponse_Response `protobuf_oneof:"response"` @@ -1335,6 +1343,7 @@ type PatchesResponse struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *PatchesResponse_Patches // *PatchesResponse_Error Response isPatchesResponse_Response `protobuf_oneof:"response"` @@ -1753,6 +1762,7 @@ type VerifySignatureResponse struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *VerifySignatureResponse_Verified // *VerifySignatureResponse_Error Response isVerifySignatureResponse_Response `protobuf_oneof:"response"` @@ -1896,6 +1906,7 @@ type GetPackageFileHashesResponse struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *GetPackageFileHashesResponse_Hashes // *GetPackageFileHashesResponse_Error Response isGetPackageFileHashesResponse_Response `protobuf_oneof:"response"` @@ -2039,6 +2050,7 @@ type GetHashesForFilesResponse struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *GetHashesForFilesResponse_Hashes // *GetHashesForFilesResponse_Error Response isGetHashesForFilesResponse_Response `protobuf_oneof:"response"` @@ -2221,6 +2233,7 @@ type FromWildcardsResponse struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *FromWildcardsResponse_EnvVars // *FromWildcardsResponse_Error Response isFromWildcardsResponse_Response `protobuf_oneof:"response"` @@ -2513,6 +2526,7 @@ type GetGlobalHashableEnvVarsResponse struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *GetGlobalHashableEnvVarsResponse_DetailedMap // *GetGlobalHashableEnvVarsResponse_Error Response isGetGlobalHashableEnvVarsResponse_Response `protobuf_oneof:"response"` @@ -2884,11 +2898,12 @@ var file_turborepo_ffi_messages_proto_rawDesc = []byte{ 0x0b, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x4d, 0x61, 0x70, 0x12, 0x16, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x42, 0x0a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x2a, 0x38, 0x0a, 0x0e, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x6e, 0x61, 0x67, + 0x2a, 0x41, 0x0a, 0x0e, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x12, 0x07, 0x0a, 0x03, 0x4e, 0x50, 0x4d, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x42, 0x45, 0x52, 0x52, 0x59, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x50, 0x4e, 0x50, 0x4d, 0x10, 0x02, - 0x12, 0x08, 0x0a, 0x04, 0x59, 0x41, 0x52, 0x4e, 0x10, 0x03, 0x42, 0x0b, 0x5a, 0x09, 0x66, 0x66, - 0x69, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x12, 0x08, 0x0a, 0x04, 0x59, 0x41, 0x52, 0x4e, 0x10, 0x03, 0x12, 0x07, 0x0a, 0x03, 0x42, 0x55, + 0x4e, 0x10, 0x04, 0x42, 0x0b, 0x5a, 0x09, 0x66, 0x66, 0x69, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/cli/internal/lockfile/bun_lockfile.go b/cli/internal/lockfile/bun_lockfile.go new file mode 100644 index 0000000000000..824737d3a22aa --- /dev/null +++ b/cli/internal/lockfile/bun_lockfile.go @@ -0,0 +1,38 @@ +package lockfile + +import ( + "github.com/vercel/turbo/cli/internal/turbopath" +) + +// BunLockfile representation of bun lockfile +type BunLockfile struct { + contents []byte +} + +var _ Lockfile = (*BunLockfile)(nil) + +// ResolvePackage Given a package and version returns the key, resolved version, and if it was found +func (l *BunLockfile) ResolvePackage(_workspacePath turbopath.AnchoredUnixPath, name string, version string) (Package, error) { + // This is only used when doing calculating the transitive deps, but Rust + // implementations do this calculation on the Rust side. + panic("Unreachable") +} + +// AllDependencies Given a lockfile key return all (dev/optional/peer) dependencies of that package +func (l *BunLockfile) AllDependencies(key string) (map[string]string, bool) { + // This is only used when doing calculating the transitive deps, but Rust + // implementations do this calculation on the Rust side. + panic("Unreachable") +} + +// DecodeBunLockfile Takes the contents of a bun lockfile and returns a struct representation +func DecodeBunLockfile(contents []byte) (*BunLockfile, error) { + return &BunLockfile{contents: contents}, nil +} + +// GlobalChange checks if there are any differences between lockfiles that would completely invalidate +// the cache. +func (l *BunLockfile) GlobalChange(other Lockfile) bool { + _, ok := other.(*BunLockfile) + return !ok +} diff --git a/cli/internal/lockfile/lockfile.go b/cli/internal/lockfile/lockfile.go index d701801434aa1..9d8a2143ca82e 100644 --- a/cli/internal/lockfile/lockfile.go +++ b/cli/internal/lockfile/lockfile.go @@ -78,6 +78,9 @@ func AllTransitiveClosures( if lf, ok := lockFile.(*YarnLockfile); ok { return rustTransitiveDeps(lf.contents, "yarn", workspaces, nil) } + if lf, ok := lockFile.(*BunLockfile); ok { + return rustTransitiveDeps(lf.contents, "bun", workspaces, nil) + } g := new(errgroup.Group) c := make(chan closureMsg, len(workspaces)) diff --git a/cli/internal/packagemanager/bun.go b/cli/internal/packagemanager/bun.go new file mode 100644 index 0000000000000..33cf88ecd6166 --- /dev/null +++ b/cli/internal/packagemanager/bun.go @@ -0,0 +1,57 @@ +package packagemanager + +import ( + "fmt" + + "github.com/vercel/turbo/cli/internal/fs" + "github.com/vercel/turbo/cli/internal/lockfile" + "github.com/vercel/turbo/cli/internal/turbopath" +) + +var nodejsBun = PackageManager{ + Name: "nodejs-bun", + Slug: "bun", + Command: "bun", + Specfile: "package.json", + Lockfile: "bun.lockb", + PackageDir: "node_modules", + ArgSeparator: func(userArgs []string) []string { + // Bun swallows a single "--" token. If the user is passing "--", we need + // to prepend our own so that the user's doesn't get swallowed. If they are not + // passing their own, we don't need the "--" token and can avoid the warning. + for _, arg := range userArgs { + if arg == "--" { + return []string{"--"} + } + } + return nil + }, + + getWorkspaceGlobs: func(rootpath turbopath.AbsoluteSystemPath) ([]string, error) { + pkg, err := fs.ReadPackageJSON(rootpath.UntypedJoin("package.json")) + if err != nil { + return nil, fmt.Errorf("package.json: %w", err) + } + if len(pkg.Workspaces) == 0 { + return nil, fmt.Errorf("package.json: no workspaces found. Turborepo requires Bun workspaces to be defined in the root package.json") + } + return pkg.Workspaces, nil + }, + + getWorkspaceIgnores: func(pm PackageManager, rootpath turbopath.AbsoluteSystemPath) ([]string, error) { + // Matches upstream values: + // Key code: https://github.com/oven-sh/bun/blob/f267c1d097923a2d2992f9f60a6dd365fe706512/src/install/lockfile.zig#L3057 + return []string{ + "**/node_modules", + "**/.git", + }, nil + }, + + canPrune: func(cwd turbopath.AbsoluteSystemPath) (bool, error) { + return false, nil + }, + + UnmarshalLockfile: func(_rootPackageJSON *fs.PackageJSON, contents []byte) (lockfile.Lockfile, error) { + return lockfile.DecodeBunLockfile(contents) + }, +} diff --git a/cli/internal/packagemanager/packagemanager.go b/cli/internal/packagemanager/packagemanager.go index 70c9533887758..7c16d94222ad0 100644 --- a/cli/internal/packagemanager/packagemanager.go +++ b/cli/internal/packagemanager/packagemanager.go @@ -64,6 +64,7 @@ var packageManagers = []PackageManager{ nodejsNpm, nodejsPnpm, nodejsPnpm6, + nodejsBun, } // GetPackageManager reads the package manager name sent by the Rust side @@ -71,6 +72,8 @@ func GetPackageManager(name string) (packageManager *PackageManager, err error) switch name { case "yarn": return &nodejsYarn, nil + case "bun": + return &nodejsBun, nil case "berry": return &nodejsBerry, nil case "npm": diff --git a/crates/turborepo-ffi/messages.proto b/crates/turborepo-ffi/messages.proto index 890ca5b0ac30c..de27736c5f3d5 100644 --- a/crates/turborepo-ffi/messages.proto +++ b/crates/turborepo-ffi/messages.proto @@ -59,6 +59,7 @@ enum PackageManager { BERRY = 1; PNPM = 2; YARN = 3; + BUN = 4; } message PackageDependency { diff --git a/crates/turborepo-ffi/src/lockfile.rs b/crates/turborepo-ffi/src/lockfile.rs index 9f1600f1dd498..baa4e79cab3e8 100644 --- a/crates/turborepo-ffi/src/lockfile.rs +++ b/crates/turborepo-ffi/src/lockfile.rs @@ -5,7 +5,8 @@ use std::{ use thiserror::Error; use turborepo_lockfiles::{ - self, BerryLockfile, Lockfile, LockfileData, NpmLockfile, Package, PnpmLockfile, Yarn1Lockfile, + self, BerryLockfile, BunLockfile, Lockfile, LockfileData, NpmLockfile, Package, PnpmLockfile, + Yarn1Lockfile, }; use super::{proto, Buffer}; @@ -54,6 +55,7 @@ fn transitive_closure_inner(buf: Buffer) -> Result berry_transitive_closure_inner(request), proto::PackageManager::Pnpm => pnpm_transitive_closure_inner(request), proto::PackageManager::Yarn => yarn_transitive_closure_inner(request), + proto::PackageManager::Bun => bun_transitive_closure_inner(request), } } @@ -126,6 +128,23 @@ fn yarn_transitive_closure_inner( Ok(dependencies.into()) } +fn bun_transitive_closure_inner( + request: proto::TransitiveDepsRequest, +) -> Result { + let proto::TransitiveDepsRequest { + contents, + workspaces, + .. + } = request; + let lockfile = + BunLockfile::from_bytes(contents.as_slice()).map_err(turborepo_lockfiles::Error::from)?; + let dependencies = turborepo_lockfiles::all_transitive_closures( + &lockfile, + workspaces.into_iter().map(|(k, v)| (k, v.into())).collect(), + )?; + Ok(dependencies.into()) +} + #[no_mangle] pub extern "C" fn subgraph(buf: Buffer) -> Buffer { use proto::subgraph_response::Response; @@ -162,6 +181,7 @@ fn subgraph_inner(buf: Buffer) -> Result, Error> { turborepo_lockfiles::pnpm_subgraph(&contents, &workspaces, &packages)? } proto::PackageManager::Yarn => turborepo_lockfiles::yarn_subgraph(&contents, &packages)?, + proto::PackageManager::Bun => turborepo_lockfiles::bun_subgraph(&contents, &packages)?, }; Ok(contents) } @@ -227,6 +247,7 @@ fn global_change_inner(buf: Buffer) -> Result { &request.curr_contents, )?), proto::PackageManager::Yarn => Ok(false), + proto::PackageManager::Bun => Ok(false), } } @@ -271,6 +292,7 @@ impl fmt::Display for proto::PackageManager { proto::PackageManager::Berry => "berry", proto::PackageManager::Pnpm => "pnpm", proto::PackageManager::Yarn => "yarn", + proto::PackageManager::Bun => "bun", }) } } diff --git a/crates/turborepo-lockfiles/src/bun/mod.rs b/crates/turborepo-lockfiles/src/bun/mod.rs index e26c65d99f9c6..6bd27e8d1ae48 100644 --- a/crates/turborepo-lockfiles/src/bun/mod.rs +++ b/crates/turborepo-lockfiles/src/bun/mod.rs @@ -19,7 +19,7 @@ pub enum Error { NonUTF8(#[from] std::str::Utf8Error), } -pub struct Yarn1Lockfile { +pub struct BunLockfile { inner: Map, } @@ -36,14 +36,14 @@ struct Entry { optional_dependencies: Option>, } -impl Yarn1Lockfile { +impl BunLockfile { pub fn from_bytes(input: &[u8]) -> Result { let input = std::str::from_utf8(input).map_err(Error::from)?; Self::from_str(input) } } -impl FromStr for Yarn1Lockfile { +impl FromStr for BunLockfile { type Err = super::Error; fn from_str(s: &str) -> Result { @@ -53,7 +53,7 @@ impl FromStr for Yarn1Lockfile { } } -impl Lockfile for Yarn1Lockfile { +impl Lockfile for BunLockfile { fn resolve_package( &self, _workspace_path: &str, @@ -113,8 +113,8 @@ impl Lockfile for Yarn1Lockfile { } } -pub fn yarn_subgraph(contents: &[u8], packages: &[String]) -> Result, crate::Error> { - let lockfile = Yarn1Lockfile::from_bytes(contents)?; +pub fn bun_subgraph(contents: &[u8], packages: &[String]) -> Result, crate::Error> { + let lockfile = BunLockfile::from_bytes(contents)?; let pruned_lockfile = lockfile.subgraph(&[], packages)?; pruned_lockfile.encode() } @@ -151,13 +151,13 @@ mod test { #[test_case(MINIMAL ; "minimal lockfile")] #[test_case(FULL ; "full lockfile")] fn test_roundtrip(input: &str) { - let lockfile = Yarn1Lockfile::from_str(input).unwrap(); + let lockfile = BunLockfile::from_str(input).unwrap(); assert_eq!(input, lockfile.to_string()); } #[test] fn test_key_splitting() { - let lockfile = Yarn1Lockfile::from_str(FULL).unwrap(); + let lockfile = BunLockfile::from_str(FULL).unwrap(); for key in [ "@babel/types@^7.18.10", "@babel/types@^7.18.6", diff --git a/crates/turborepo-lockfiles/src/bun/ser.rs b/crates/turborepo-lockfiles/src/bun/ser.rs index 57f6be81fe4ac..456b6410370d2 100644 --- a/crates/turborepo-lockfiles/src/bun/ser.rs +++ b/crates/turborepo-lockfiles/src/bun/ser.rs @@ -4,11 +4,11 @@ use std::{ fmt, }; -use super::{Entry, Yarn1Lockfile}; +use super::{BunLockfile, Entry}; const INDENT: &str = " "; -impl Yarn1Lockfile { +impl BunLockfile { fn reverse_lookup(&self) -> HashMap<&Entry, HashSet<&str>> { let mut reverse_lookup = HashMap::new(); for (key, value) in self.inner.iter() { @@ -19,7 +19,7 @@ impl Yarn1Lockfile { } } -impl fmt::Display for Yarn1Lockfile { +impl fmt::Display for BunLockfile { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str( "# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.\n# yarn lockfile \ diff --git a/crates/turborepo-lockfiles/src/error.rs b/crates/turborepo-lockfiles/src/error.rs index 84724790da9a8..9a1f95928736f 100644 --- a/crates/turborepo-lockfiles/src/error.rs +++ b/crates/turborepo-lockfiles/src/error.rs @@ -19,6 +19,8 @@ pub enum Error { #[error(transparent)] Yarn1(#[from] crate::yarn1::Error), #[error(transparent)] + Bun(#[from] crate::bun::Error), + #[error(transparent)] Berry(#[from] crate::berry::Error), #[error("lockfile contains invalid path: {0}")] Path(#[from] turbopath::PathError), From af190ad05721ee23a4fea3c9c4353799f7880473 Mon Sep 17 00:00:00 2001 From: Nathan Hammond Date: Tue, 12 Sep 2023 12:13:54 +0800 Subject: [PATCH 03/11] Refactor Go lockfile reading code. --- cli/internal/packagemanager/berry.go | 16 +++++++++- cli/internal/packagemanager/bun.go | 30 +++++++++++++++++-- cli/internal/packagemanager/npm.go | 16 +++++++++- cli/internal/packagemanager/packagemanager.go | 15 ++++++++-- .../packagemanager/packagemanager_test.go | 9 ++++++ cli/internal/packagemanager/pnpm.go | 16 +++++++++- cli/internal/packagemanager/pnpm6.go | 16 +++++++++- cli/internal/packagemanager/yarn.go | 16 +++++++++- cli/internal/run/global_hash.go | 5 ++-- cli/internal/scope/scope.go | 13 ++++---- cli/internal/scope/scope_test.go | 2 +- 11 files changed, 134 insertions(+), 20 deletions(-) diff --git a/cli/internal/packagemanager/berry.go b/cli/internal/packagemanager/berry.go index 8f69498c02765..668a95ebd4ae3 100644 --- a/cli/internal/packagemanager/berry.go +++ b/cli/internal/packagemanager/berry.go @@ -9,12 +9,14 @@ import ( "github.com/vercel/turbo/cli/internal/turbopath" ) +const berryLockfile = "yarn.lock" + var nodejsBerry = PackageManager{ Name: "nodejs-berry", Slug: "yarn", Command: "yarn", Specfile: "package.json", - Lockfile: "yarn.lock", + Lockfile: berryLockfile, PackageDir: "node_modules", ArgSeparator: func(_userArgs []string) []string { return nil }, @@ -43,6 +45,18 @@ var nodejsBerry = PackageManager{ return true, nil }, + GetLockfileName: func(_ turbopath.AbsoluteSystemPath) string { + return berryLockfile + }, + + GetLockfilePath: func(projectDirectory turbopath.AbsoluteSystemPath) turbopath.AbsoluteSystemPath { + return projectDirectory.UntypedJoin(berryLockfile) + }, + + GetLockfileContents: func(projectDirectory turbopath.AbsoluteSystemPath) ([]byte, error) { + return projectDirectory.UntypedJoin(berryLockfile).ReadFile() + }, + UnmarshalLockfile: func(rootPackageJSON *fs.PackageJSON, contents []byte) (lockfile.Lockfile, error) { var resolutions map[string]string if untypedResolutions, ok := rootPackageJSON.RawJSON["resolutions"]; ok { diff --git a/cli/internal/packagemanager/bun.go b/cli/internal/packagemanager/bun.go index 33cf88ecd6166..2b2116e64d3e4 100644 --- a/cli/internal/packagemanager/bun.go +++ b/cli/internal/packagemanager/bun.go @@ -2,18 +2,26 @@ package packagemanager import ( "fmt" + "os/exec" "github.com/vercel/turbo/cli/internal/fs" "github.com/vercel/turbo/cli/internal/lockfile" "github.com/vercel/turbo/cli/internal/turbopath" ) +const command = "bun" +const bunLockfile = "bun.lockb" + +func getLockfilePath(rootPath turbopath.AbsoluteSystemPath) turbopath.AbsoluteSystemPath { + return rootPath.UntypedJoin(bunLockfile) +} + var nodejsBun = PackageManager{ Name: "nodejs-bun", Slug: "bun", - Command: "bun", + Command: command, Specfile: "package.json", - Lockfile: "bun.lockb", + Lockfile: bunLockfile, PackageDir: "node_modules", ArgSeparator: func(userArgs []string) []string { // Bun swallows a single "--" token. If the user is passing "--", we need @@ -48,7 +56,23 @@ var nodejsBun = PackageManager{ }, canPrune: func(cwd turbopath.AbsoluteSystemPath) (bool, error) { - return false, nil + return true, nil + }, + + GetLockfileName: func(rootPath turbopath.AbsoluteSystemPath) string { + return bunLockfile + }, + + GetLockfilePath: func(rootPath turbopath.AbsoluteSystemPath) turbopath.AbsoluteSystemPath { + return getLockfilePath(rootPath) + }, + + GetLockfileContents: func(projectDirectory turbopath.AbsoluteSystemPath) ([]byte, error) { + lockfilePath := getLockfilePath(projectDirectory) + cmd := exec.Command(command, lockfilePath.ToString()) + cmd.Dir = projectDirectory.ToString() + + return cmd.Output() }, UnmarshalLockfile: func(_rootPackageJSON *fs.PackageJSON, contents []byte) (lockfile.Lockfile, error) { diff --git a/cli/internal/packagemanager/npm.go b/cli/internal/packagemanager/npm.go index 5c8e4d08d469c..8c082b962213b 100644 --- a/cli/internal/packagemanager/npm.go +++ b/cli/internal/packagemanager/npm.go @@ -8,12 +8,14 @@ import ( "github.com/vercel/turbo/cli/internal/turbopath" ) +const npmLockfile = "package-lock.json" + var nodejsNpm = PackageManager{ Name: "nodejs-npm", Slug: "npm", Command: "npm", Specfile: "package.json", - Lockfile: "package-lock.json", + Lockfile: npmLockfile, PackageDir: "node_modules", ArgSeparator: func(_userArgs []string) []string { return []string{"--"} }, @@ -42,6 +44,18 @@ var nodejsNpm = PackageManager{ return true, nil }, + GetLockfileName: func(_ turbopath.AbsoluteSystemPath) string { + return npmLockfile + }, + + GetLockfilePath: func(projectDirectory turbopath.AbsoluteSystemPath) turbopath.AbsoluteSystemPath { + return projectDirectory.UntypedJoin(npmLockfile) + }, + + GetLockfileContents: func(projectDirectory turbopath.AbsoluteSystemPath) ([]byte, error) { + return projectDirectory.UntypedJoin(npmLockfile).ReadFile() + }, + UnmarshalLockfile: func(_rootPackageJSON *fs.PackageJSON, contents []byte) (lockfile.Lockfile, error) { return lockfile.DecodeNpmLockfile(contents) }, diff --git a/cli/internal/packagemanager/packagemanager.go b/cli/internal/packagemanager/packagemanager.go index 7c16d94222ad0..c45a90eeddc16 100644 --- a/cli/internal/packagemanager/packagemanager.go +++ b/cli/internal/packagemanager/packagemanager.go @@ -51,6 +51,15 @@ type PackageManager struct { // Detect if Turbo knows how to produce a pruned workspace for the project canPrune func(cwd turbopath.AbsoluteSystemPath) (bool, error) + // Gets lockfile name. + GetLockfileName func(projectDirectory turbopath.AbsoluteSystemPath) string + + // Gets lockfile path. + GetLockfilePath func(projectDirectory turbopath.AbsoluteSystemPath) turbopath.AbsoluteSystemPath + + // Read from disk a lockfile for a package manager. + GetLockfileContents func(projectDirectory turbopath.AbsoluteSystemPath) ([]byte, error) + // Read a lockfile for a given package manager UnmarshalLockfile func(rootPackageJSON *fs.PackageJSON, contents []byte) (lockfile.Lockfile, error) @@ -122,13 +131,13 @@ func (pm PackageManager) ReadLockfile(projectDirectory turbopath.AbsoluteSystemP if pm.UnmarshalLockfile == nil { return nil, nil } - contents, err := projectDirectory.UntypedJoin(pm.Lockfile).ReadFile() + contents, err := pm.GetLockfileContents(projectDirectory) if err != nil { - return nil, fmt.Errorf("reading %s: %w", pm.Lockfile, err) + return nil, fmt.Errorf("reading %s: %w", pm.GetLockfilePath(projectDirectory), err) } lf, err := pm.UnmarshalLockfile(rootPackageJSON, contents) if err != nil { - return nil, errors.Wrapf(err, "error in %v", pm.Lockfile) + return nil, errors.Wrapf(err, "error in %v", pm.GetLockfilePath(projectDirectory)) } return lf, nil } diff --git a/cli/internal/packagemanager/packagemanager_test.go b/cli/internal/packagemanager/packagemanager_test.go index 3be3b57cfedff..c22733a6c1413 100644 --- a/cli/internal/packagemanager/packagemanager_test.go +++ b/cli/internal/packagemanager/packagemanager_test.go @@ -26,6 +26,7 @@ func Test_GetWorkspaces(t *testing.T) { repoRoot, err := fs.GetCwd(cwd) assert.NilError(t, err, "GetCwd") rootPath := map[string]turbopath.AbsoluteSystemPath{ + "nodejs-bun": repoRoot.UntypedJoin("../../../examples/with-yarn"), "nodejs-npm": repoRoot.UntypedJoin("../../../examples/with-yarn"), "nodejs-berry": repoRoot.UntypedJoin("../../../examples/with-yarn"), "nodejs-yarn": repoRoot.UntypedJoin("../../../examples/with-yarn"), @@ -34,6 +35,13 @@ func Test_GetWorkspaces(t *testing.T) { } want := map[string][]string{ + "nodejs-bun": { + filepath.ToSlash(filepath.Join(cwd, "../../../examples/with-yarn/apps/docs/package.json")), + filepath.ToSlash(filepath.Join(cwd, "../../../examples/with-yarn/apps/web/package.json")), + filepath.ToSlash(filepath.Join(cwd, "../../../examples/with-yarn/packages/eslint-config-custom/package.json")), + filepath.ToSlash(filepath.Join(cwd, "../../../examples/with-yarn/packages/tsconfig/package.json")), + filepath.ToSlash(filepath.Join(cwd, "../../../examples/with-yarn/packages/ui/package.json")), + }, "nodejs-npm": { filepath.ToSlash(filepath.Join(cwd, "../../../examples/with-yarn/apps/docs/package.json")), filepath.ToSlash(filepath.Join(cwd, "../../../examples/with-yarn/apps/web/package.json")), @@ -117,6 +125,7 @@ func Test_GetWorkspaceIgnores(t *testing.T) { cwd, err := fs.GetCwd(cwdRaw) assert.NilError(t, err, "GetCwd") want := map[string][]string{ + "nodejs-bun": {"**/node_modules", "**/.git"}, "nodejs-npm": {"**/node_modules/**"}, "nodejs-berry": {"**/node_modules", "**/.git", "**/.yarn"}, "nodejs-yarn": {"apps/*/node_modules/**", "packages/*/node_modules/**"}, diff --git a/cli/internal/packagemanager/pnpm.go b/cli/internal/packagemanager/pnpm.go index f1f6069801c89..96d7382592c6c 100644 --- a/cli/internal/packagemanager/pnpm.go +++ b/cli/internal/packagemanager/pnpm.go @@ -12,6 +12,8 @@ import ( "github.com/vercel/turbo/cli/internal/yaml" ) +const pnpmLockfile = "pnpm-lock.yaml" + // PnpmWorkspaces is a representation of workspace package globs found // in pnpm-workspace.yaml type PnpmWorkspaces struct { @@ -79,7 +81,7 @@ var nodejsPnpm = PackageManager{ Slug: "pnpm", Command: "pnpm", Specfile: "package.json", - Lockfile: "pnpm-lock.yaml", + Lockfile: pnpmLockfile, PackageDir: "node_modules", // pnpm v7+ changed their handling of '--'. We no longer need to pass it to pass args to // the script being run, and in fact doing so will cause the '--' to be passed through verbatim, @@ -98,6 +100,18 @@ var nodejsPnpm = PackageManager{ return true, nil }, + GetLockfileName: func(_ turbopath.AbsoluteSystemPath) string { + return pnpmLockfile + }, + + GetLockfilePath: func(projectDirectory turbopath.AbsoluteSystemPath) turbopath.AbsoluteSystemPath { + return projectDirectory.UntypedJoin(pnpmLockfile) + }, + + GetLockfileContents: func(projectDirectory turbopath.AbsoluteSystemPath) ([]byte, error) { + return projectDirectory.UntypedJoin(pnpmLockfile).ReadFile() + }, + UnmarshalLockfile: func(_rootPackageJSON *fs.PackageJSON, contents []byte) (lockfile.Lockfile, error) { return lockfile.DecodePnpmLockfile(contents) }, diff --git a/cli/internal/packagemanager/pnpm6.go b/cli/internal/packagemanager/pnpm6.go index 489962dc11433..d19076910283e 100644 --- a/cli/internal/packagemanager/pnpm6.go +++ b/cli/internal/packagemanager/pnpm6.go @@ -6,6 +6,8 @@ import ( "github.com/vercel/turbo/cli/internal/turbopath" ) +const pnpm6Lockfile = "pnpm-lock.yaml" + // Pnpm6Workspaces is a representation of workspace package globs found // in pnpm-workspace.yaml type Pnpm6Workspaces struct { @@ -17,7 +19,7 @@ var nodejsPnpm6 = PackageManager{ Slug: "pnpm", Command: "pnpm", Specfile: "package.json", - Lockfile: "pnpm-lock.yaml", + Lockfile: pnpm6Lockfile, PackageDir: "node_modules", ArgSeparator: func(_userArgs []string) []string { return []string{"--"} }, WorkspaceConfigurationPath: "pnpm-workspace.yaml", @@ -30,6 +32,18 @@ var nodejsPnpm6 = PackageManager{ return true, nil }, + GetLockfileName: func(_ turbopath.AbsoluteSystemPath) string { + return pnpm6Lockfile + }, + + GetLockfilePath: func(projectDirectory turbopath.AbsoluteSystemPath) turbopath.AbsoluteSystemPath { + return projectDirectory.UntypedJoin(pnpm6Lockfile) + }, + + GetLockfileContents: func(projectDirectory turbopath.AbsoluteSystemPath) ([]byte, error) { + return projectDirectory.UntypedJoin(pnpm6Lockfile).ReadFile() + }, + UnmarshalLockfile: func(_rootPackageJSON *fs.PackageJSON, contents []byte) (lockfile.Lockfile, error) { return lockfile.DecodePnpmLockfile(contents) }, diff --git a/cli/internal/packagemanager/yarn.go b/cli/internal/packagemanager/yarn.go index 4044b21eb903a..53241b1194a10 100644 --- a/cli/internal/packagemanager/yarn.go +++ b/cli/internal/packagemanager/yarn.go @@ -17,12 +17,14 @@ func (e *NoWorkspacesFoundError) Error() string { return "package.json: no workspaces found. Turborepo requires Yarn workspaces to be defined in the root package.json" } +const yarnLockfile = "yarn.lock" + var nodejsYarn = PackageManager{ Name: "nodejs-yarn", Slug: "yarn", Command: "yarn", Specfile: "package.json", - Lockfile: "yarn.lock", + Lockfile: yarnLockfile, PackageDir: "node_modules", ArgSeparator: func(userArgs []string) []string { // Yarn warns and swallows a "--" token. If the user is passing "--", we need @@ -79,6 +81,18 @@ var nodejsYarn = PackageManager{ return true, nil }, + GetLockfileName: func(_ turbopath.AbsoluteSystemPath) string { + return yarnLockfile + }, + + GetLockfilePath: func(projectDirectory turbopath.AbsoluteSystemPath) turbopath.AbsoluteSystemPath { + return projectDirectory.UntypedJoin(yarnLockfile) + }, + + GetLockfileContents: func(projectDirectory turbopath.AbsoluteSystemPath) ([]byte, error) { + return projectDirectory.UntypedJoin(yarnLockfile).ReadFile() + }, + UnmarshalLockfile: func(_rootPackageJSON *fs.PackageJSON, contents []byte) (lockfile.Lockfile, error) { return lockfile.DecodeYarnLockfile(contents) }, diff --git a/cli/internal/run/global_hash.go b/cli/internal/run/global_hash.go index 32a9eb4baa5f8..8bcc98b96670b 100644 --- a/cli/internal/run/global_hash.go +++ b/cli/internal/run/global_hash.go @@ -121,8 +121,9 @@ func getGlobalHashInputs( if lockFile == nil { // If we don't have lockfile information available, add the specfile and lockfile to global deps globalDeps.Add(filepath.Join(rootpath.ToStringDuringMigration(), packageManager.Specfile)) - if rootpath.UntypedJoin(packageManager.Lockfile).Exists() { - globalDeps.Add(filepath.Join(rootpath.ToStringDuringMigration(), packageManager.Lockfile)) + lockfilePath := packageManager.GetLockfilePath(rootpath) + if lockfilePath.Exists() { + globalDeps.Add(lockfilePath) } } diff --git a/cli/internal/scope/scope.go b/cli/internal/scope/scope.go index 3615f9e1e142d..1804c7df78b8b 100644 --- a/cli/internal/scope/scope.go +++ b/cli/internal/scope/scope.go @@ -205,7 +205,7 @@ func calculateInference(repoRoot turbopath.AbsoluteSystemPath, pkgInferencePath }, nil } -func (o *Opts) getPackageChangeFunc(scm scm.SCM, cwd turbopath.AbsoluteSystemPath, ctx *context.Context) scope_filter.PackagesChangedInRange { +func (o *Opts) getPackageChangeFunc(scm scm.SCM, repoRoot turbopath.AbsoluteSystemPath, ctx *context.Context) scope_filter.PackagesChangedInRange { return func(fromRef string, toRef string) (util.Set, error) { // We could filter changed files at the git level, since it's possible // that the changes we're interested in are scoped, but we need to handle @@ -213,7 +213,7 @@ func (o *Opts) getPackageChangeFunc(scm scm.SCM, cwd turbopath.AbsoluteSystemPat // scope changed files more deeply if we know there are no global dependencies. var changedFiles []string if fromRef != "" { - scmChangedFiles, err := scm.ChangedFiles(fromRef, toRef, cwd.ToStringDuringMigration()) + scmChangedFiles, err := scm.ChangedFiles(fromRef, toRef, repoRoot.ToStringDuringMigration()) if err != nil { return nil, err } @@ -239,7 +239,7 @@ func (o *Opts) getPackageChangeFunc(scm scm.SCM, cwd turbopath.AbsoluteSystemPat } changedPkgs := getChangedPackages(filteredChangedFiles, ctx.WorkspaceInfos) - if lockfileChanges, fullChanges := getChangesFromLockfile(scm, ctx, changedFiles, fromRef); !fullChanges { + if lockfileChanges, fullChanges := getChangesFromLockfile(repoRoot, scm, ctx, changedFiles, fromRef); !fullChanges { for _, pkg := range lockfileChanges { changedPkgs.Add(pkg) } @@ -251,8 +251,8 @@ func (o *Opts) getPackageChangeFunc(scm scm.SCM, cwd turbopath.AbsoluteSystemPat } } -func getChangesFromLockfile(scm scm.SCM, ctx *context.Context, changedFiles []string, fromRef string) ([]string, bool) { - lockfileFilter, err := filter.Compile([]string{ctx.PackageManager.Lockfile}) +func getChangesFromLockfile(repoRoot turbopath.AbsoluteSystemPath, scm scm.SCM, ctx *context.Context, changedFiles []string, fromRef string) ([]string, bool) { + lockfileFilter, err := filter.Compile([]string{ctx.PackageManager.GetLockfileName(repoRoot)}) if err != nil { panic(fmt.Sprintf("Lockfile is invalid glob: %v", err)) } @@ -271,7 +271,8 @@ func getChangesFromLockfile(scm scm.SCM, ctx *context.Context, changedFiles []st return nil, true } - prevContents, err := scm.PreviousContent(fromRef, ctx.PackageManager.Lockfile) + // FIXME: If you move your bun lockfile then we don't track that move into the history. + prevContents, err := scm.PreviousContent(fromRef, ctx.PackageManager.GetLockfileName(repoRoot)) if err != nil { // unable to reconstruct old lockfile, assume everything changed return nil, true diff --git a/cli/internal/scope/scope_test.go b/cli/internal/scope/scope_test.go index ba03a68bf457c..5f0525b244b92 100644 --- a/cli/internal/scope/scope_test.go +++ b/cli/internal/scope/scope_test.go @@ -530,7 +530,7 @@ func TestResolvePackages(t *testing.T) { }, root, scm, &context.Context{ WorkspaceInfos: workspaceInfos, WorkspaceNames: packageNames, - PackageManager: &packagemanager.PackageManager{Lockfile: tc.lockfile, UnmarshalLockfile: readLockfile}, + PackageManager: &packagemanager.PackageManager{Lockfile: tc.lockfile, UnmarshalLockfile: readLockfile, GetLockfileName: func(_ turbopath.AbsoluteSystemPath) string { return tc.lockfile }}, WorkspaceGraph: graph, RootNode: "root", Lockfile: tc.currLockfile, From 31ee66b84356386bededaeae26d84e0c5dd0f3ed Mon Sep 17 00:00:00 2001 From: Nathan Hammond Date: Wed, 13 Sep 2023 09:55:08 +0800 Subject: [PATCH 04/11] Weak type system means runtime panics. --- cli/internal/packagemanager/packagemanager.go | 4 ++-- cli/internal/run/global_hash.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/internal/packagemanager/packagemanager.go b/cli/internal/packagemanager/packagemanager.go index c45a90eeddc16..de5c52587889b 100644 --- a/cli/internal/packagemanager/packagemanager.go +++ b/cli/internal/packagemanager/packagemanager.go @@ -133,11 +133,11 @@ func (pm PackageManager) ReadLockfile(projectDirectory turbopath.AbsoluteSystemP } contents, err := pm.GetLockfileContents(projectDirectory) if err != nil { - return nil, fmt.Errorf("reading %s: %w", pm.GetLockfilePath(projectDirectory), err) + return nil, fmt.Errorf("reading %s: %w", pm.GetLockfilePath(projectDirectory).ToString(), err) } lf, err := pm.UnmarshalLockfile(rootPackageJSON, contents) if err != nil { - return nil, errors.Wrapf(err, "error in %v", pm.GetLockfilePath(projectDirectory)) + return nil, errors.Wrapf(err, "error in %v", pm.GetLockfilePath(projectDirectory).ToString()) } return lf, nil } diff --git a/cli/internal/run/global_hash.go b/cli/internal/run/global_hash.go index 8bcc98b96670b..55173796f0523 100644 --- a/cli/internal/run/global_hash.go +++ b/cli/internal/run/global_hash.go @@ -123,7 +123,7 @@ func getGlobalHashInputs( globalDeps.Add(filepath.Join(rootpath.ToStringDuringMigration(), packageManager.Specfile)) lockfilePath := packageManager.GetLockfilePath(rootpath) if lockfilePath.Exists() { - globalDeps.Add(lockfilePath) + globalDeps.Add(lockfilePath.ToString()) } } From f8a8f4c7d76de0fd0897e710110ad8440de488e4 Mon Sep 17 00:00:00 2001 From: Nathan Hammond Date: Wed, 13 Sep 2023 12:55:22 +0800 Subject: [PATCH 05/11] Disable pruning for Bun. --- cli/internal/packagemanager/bun.go | 2 +- crates/turborepo-ffi/src/lockfile.rs | 4 +- crates/turborepo-lib/src/commands/prune.rs | 9 + crates/turborepo-lockfiles/src/bun/mod.rs | 25 +-- crates/turborepo-lockfiles/src/bun/ser.rs | 215 --------------------- crates/turborepo-lockfiles/src/lib.rs | 2 +- 6 files changed, 18 insertions(+), 239 deletions(-) delete mode 100644 crates/turborepo-lockfiles/src/bun/ser.rs diff --git a/cli/internal/packagemanager/bun.go b/cli/internal/packagemanager/bun.go index 2b2116e64d3e4..4f908ef2f3d2c 100644 --- a/cli/internal/packagemanager/bun.go +++ b/cli/internal/packagemanager/bun.go @@ -56,7 +56,7 @@ var nodejsBun = PackageManager{ }, canPrune: func(cwd turbopath.AbsoluteSystemPath) (bool, error) { - return true, nil + return false, nil }, GetLockfileName: func(rootPath turbopath.AbsoluteSystemPath) string { diff --git a/crates/turborepo-ffi/src/lockfile.rs b/crates/turborepo-ffi/src/lockfile.rs index baa4e79cab3e8..42f340f208883 100644 --- a/crates/turborepo-ffi/src/lockfile.rs +++ b/crates/turborepo-ffi/src/lockfile.rs @@ -181,7 +181,9 @@ fn subgraph_inner(buf: Buffer) -> Result, Error> { turborepo_lockfiles::pnpm_subgraph(&contents, &workspaces, &packages)? } proto::PackageManager::Yarn => turborepo_lockfiles::yarn_subgraph(&contents, &packages)?, - proto::PackageManager::Bun => turborepo_lockfiles::bun_subgraph(&contents, &packages)?, + proto::PackageManager::Bun => { + return Err(Error::UnsupportedPackageManager(proto::PackageManager::Bun)) + } }; Ok(contents) } diff --git a/crates/turborepo-lib/src/commands/prune.rs b/crates/turborepo-lib/src/commands/prune.rs index f15dc23d0bda6..f4e8c7970645b 100644 --- a/crates/turborepo-lib/src/commands/prune.rs +++ b/crates/turborepo-lib/src/commands/prune.rs @@ -40,6 +40,8 @@ pub enum Error { MissingWorkspace(WorkspaceName), #[error("Cannot prune without parsed lockfile")] MissingLockfile, + #[error("Prune is not supported for Bun")] + BunUnsupported, } // Files that should be copied from root and if they're required for install @@ -71,6 +73,13 @@ pub fn prune( ) -> Result<(), Error> { let prune = Prune::new(base, scope, docker, output_dir)?; + if matches!( + prune.package_graph.package_manager(), + crate::package_manager::PackageManager::Bun + ) { + return Err(Error::BunUnsupported); + } + println!( "Generating pruned monorepo for {} in {}", base.ui.apply(BOLD.apply_to(scope.join(", "))), diff --git a/crates/turborepo-lockfiles/src/bun/mod.rs b/crates/turborepo-lockfiles/src/bun/mod.rs index 6bd27e8d1ae48..e7624b34f2f0c 100644 --- a/crates/turborepo-lockfiles/src/bun/mod.rs +++ b/crates/turborepo-lockfiles/src/bun/mod.rs @@ -5,7 +5,6 @@ use serde::Deserialize; use crate::Lockfile; mod de; -mod ser; type Map = std::collections::BTreeMap; @@ -17,6 +16,8 @@ pub enum Error { SymlStructure(#[from] serde_json::Error), #[error("unexpected non-utf8 yarn.lock")] NonUTF8(#[from] std::str::Utf8Error), + #[error("Turborepo cannot serialize Bun lockfiles.")] + NotImplemented(), } pub struct BunLockfile { @@ -105,20 +106,14 @@ impl Lockfile for BunLockfile { } fn encode(&self) -> Result, crate::Error> { - Ok(self.to_string().into_bytes()) + Err(crate::Error::Bun(Error::NotImplemented())) } fn global_change_key(&self) -> Vec { - vec![b'y', b'a', b'r', b'n', 0] + vec![b'b', b'u', b'n', 0] } } -pub fn bun_subgraph(contents: &[u8], packages: &[String]) -> Result, crate::Error> { - let lockfile = BunLockfile::from_bytes(contents)?; - let pruned_lockfile = lockfile.subgraph(&[], packages)?; - pruned_lockfile.encode() -} - impl Entry { fn dependency_entries(&self) -> impl Iterator + '_ { self.dependencies @@ -140,21 +135,9 @@ fn possible_keys<'a>(name: &'a str, version: &'a str) -> impl Iterator HashMap<&Entry, HashSet<&str>> { - let mut reverse_lookup = HashMap::new(); - for (key, value) in self.inner.iter() { - let keys: &mut HashSet<&str> = reverse_lookup.entry(value).or_default(); - keys.insert(key); - } - reverse_lookup - } -} - -impl fmt::Display for BunLockfile { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str( - "# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.\n# yarn lockfile \ - v1\n\n", - )?; - let reverse_lookup = self.reverse_lookup(); - let mut added_keys: HashSet<&str> = HashSet::with_capacity(self.inner.len()); - for (key, entry) in self.inner.iter() { - if added_keys.contains(key.as_str()) { - continue; - } - - let all_keys = reverse_lookup - .get(entry) - .expect("entry in lockfile should appear as a key in reverse lookup"); - added_keys.extend(all_keys); - let mut keys = all_keys.iter().copied().collect::>(); - // Keys must be sorted before they get wrapped - keys.sort(); - - let wrapped_keys = keys.into_iter().map(maybe_wrap).collect::>(); - let key_line = wrapped_keys.join(", "); - - f.write_fmt(format_args!("\n{}:\n{}\n", key_line, entry))?; - } - Ok(()) - } -} - -impl fmt::Display for Entry { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let mut leading = LeadingNewline::new(); - if let Some(name) = &self.name { - f.write_fmt(format_args!( - "{}{INDENT}name {}", - leading.leading(), - maybe_wrap(name) - ))?; - } - f.write_fmt(format_args!( - "{}{INDENT}version {}", - leading.leading(), - maybe_wrap(&self.version) - ))?; - if let Some(uid) = &self.uid { - f.write_fmt(format_args!( - "{}{INDENT}uid {}", - leading.leading(), - maybe_wrap(uid) - ))?; - } - if let Some(resolved) = &self.resolved { - f.write_fmt(format_args!( - "{}{INDENT}resolved {}", - leading.leading(), - maybe_wrap(resolved) - ))?; - } - if let Some(integrity) = &self.integrity { - f.write_fmt(format_args!( - "{}{INDENT}integrity {}", - leading.leading(), - maybe_wrap(integrity) - ))?; - } - if let Some(registry) = &self.registry { - f.write_fmt(format_args!( - "{}{INDENT}integrity {}", - leading.leading(), - maybe_wrap(registry) - ))?; - } - // encode deps and optional deps - if let Some(deps) = &self.dependencies { - f.write_fmt(format_args!("{}{INDENT}dependencies:", leading.leading()))?; - encode_map(deps.iter().map(|(k, v)| (k.as_ref(), v.as_ref())), f)?; - } - if let Some(optional_deps) = &self.optional_dependencies { - f.write_fmt(format_args!( - "{}{INDENT}optionalDependencies:", - leading.leading() - ))?; - encode_map( - optional_deps.iter().map(|(k, v)| (k.as_ref(), v.as_ref())), - f, - )?; - } - Ok(()) - } -} - -#[derive(Debug, Clone, Copy)] -enum LeadingNewline { - First, - Rest, -} - -impl LeadingNewline { - fn new() -> Self { - Self::First - } - - fn leading(&mut self) -> &'static str { - let res = match self { - LeadingNewline::First => "", - LeadingNewline::Rest => "\n", - }; - *self = Self::Rest; - res - } -} - -fn encode_map<'a, I: Iterator>( - entries: I, - f: &mut fmt::Formatter<'_>, -) -> fmt::Result { - let mut wrapped_entries = entries - .map(|(k, v)| (maybe_wrap(k), maybe_wrap(v))) - .collect::>(); - wrapped_entries.sort_unstable_by(|(k1, _), (k2, _)| k1.cmp(k2)); - // we sort the via wrapped keys - // then we write each line with the value wrapped as well - for (key, value) in wrapped_entries { - f.write_fmt(format_args!("\n{INDENT}{INDENT}{key} {value}"))?; - } - - Ok(()) -} - -fn maybe_wrap(s: &str) -> Cow { - match should_wrap_key(s) { - // yarn uses JSON.stringify to escape strings - // we approximate this behavior using serde_json - true => serde_json::to_string(s) - .expect("failed at encoding string as json") - .into(), - false => s.into(), - } -} - -// Determines if we need to wrap a key -fn should_wrap_key(s: &str) -> bool { - // Wrap if it starts with a syml keyword - s.starts_with("true") || - s.starts_with("false") || - // Wrap if it doesn't start with a-zA-Z - s.chars().next().map_or(false, |c| !c.is_ascii_alphabetic()) || - // Wrap if it contains any unwanted chars - s.chars().any(|c| matches!( - c, - ' ' | ':' | '\t' | '\r' | '\u{000B}' | '\u{000C}' | '\n' | '\\' | '"' | ',' | '[' | ']' - )) -} - -#[cfg(test)] -mod test { - use pretty_assertions::assert_eq; - - use super::*; - - #[test] - fn test_should_wrap() { - assert!(should_wrap_key("jsx-ast-utils@^2.4.1 || ^3.0.0")) - } - - #[test] - fn test_basic_serialization() { - let entry = Entry { - version: "12.2.5".into(), - resolved: Some("https://registry.yarnpkg.com/next/-/next-12.2.5.tgz#14fb5975e8841fad09553b8ef41fe1393602b717".into()), - integrity: Some("sha512-tBdjqX5XC/oFs/6gxrZhjmiq90YWizUYU6qOWAfat7zJwrwapJ+BYgX2PmiacunXMaRpeVT4vz5MSPSLgNkrpA==".into()), - dependencies: Some(vec![ - ("@next/env".into(), "12.2.5".into()), - ("caniuse-lite".into(), "^1.0.30001332".into()), - ("postcss".into(), "8.4.14".into()), - ].into_iter().collect()), - optional_dependencies: Some(vec![("@next/swc-win32-x64-msvc".into(), "12.2.5".into())].into_iter().collect()), - ..Default::default() - }; - assert_eq!( - entry.to_string(), - r#" version "12.2.5" - resolved "https://registry.yarnpkg.com/next/-/next-12.2.5.tgz#14fb5975e8841fad09553b8ef41fe1393602b717" - integrity sha512-tBdjqX5XC/oFs/6gxrZhjmiq90YWizUYU6qOWAfat7zJwrwapJ+BYgX2PmiacunXMaRpeVT4vz5MSPSLgNkrpA== - dependencies: - "@next/env" "12.2.5" - caniuse-lite "^1.0.30001332" - postcss "8.4.14" - optionalDependencies: - "@next/swc-win32-x64-msvc" "12.2.5""# - ); - } -} diff --git a/crates/turborepo-lockfiles/src/lib.rs b/crates/turborepo-lockfiles/src/lib.rs index 7cc410669d22b..571ab475b201f 100644 --- a/crates/turborepo-lockfiles/src/lib.rs +++ b/crates/turborepo-lockfiles/src/lib.rs @@ -10,7 +10,7 @@ mod yarn1; use std::collections::{HashMap, HashSet}; pub use berry::{Error as BerryError, *}; -pub use bun::{bun_subgraph, BunLockfile}; +pub use bun::BunLockfile; pub use error::Error; pub use npm::*; pub use pnpm::{pnpm_global_change, pnpm_subgraph, PnpmLockfile}; From 636b3c5f534df57cbcc4e559f5f275fe1349b78b Mon Sep 17 00:00:00 2001 From: Nathan Hammond Date: Thu, 14 Sep 2023 12:01:15 +0800 Subject: [PATCH 06/11] SUpport bunfig for lockfile. --- Cargo.lock | 49 +++++- crates/turborepo-lib/Cargo.toml | 1 + .../turborepo-lib/src/package_manager/bun.rs | 148 +++++++++++++++--- .../turborepo-lib/src/package_manager/mod.rs | 34 +++- crates/turborepo-lib/src/run/global_hash.rs | 7 +- .../src/run/scope/change_detector.rs | 4 +- 6 files changed, 205 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 751a0d4cd0eca..adf31ace30d84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5369,7 +5369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" dependencies = [ "once_cell", - "toml_edit", + "toml_edit 0.19.8", ] [[package]] @@ -6335,9 +6335,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" +checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" dependencies = [ "serde", ] @@ -8573,14 +8573,26 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit", + "toml_edit 0.19.8", +] + +[[package]] +name = "toml" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c226a7bba6d859b63c92c4b4fe69c5b6b72d0cb897dbc8e6012298e6154cb56e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit 0.20.0", ] [[package]] name = "toml_datetime" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" dependencies = [ "serde", ] @@ -8595,7 +8607,20 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "winnow", + "winnow 0.4.1", +] + +[[package]] +name = "toml_edit" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ff63e60a958cefbb518ae1fd6566af80d9d4be430a33f3723dfc47d1d411d95" +dependencies = [ + "indexmap 2.0.0", + "serde", + "serde_spanned", + "toml_datetime", + "winnow 0.5.15", ] [[package]] @@ -9953,6 +9978,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-util", + "toml 0.8.0", "tonic", "tonic-build", "tonic-reflection", @@ -11288,6 +11314,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "winnow" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.10.1" diff --git a/crates/turborepo-lib/Cargo.toml b/crates/turborepo-lib/Cargo.toml index 003dbac1226eb..512710d57714b 100644 --- a/crates/turborepo-lib/Cargo.toml +++ b/crates/turborepo-lib/Cargo.toml @@ -98,6 +98,7 @@ num_cpus = "1.15.0" owo-colors.workspace = true rayon = "1.7.0" regex.workspace = true +toml = "0.8.0" tracing-appender = "0.2.2" tracing-chrome = { version = "0.7.1", optional = true } tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } diff --git a/crates/turborepo-lib/src/package_manager/bun.rs b/crates/turborepo-lib/src/package_manager/bun.rs index 4ed64cb207d46..442be6ad38da1 100644 --- a/crates/turborepo-lib/src/package_manager/bun.rs +++ b/crates/turborepo-lib/src/package_manager/bun.rs @@ -1,8 +1,125 @@ -use turbopath::AbsoluteSystemPath; +use std::{fs, path::PathBuf}; + +use serde::{Deserialize, Serialize}; +use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf}; use crate::package_manager::{Error, PackageManager}; -pub const LOCKFILE: &str = "bun.lockb"; +const GLOBAL_BUNFIG: &'static str = ".bunfig.toml"; +const LOCAL_BUNFIG: &'static str = "bunfig.toml"; +pub const LOCKFILE: &'static str = "bun.lockb"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct Bunfig { + pub install: Option, +} + +impl Default for Bunfig { + fn default() -> Self { + Self { + install: Some(Default::default()), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct Install { + pub lockfile: Option, +} + +impl Default for Install { + fn default() -> Self { + Self { + lockfile: Some(Default::default()), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct Lockfile { + pub path: Option, +} + +impl Default for Lockfile { + fn default() -> Self { + Self { + path: Some(String::from("bun.lockb")), + } + } +} + +fn global_bunfig() -> Result, std::io::Error> { + // Ordering: https://github.com/oven-sh/bun/blob/75b5c715405d49b5026c13143efecd580d27be1b/src/cli.zig#L303 + if let Some(data_dir) = std::env::var_os("XDG_CONFIG_HOME").or_else(|| std::env::var_os("HOME")) + { + process_bunfig_read(fs::read_to_string( + PathBuf::from(data_dir).join(GLOBAL_BUNFIG), + )) + } else { + Ok(None) + } +} + +fn local_bunfig(repo_root: &AbsoluteSystemPath) -> Result, std::io::Error> { + process_bunfig_read(repo_root.join_component(LOCAL_BUNFIG).read_to_string()) +} + +fn process_bunfig_read( + read_result: Result, +) -> Result, std::io::Error> { + match read_result { + Ok(bunfig_contents) => { + let deserialize_result: Bunfig = toml::from_str(&bunfig_contents).unwrap(); + Ok(Some(deserialize_result)) + } + Err(error) => { + if matches!(error.kind(), std::io::ErrorKind::NotFound) { + Ok(None) + } else { + Err(error) + } + } + } +} + +pub fn get_lockfile_path( + repo_root: &AbsoluteSystemPath, +) -> Result { + let global = global_bunfig()?; + let local = local_bunfig(repo_root)?; + + let path = match (global, local) { + (None, None) => None, + (None, Some(local_bunfig)) => local_bunfig + .install + .and_then(|install| install.lockfile) + .and_then(|lockfile| lockfile.path) + .and_then(|path| if path.is_empty() { None } else { Some(path) }), + (Some(global_bunfig), None) => global_bunfig + .install + .and_then(|install| install.lockfile) + .and_then(|lockfile| lockfile.path) + .and_then(|path| if path.is_empty() { None } else { Some(path) }), + (Some(global_bunfig), Some(local_bunfig)) => { + // Bun shallow merges. + local_bunfig + .install + .or(global_bunfig.install) + .and_then(|install| install.lockfile) + .and_then(|lockfile| lockfile.path) + .and_then(|path| if path.is_empty() { None } else { Some(path) }) + } + }; + + match path { + Some(path) => { + // This is possibly garbage-in. + // But if it is garbage-in, we can be garbage out. + Ok(AbsoluteSystemPathBuf::from_unknown(repo_root, path)) + } + None => Ok(repo_root.join_component(LOCKFILE)), + } +} pub struct BunDetector<'a> { repo_root: &'a AbsoluteSystemPath, @@ -27,9 +144,10 @@ impl<'a> Iterator for BunDetector<'a> { } self.found = true; - let package_json = self.repo_root.join_component(LOCKFILE); + let bunfig_lockfile_path = get_lockfile_path(self.repo_root) + .unwrap_or_else(|_| self.repo_root.join_component(LOCKFILE)); - if package_json.exists() { + if bunfig_lockfile_path.exists() { Some(Ok(PackageManager::Bun)) } else { None @@ -39,25 +157,11 @@ impl<'a> Iterator for BunDetector<'a> { #[cfg(test)] mod tests { - use std::fs::File; - - use anyhow::Result; - use tempfile::tempdir; - use turbopath::AbsoluteSystemPathBuf; - - use super::LOCKFILE; - use crate::package_manager::PackageManager; + use super::*; #[test] - fn test_detect_bun() -> Result<()> { - let repo_root = tempdir()?; - let repo_root_path = AbsoluteSystemPathBuf::try_from(repo_root.path())?; - - let lockfile_path = repo_root.path().join(LOCKFILE); - File::create(lockfile_path)?; - let package_manager = PackageManager::detect_package_manager(&repo_root_path)?; - assert_eq!(package_manager, PackageManager::Bun); - - Ok(()) + fn test_get_lockfile_path() { + let a = AbsoluteSystemPathBuf::new("/Users/nathanhammond/repos/triage/test-bun").unwrap(); + let out = get_lockfile_path(&a); } } diff --git a/crates/turborepo-lib/src/package_manager/mod.rs b/crates/turborepo-lib/src/package_manager/mod.rs index 87ef8bf9cbc33..87b42bca42e61 100644 --- a/crates/turborepo-lib/src/package_manager/mod.rs +++ b/crates/turborepo-lib/src/package_manager/mod.rs @@ -7,6 +7,7 @@ use std::{ backtrace, fmt::{self, Display}, fs, + process::Command, }; use globwalk::fix_glob_pattern; @@ -19,6 +20,7 @@ use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, RelativeUnixPath}; use turborepo_lockfiles::Lockfile; use turborepo_ui::{UI, UNDERLINE}; use wax::{Any, Glob, Pattern}; +use which::which; use crate::{ package_json::PackageJson, @@ -452,6 +454,22 @@ impl PackageManager { } } + pub fn lockfile_path( + &self, + root_path: &AbsoluteSystemPath, + ) -> Result { + match self { + PackageManager::Bun => bun::get_lockfile_path(root_path), + PackageManager::Npm => Ok(root_path.join_component(npm::LOCKFILE)), + PackageManager::Pnpm | PackageManager::Pnpm6 => { + Ok(root_path.join_component(pnpm::LOCKFILE)) + } + PackageManager::Yarn | PackageManager::Berry => { + Ok(root_path.join_component(yarn::LOCKFILE)) + } + } + } + pub fn workspace_configuration_path(&self) -> Option<&'static str> { match self { PackageManager::Pnpm | PackageManager::Pnpm6 => Some("pnpm-workspace.yaml"), @@ -467,7 +485,17 @@ impl PackageManager { root_path: &AbsoluteSystemPath, root_package_json: &PackageJson, ) -> Result, Error> { - let contents = root_path.join_component(self.lockfile_name()).read()?; + let lockfile_path = self.lockfile_path(root_path)?; + let contents = match self { + PackageManager::Bun => { + Command::new(which("bun")?) + .arg(lockfile_path.to_string()) + .current_dir(root_path.to_string()) + .output()? + .stdout + } + _ => lockfile_path.read()?, + }; self.parse_lockfile(root_package_json, &contents) } @@ -515,10 +543,6 @@ impl PackageManager { } } } - - pub fn lockfile_path(&self, turbo_root: &AbsoluteSystemPath) -> AbsoluteSystemPathBuf { - turbo_root.join_component(self.lockfile_name()) - } } #[cfg(test)] diff --git a/crates/turborepo-lib/src/run/global_hash.rs b/crates/turborepo-lib/src/run/global_hash.rs index d2b119b5ebb96..7ba299659111e 100644 --- a/crates/turborepo-lib/src/run/global_hash.rs +++ b/crates/turborepo-lib/src/run/global_hash.rs @@ -78,9 +78,10 @@ pub fn get_global_hash_inputs<'a, L: ?Sized + Lockfile>( if lockfile.is_none() { global_deps.insert(root_path.join_component("package.json")); - let lockfile_path = package_manager.lockfile_path(root_path); - if lockfile_path.exists() { - global_deps.insert(lockfile_path); + if let Ok(lockfile_path) = package_manager.lockfile_path(root_path) { + if lockfile_path.exists() { + global_deps.insert(lockfile_path); + } } } diff --git a/crates/turborepo-lib/src/run/scope/change_detector.rs b/crates/turborepo-lib/src/run/scope/change_detector.rs index 9f8363f224b9a..5e50b83956cec 100644 --- a/crates/turborepo-lib/src/run/scope/change_detector.rs +++ b/crates/turborepo-lib/src/run/scope/change_detector.rs @@ -158,7 +158,7 @@ impl<'a> SCMChangeDetector<'a> { let lockfile_path = self .pkg_graph .package_manager() - .lockfile_path(self.turbo_root); + .lockfile_path(self.turbo_root)?; let matcher = wax::Glob::new(lockfile_path.as_str())?; @@ -184,6 +184,8 @@ impl<'a> SCMChangeDetector<'a> { pub enum ChangeDetectError { #[error("SCM error: {0}")] Scm(#[from] turborepo_scm::Error), + #[error("IO error: {0}")] + Io(#[from] std::io::Error), #[error("Wax error: {0}")] Wax(#[from] wax::BuildError), #[error("Package manager error: {0}")] From 1cb8710bc24e835b7fddd7108dc422c27dc99924 Mon Sep 17 00:00:00 2001 From: Nathan Hammond Date: Thu, 14 Sep 2023 15:35:11 +0800 Subject: [PATCH 07/11] Revert "SUpport bunfig for lockfile." This reverts commit 26ec64cb3db6d4e5044975eae42e6b2b7284a133. --- Cargo.lock | 53 ++----- crates/turborepo-lib/Cargo.toml | 1 - .../turborepo-lib/src/package_manager/bun.rs | 148 +++--------------- .../turborepo-lib/src/package_manager/mod.rs | 34 +--- crates/turborepo-lib/src/run/global_hash.rs | 7 +- .../src/run/scope/change_detector.rs | 4 +- 6 files changed, 40 insertions(+), 207 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index adf31ace30d84..3fbe5fd7ed630 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5369,7 +5369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" dependencies = [ "once_cell", - "toml_edit 0.19.8", + "toml_edit", ] [[package]] @@ -6335,9 +6335,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.3" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" dependencies = [ "serde", ] @@ -8573,26 +8573,14 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.19.8", -] - -[[package]] -name = "toml" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c226a7bba6d859b63c92c4b4fe69c5b6b72d0cb897dbc8e6012298e6154cb56e" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit 0.20.0", + "toml_edit", ] [[package]] name = "toml_datetime" -version = "0.6.3" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" dependencies = [ "serde", ] @@ -8607,20 +8595,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "winnow 0.4.1", -] - -[[package]] -name = "toml_edit" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ff63e60a958cefbb518ae1fd6566af80d9d4be430a33f3723dfc47d1d411d95" -dependencies = [ - "indexmap 2.0.0", - "serde", - "serde_spanned", - "toml_datetime", - "winnow 0.5.15", + "winnow", ] [[package]] @@ -9978,7 +9953,6 @@ dependencies = [ "tokio", "tokio-stream", "tokio-util", - "toml 0.8.0", "tonic", "tonic-build", "tonic-reflection", @@ -10095,8 +10069,8 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if 1.0.0", - "rand 0.8.5", + "cfg-if 0.1.10", + "rand 0.4.6", "static_assertions", ] @@ -11314,15 +11288,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "winnow" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" -dependencies = [ - "memchr", -] - [[package]] name = "winreg" version = "0.10.1" diff --git a/crates/turborepo-lib/Cargo.toml b/crates/turborepo-lib/Cargo.toml index 512710d57714b..003dbac1226eb 100644 --- a/crates/turborepo-lib/Cargo.toml +++ b/crates/turborepo-lib/Cargo.toml @@ -98,7 +98,6 @@ num_cpus = "1.15.0" owo-colors.workspace = true rayon = "1.7.0" regex.workspace = true -toml = "0.8.0" tracing-appender = "0.2.2" tracing-chrome = { version = "0.7.1", optional = true } tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } diff --git a/crates/turborepo-lib/src/package_manager/bun.rs b/crates/turborepo-lib/src/package_manager/bun.rs index 442be6ad38da1..4ed64cb207d46 100644 --- a/crates/turborepo-lib/src/package_manager/bun.rs +++ b/crates/turborepo-lib/src/package_manager/bun.rs @@ -1,125 +1,8 @@ -use std::{fs, path::PathBuf}; - -use serde::{Deserialize, Serialize}; -use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf}; +use turbopath::AbsoluteSystemPath; use crate::package_manager::{Error, PackageManager}; -const GLOBAL_BUNFIG: &'static str = ".bunfig.toml"; -const LOCAL_BUNFIG: &'static str = "bunfig.toml"; -pub const LOCKFILE: &'static str = "bun.lockb"; - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct Bunfig { - pub install: Option, -} - -impl Default for Bunfig { - fn default() -> Self { - Self { - install: Some(Default::default()), - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct Install { - pub lockfile: Option, -} - -impl Default for Install { - fn default() -> Self { - Self { - lockfile: Some(Default::default()), - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct Lockfile { - pub path: Option, -} - -impl Default for Lockfile { - fn default() -> Self { - Self { - path: Some(String::from("bun.lockb")), - } - } -} - -fn global_bunfig() -> Result, std::io::Error> { - // Ordering: https://github.com/oven-sh/bun/blob/75b5c715405d49b5026c13143efecd580d27be1b/src/cli.zig#L303 - if let Some(data_dir) = std::env::var_os("XDG_CONFIG_HOME").or_else(|| std::env::var_os("HOME")) - { - process_bunfig_read(fs::read_to_string( - PathBuf::from(data_dir).join(GLOBAL_BUNFIG), - )) - } else { - Ok(None) - } -} - -fn local_bunfig(repo_root: &AbsoluteSystemPath) -> Result, std::io::Error> { - process_bunfig_read(repo_root.join_component(LOCAL_BUNFIG).read_to_string()) -} - -fn process_bunfig_read( - read_result: Result, -) -> Result, std::io::Error> { - match read_result { - Ok(bunfig_contents) => { - let deserialize_result: Bunfig = toml::from_str(&bunfig_contents).unwrap(); - Ok(Some(deserialize_result)) - } - Err(error) => { - if matches!(error.kind(), std::io::ErrorKind::NotFound) { - Ok(None) - } else { - Err(error) - } - } - } -} - -pub fn get_lockfile_path( - repo_root: &AbsoluteSystemPath, -) -> Result { - let global = global_bunfig()?; - let local = local_bunfig(repo_root)?; - - let path = match (global, local) { - (None, None) => None, - (None, Some(local_bunfig)) => local_bunfig - .install - .and_then(|install| install.lockfile) - .and_then(|lockfile| lockfile.path) - .and_then(|path| if path.is_empty() { None } else { Some(path) }), - (Some(global_bunfig), None) => global_bunfig - .install - .and_then(|install| install.lockfile) - .and_then(|lockfile| lockfile.path) - .and_then(|path| if path.is_empty() { None } else { Some(path) }), - (Some(global_bunfig), Some(local_bunfig)) => { - // Bun shallow merges. - local_bunfig - .install - .or(global_bunfig.install) - .and_then(|install| install.lockfile) - .and_then(|lockfile| lockfile.path) - .and_then(|path| if path.is_empty() { None } else { Some(path) }) - } - }; - - match path { - Some(path) => { - // This is possibly garbage-in. - // But if it is garbage-in, we can be garbage out. - Ok(AbsoluteSystemPathBuf::from_unknown(repo_root, path)) - } - None => Ok(repo_root.join_component(LOCKFILE)), - } -} +pub const LOCKFILE: &str = "bun.lockb"; pub struct BunDetector<'a> { repo_root: &'a AbsoluteSystemPath, @@ -144,10 +27,9 @@ impl<'a> Iterator for BunDetector<'a> { } self.found = true; - let bunfig_lockfile_path = get_lockfile_path(self.repo_root) - .unwrap_or_else(|_| self.repo_root.join_component(LOCKFILE)); + let package_json = self.repo_root.join_component(LOCKFILE); - if bunfig_lockfile_path.exists() { + if package_json.exists() { Some(Ok(PackageManager::Bun)) } else { None @@ -157,11 +39,25 @@ impl<'a> Iterator for BunDetector<'a> { #[cfg(test)] mod tests { - use super::*; + use std::fs::File; + + use anyhow::Result; + use tempfile::tempdir; + use turbopath::AbsoluteSystemPathBuf; + + use super::LOCKFILE; + use crate::package_manager::PackageManager; #[test] - fn test_get_lockfile_path() { - let a = AbsoluteSystemPathBuf::new("/Users/nathanhammond/repos/triage/test-bun").unwrap(); - let out = get_lockfile_path(&a); + fn test_detect_bun() -> Result<()> { + let repo_root = tempdir()?; + let repo_root_path = AbsoluteSystemPathBuf::try_from(repo_root.path())?; + + let lockfile_path = repo_root.path().join(LOCKFILE); + File::create(lockfile_path)?; + let package_manager = PackageManager::detect_package_manager(&repo_root_path)?; + assert_eq!(package_manager, PackageManager::Bun); + + Ok(()) } } diff --git a/crates/turborepo-lib/src/package_manager/mod.rs b/crates/turborepo-lib/src/package_manager/mod.rs index 87b42bca42e61..87ef8bf9cbc33 100644 --- a/crates/turborepo-lib/src/package_manager/mod.rs +++ b/crates/turborepo-lib/src/package_manager/mod.rs @@ -7,7 +7,6 @@ use std::{ backtrace, fmt::{self, Display}, fs, - process::Command, }; use globwalk::fix_glob_pattern; @@ -20,7 +19,6 @@ use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, RelativeUnixPath}; use turborepo_lockfiles::Lockfile; use turborepo_ui::{UI, UNDERLINE}; use wax::{Any, Glob, Pattern}; -use which::which; use crate::{ package_json::PackageJson, @@ -454,22 +452,6 @@ impl PackageManager { } } - pub fn lockfile_path( - &self, - root_path: &AbsoluteSystemPath, - ) -> Result { - match self { - PackageManager::Bun => bun::get_lockfile_path(root_path), - PackageManager::Npm => Ok(root_path.join_component(npm::LOCKFILE)), - PackageManager::Pnpm | PackageManager::Pnpm6 => { - Ok(root_path.join_component(pnpm::LOCKFILE)) - } - PackageManager::Yarn | PackageManager::Berry => { - Ok(root_path.join_component(yarn::LOCKFILE)) - } - } - } - pub fn workspace_configuration_path(&self) -> Option<&'static str> { match self { PackageManager::Pnpm | PackageManager::Pnpm6 => Some("pnpm-workspace.yaml"), @@ -485,17 +467,7 @@ impl PackageManager { root_path: &AbsoluteSystemPath, root_package_json: &PackageJson, ) -> Result, Error> { - let lockfile_path = self.lockfile_path(root_path)?; - let contents = match self { - PackageManager::Bun => { - Command::new(which("bun")?) - .arg(lockfile_path.to_string()) - .current_dir(root_path.to_string()) - .output()? - .stdout - } - _ => lockfile_path.read()?, - }; + let contents = root_path.join_component(self.lockfile_name()).read()?; self.parse_lockfile(root_package_json, &contents) } @@ -543,6 +515,10 @@ impl PackageManager { } } } + + pub fn lockfile_path(&self, turbo_root: &AbsoluteSystemPath) -> AbsoluteSystemPathBuf { + turbo_root.join_component(self.lockfile_name()) + } } #[cfg(test)] diff --git a/crates/turborepo-lib/src/run/global_hash.rs b/crates/turborepo-lib/src/run/global_hash.rs index 7ba299659111e..d2b119b5ebb96 100644 --- a/crates/turborepo-lib/src/run/global_hash.rs +++ b/crates/turborepo-lib/src/run/global_hash.rs @@ -78,10 +78,9 @@ pub fn get_global_hash_inputs<'a, L: ?Sized + Lockfile>( if lockfile.is_none() { global_deps.insert(root_path.join_component("package.json")); - if let Ok(lockfile_path) = package_manager.lockfile_path(root_path) { - if lockfile_path.exists() { - global_deps.insert(lockfile_path); - } + let lockfile_path = package_manager.lockfile_path(root_path); + if lockfile_path.exists() { + global_deps.insert(lockfile_path); } } diff --git a/crates/turborepo-lib/src/run/scope/change_detector.rs b/crates/turborepo-lib/src/run/scope/change_detector.rs index 5e50b83956cec..9f8363f224b9a 100644 --- a/crates/turborepo-lib/src/run/scope/change_detector.rs +++ b/crates/turborepo-lib/src/run/scope/change_detector.rs @@ -158,7 +158,7 @@ impl<'a> SCMChangeDetector<'a> { let lockfile_path = self .pkg_graph .package_manager() - .lockfile_path(self.turbo_root)?; + .lockfile_path(self.turbo_root); let matcher = wax::Glob::new(lockfile_path.as_str())?; @@ -184,8 +184,6 @@ impl<'a> SCMChangeDetector<'a> { pub enum ChangeDetectError { #[error("SCM error: {0}")] Scm(#[from] turborepo_scm::Error), - #[error("IO error: {0}")] - Io(#[from] std::io::Error), #[error("Wax error: {0}")] Wax(#[from] wax::BuildError), #[error("Package manager error: {0}")] From c39f452abb4ef23387db8fb9c5d27774eeb76016 Mon Sep 17 00:00:00 2001 From: Nathan Hammond Date: Thu, 14 Sep 2023 15:46:51 +0800 Subject: [PATCH 08/11] Seamless reading of bun lockfile. --- crates/turborepo-lib/src/package_manager/mod.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/turborepo-lib/src/package_manager/mod.rs b/crates/turborepo-lib/src/package_manager/mod.rs index 87ef8bf9cbc33..d816b1893de99 100644 --- a/crates/turborepo-lib/src/package_manager/mod.rs +++ b/crates/turborepo-lib/src/package_manager/mod.rs @@ -7,6 +7,7 @@ use std::{ backtrace, fmt::{self, Display}, fs, + process::Command, }; use globwalk::fix_glob_pattern; @@ -19,6 +20,7 @@ use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, RelativeUnixPath}; use turborepo_lockfiles::Lockfile; use turborepo_ui::{UI, UNDERLINE}; use wax::{Any, Glob, Pattern}; +use which::which; use crate::{ package_json::PackageJson, @@ -467,7 +469,17 @@ impl PackageManager { root_path: &AbsoluteSystemPath, root_package_json: &PackageJson, ) -> Result, Error> { - let contents = root_path.join_component(self.lockfile_name()).read()?; + let lockfile_path = self.lockfile_path(root_path); + let contents = match self { + PackageManager::Bun => { + Command::new(which("bun")?) + .arg(lockfile_path.to_string()) + .current_dir(root_path.to_string()) + .output()? + .stdout + } + _ => lockfile_path.read()?, + }; self.parse_lockfile(root_package_json, &contents) } From 34cd7f95d7eef48e8f45e05d4cb3218076aa51b6 Mon Sep 17 00:00:00 2001 From: Nathan Hammond Date: Thu, 14 Sep 2023 15:59:29 +0800 Subject: [PATCH 09/11] Correct object name. --- crates/turborepo-lib/src/package_manager/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/turborepo-lib/src/package_manager/mod.rs b/crates/turborepo-lib/src/package_manager/mod.rs index d816b1893de99..ecdaea848af15 100644 --- a/crates/turborepo-lib/src/package_manager/mod.rs +++ b/crates/turborepo-lib/src/package_manager/mod.rs @@ -497,7 +497,7 @@ impl PackageManager { Box::new(turborepo_lockfiles::Yarn1Lockfile::from_bytes(contents)?) } PackageManager::Bun => { - Box::new(turborepo_lockfiles::Yarn1Lockfile::from_bytes(contents)?) + Box::new(turborepo_lockfiles::BunLockfile::from_bytes(contents)?) } PackageManager::Berry => Box::new(turborepo_lockfiles::BerryLockfile::load( contents, From 0bed28a00669ecae0f03145b696cf983cc062be3 Mon Sep 17 00:00:00 2001 From: Nathan Hammond Date: Thu, 14 Sep 2023 16:02:26 +0800 Subject: [PATCH 10/11] Make Go lint happy. --- cli/internal/lockfile/bun_lockfile.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/internal/lockfile/bun_lockfile.go b/cli/internal/lockfile/bun_lockfile.go index 824737d3a22aa..058c4b3dd493e 100644 --- a/cli/internal/lockfile/bun_lockfile.go +++ b/cli/internal/lockfile/bun_lockfile.go @@ -12,14 +12,14 @@ type BunLockfile struct { var _ Lockfile = (*BunLockfile)(nil) // ResolvePackage Given a package and version returns the key, resolved version, and if it was found -func (l *BunLockfile) ResolvePackage(_workspacePath turbopath.AnchoredUnixPath, name string, version string) (Package, error) { +func (l *BunLockfile) ResolvePackage(_ turbopath.AnchoredUnixPath, _ string, _ string) (Package, error) { // This is only used when doing calculating the transitive deps, but Rust // implementations do this calculation on the Rust side. panic("Unreachable") } // AllDependencies Given a lockfile key return all (dev/optional/peer) dependencies of that package -func (l *BunLockfile) AllDependencies(key string) (map[string]string, bool) { +func (l *BunLockfile) AllDependencies(_ string) (map[string]string, bool) { // This is only used when doing calculating the transitive deps, but Rust // implementations do this calculation on the Rust side. panic("Unreachable") From ed3f4ff9eb72a1a17b7b18c08c676f604e20de9d Mon Sep 17 00:00:00 2001 From: Nathan Hammond Date: Thu, 14 Sep 2023 17:24:42 +0800 Subject: [PATCH 11/11] Make sure that tests get duplicated too. --- crates/turborepo-lib/src/package_manager/mod.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/turborepo-lib/src/package_manager/mod.rs b/crates/turborepo-lib/src/package_manager/mod.rs index ecdaea848af15..0cc9d7a43dbc5 100644 --- a/crates/turborepo-lib/src/package_manager/mod.rs +++ b/crates/turborepo-lib/src/package_manager/mod.rs @@ -577,6 +577,7 @@ mod tests { PackageManager::Berry, PackageManager::Yarn, PackageManager::Npm, + PackageManager::Bun, ] { let found = mgr.get_package_jsons(&with_yarn).unwrap(); let found: HashSet = HashSet::from_iter(found); @@ -700,6 +701,13 @@ mod tests { expected_version: "111.0.1".to_owned(), expected_error: false, }, + TestCase { + name: "supports bun".to_owned(), + package_manager: "bun@1.0.1".to_owned(), + expected_manager: "bun".to_owned(), + expected_version: "1.0.1".to_owned(), + expected_error: false, + }, ]; for case in tests { @@ -739,6 +747,10 @@ mod tests { let package_manager = PackageManager::read_package_manager(&package_json)?; assert_eq!(package_manager, Some(PackageManager::Pnpm)); + package_json.package_manager = Some("bun@1.0.1".to_string()); + let package_manager = PackageManager::read_package_manager(&package_json)?; + assert_eq!(package_manager, Some(PackageManager::Bun)); + Ok(()) }