Revert "fix(turbopack): Use vergen-git2 instead of shadow-rs for napi and next-api crates to fix stale git lock files" (#76879)

Reverts #76773

> Looks like we broke compilation for our linux swc builds for musl/gnu

Author: bgw

Cargo.lock

@@ -426,8 +426,6 @@ unicode-segmentation = "1.10.1"
 unsize = "1.1.0"
 url = "2.2.2"
 urlencoding = "2.1.2"
-vergen = { version = "9.0.4", features = ["cargo"] }
-vergen-git2 = { version = "1.0.5", features = ["cargo"] }
 webbrowser = "0.8.7"
 
 [patch.crates-io]

Cargo.toml

@@ -66,13 +66,14 @@ rand = { workspace = true }
 rustc-hash = { workspace = true }
 serde = "1"
 serde_json = "1"
+shadow-rs = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
 tracing-chrome = "0.5.0"
 url = { workspace = true }
 urlencoding = { workspace = true }
 once_cell = { workspace = true }
-dashmap = { workspace = true }
+dashmap = "6.1.0"
 
 swc_core = { workspace = true, features = [
     "base_concurrent",
@@ -138,11 +139,11 @@ turbo-tasks-malloc = { workspace = true, default-features = false }
 tokio = { workspace = true, features = ["full"] }
 
 [build-dependencies]
-anyhow = { workspace = true }
 napi-build = "2"
 serde = { workspace = true }
 serde_json = { workspace = true }
-vergen-git2 = { workspace = true }
+# It is not a mistake this dependency is specified in dep / build-dep both.
+shadow-rs = { workspace = true }
 
 # build-dependencies for the native, non-wasm32 build
 [target.'cfg(not(target_arch = "wasm32"))'.build-dependencies]

crates/napi/Cargo.toml

@@ -1,52 +1,18 @@
-use std::{env, process::Command, str};
+use std::fs;
 
 extern crate napi_build;
 
-fn main() -> anyhow::Result<()> {
-    println!("cargo:rerun-if-env-changed=CI");
-    let is_ci = env::var("CI").is_ok_and(|value| !value.is_empty());
-
+fn main() {
     // Generates, stores build-time information as static values.
     // There are some places relying on correct values for this (i.e telemetry),
     // So failing build if this fails.
-    let cargo = vergen_git2::CargoBuilder::default()
-        .target_triple(true)
-        .build()?;
-    // We use the git dirty state to disable persistent caching (persistent caching relies on a
-    // commit hash to be safe). One tradeoff of this is that we must invalidate the rust build more
-    // often.
-    //
-    // This invalidates the build if any untracked files change. That's sufficient for the case
-    // where we transition from dirty to clean.
-    //
-    // There's an edge-case here where the repository could be newly dirty, but we can't know
-    // because our build hasn't been invalidated, since the untracked files weren't untracked last
-    // time we ran. That will cause us to incorrectly report ourselves as clean.
-    //
-    // However, in practice that shouldn't be much of an issue: If no other dependency of this
-    // top-level crate has changed (which would've triggered our rebuild), then the resulting binary
-    // must be equivalent to a clean build anyways. Therefore, persistent caching using the HEAD
-    // commit hash as a version is okay.
-    let git = vergen_git2::Git2Builder::default()
-        .dirty(/* include_untracked */ true)
-        .describe(
-            /* tags */ true,
-            /* dirty */ !is_ci, // suppress the dirty suffix in CI
-            /* matches */ Some("v[0-9]*"), // find the last version tag
-        )
-        .build()?;
-    vergen_git2::Emitter::default()
-        .add_instructions(&cargo)?
-        .add_instructions(&git)?
-        .fail_on_error()
-        .emit()?;
+    shadow_rs::ShadowBuilder::builder()
+        .build()
+        .expect("Should able to generate build time information");
 
-    match Command::new("git").args(["rev-parse", "HEAD"]).output() {
-        Ok(out) if out.status.success() => println!(
-            "cargo:warning=git HEAD: {}",
-            str::from_utf8(&out.stdout).unwrap()
-        ),
-        _ => println!("cargo:warning=`git rev-parse HEAD` failed"),
+    let git_head = fs::read_to_string("../../.git/HEAD").unwrap_or_default();
+    if !git_head.is_empty() && !git_head.starts_with("ref: ") {
+        println!("cargo:warning=git version {}", git_head);
     }
 
     #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))]
@@ -70,6 +36,4 @@ fn main() -> anyhow::Result<()> {
 
     #[cfg(not(target_arch = "wasm32"))]
     turbo_tasks_build::generate_register();
-
-    Ok(())
 }

crates/napi/build.rs

@@ -61,6 +61,9 @@ pub mod turbo_trace_server;
 pub mod turbopack;
 pub mod util;
 
+// Declare build-time information variables generated in build.rs
+shadow_rs::shadow!(build);
+
 #[cfg(not(any(feature = "__internal_dhat-heap", feature = "__internal_dhat-ad-hoc")))]
 #[global_allocator]
 static ALLOC: turbo_tasks_malloc::TurboMalloc = turbo_tasks_malloc::TurboMalloc;

crates/napi/src/lib.rs

@@ -13,8 +13,7 @@ use turbo_tasks::{
     TryJoinIterExt, TurboTasks, TurboTasksApi, UpdateInfo, Vc,
 };
 use turbo_tasks_backend::{
-    default_backing_storage, noop_backing_storage, DefaultBackingStorage, GitVersionInfo,
-    NoopBackingStorage,
+    default_backing_storage, noop_backing_storage, DefaultBackingStorage, NoopBackingStorage,
 };
 use turbo_tasks_fs::FileContent;
 use turbopack_core::{
@@ -131,10 +130,26 @@ pub fn create_turbo_tasks(
     dependency_tracking: bool,
 ) -> Result<NextTurboTasks> {
     Ok(if persistent_caching {
-        let version_info = GitVersionInfo {
-            describe: env!("VERGEN_GIT_DESCRIBE"),
-            dirty: option_env!("CI").is_none_or(|value| value.is_empty())
-                && env!("VERGEN_GIT_DIRTY") == "true",
+        let dirty_suffix = if crate::build::GIT_CLEAN
+            || option_env!("CI").is_some_and(|value| !value.is_empty())
+        {
+            ""
+        } else {
+            "-dirty"
+        };
+        #[allow(
+            clippy::const_is_empty,
+            reason = "LAST_TAG might be empty if the tag can't be determined"
+        )]
+        let version_info = if crate::build::LAST_TAG.is_empty() {
+            format!("{}{}", crate::build::SHORT_COMMIT, dirty_suffix)
+        } else {
+            format!(
+                "{}-{}{}",
+                crate::build::LAST_TAG,
+                crate::build::SHORT_COMMIT,
+                dirty_suffix
+            )
         };
         NextTurboTasks::PersistentCaching(TurboTasks::new(
             turbo_tasks_backend::TurboTasksBackend::new(

crates/napi/src/next_api/utils.rs

@@ -127,8 +127,8 @@ pub fn log_internal_error_and_inform(err_info: &str) {
 }
 
 #[napi]
-pub fn get_target_triple() -> &'static str {
-    env!("VERGEN_CARGO_TARGET_TRIPLE")
+pub fn get_target_triple() -> String {
+    crate::build::BUILD_TARGET.to_string()
 }
 
 pub trait MapErr<T>: Into<Result<T, anyhow::Error>> {

crates/napi/src/util.rs

@@ -24,6 +24,7 @@ regex = { workspace = true }
 rustc-hash = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
+shadow-rs = { workspace = true }
 swc_core = { workspace = true }
 tracing = { workspace = true }
 turbo-rcstr = { workspace = true }
@@ -42,6 +43,6 @@ turbopack-node = { workspace = true }
 turbopack-nodejs = { workspace = true }
 
 [build-dependencies]
-anyhow = { workspace = true }
+# It is not a mistake this dependency is specified in dep / build-dep both.
+shadow-rs = { workspace = true }
 turbo-tasks-build = { workspace = true }
-vergen = { workspace = true }

crates/next-api/Cargo.toml

@@ -1,18 +1,13 @@
 use turbo_tasks_build::generate_register;
 
-fn main() -> anyhow::Result<()> {
+fn main() {
     // Generates, stores build-time information as static values.
     // There are some places relying on correct values for this (i.e telemetry),
     // So failing build if this fails.
-    let cargo = vergen::CargoBuilder::default()
-        .target_triple(true)
-        .build()?;
-    vergen::Emitter::default()
-        .add_instructions(&cargo)?
-        .fail_on_error()
-        .emit()?;
+    shadow_rs::ShadowBuilder::builder()
+        .build_pattern(shadow_rs::BuildPattern::Lazy)
+        .build()
+        .expect("Should able to generate build time information");
 
     generate_register();
-
-    Ok(())
 }

crates/next-api/build.rs

@@ -23,6 +23,9 @@ mod server_actions;
 mod versioned_content_map;
 mod webpack_stats;
 
+// Declare build-time information variables generated in build.rs
+shadow_rs::shadow!(build);
+
 pub fn register() {
     next_core::register();
     turbopack_nodejs::register();

crates/next-api/src/lib.rs

@@ -68,6 +68,7 @@ use turbopack_nodejs::NodeJsChunkingContext;
 
 use crate::{
     app::{AppProject, OptionAppProject, ECMASCRIPT_CLIENT_TRANSITION_NAME},
+    build,
     empty::EmptyEndpoint,
     entrypoints::Entrypoints,
     instrumentation::InstrumentationEndpoint,
@@ -1068,7 +1069,7 @@ impl Project {
         // First, emit an event for the binary target triple.
         // This is different to webpack-config; when this is being called,
         // it is always using SWC so we don't check swc here.
-        emit_event(env!("VERGEN_CARGO_TARGET_TRIPLE"), true);
+        emit_event(build::BUILD_TARGET, true);
 
         // Go over jsconfig and report enabled features.
         let compiler_options = self.js_config().compiler_options().await?;

crates/next-api/src/project.rs

@@ -16,5 +16,25 @@ workspace = true
 next-core = { workspace = true }
 turbopack-core = { workspace = true }
 
+#turbopack-binding = { workspace = true, features = [
+#  "__turbo_tasks",
+#  "__turbo_tasks_memory",
+#  "__turbo_tasks_env",
+#  "__turbo_tasks_fs",
+#  "__turbo_tasks_memory",
+#  "__turbopack",
+#  "__turbopack_nodejs",
+#  "__turbopack_core",
+#  "__turbopack_browser",
+#  "__turbopack_ecmascript",
+#  "__turbopack_ecmascript_runtime",
+#  "__turbopack_env",
+#  "__turbopack_node",
+#] }
+
 [build-dependencies]
 turbo-tasks-build = { workspace = true }
+vergen = { version = "7.3.2", default-features = false, features = [
+  "cargo",
+  "build",
+] }

crates/next-build/Cargo.toml

@@ -1,5 +1,10 @@
 use turbo_tasks_build::generate_register;
+use vergen::{vergen, Config};
 
 fn main() {
     generate_register();
+
+    // Attempt to collect some build time env values but will skip if there are any
+    // errors.
+    let _ = vergen(Config::default());
 }

crates/next-build/build.rs

@@ -7,31 +7,24 @@ use std::{
 
 use anyhow::Result;
 
-/// Information gathered by `vergen_git2` in the top-level binary crate and passed down. This
-/// information must be computed in the top-level crate for cargo incremental compilation to work
-/// correctly.
-///
-/// See `crates/napi/build.rs` for details.
-pub struct GitVersionInfo<'a> {
-    /// Output of `git describe --match 'v[0-9]' --dirty`.
-    pub describe: &'a str,
-    /// Is the git repository dirty? Always forced to `false` when the `CI` environment variable is
-    /// set and non-empty.
-    pub dirty: bool,
-}
-
 /// Specifies many databases that have a different version than the current one are retained.
 /// For example if MAX_OTHER_DB_VERSIONS is 2, there can be at most 3 databases in the directory,
 /// the current one and two older/newer ones.
 const MAX_OTHER_DB_VERSIONS: usize = 2;
 
-pub fn handle_db_versioning(base_path: &Path, version_info: &GitVersionInfo) -> Result<PathBuf> {
+pub fn handle_db_versioning(base_path: &Path, version_info: &str) -> Result<PathBuf> {
     if let Ok(version) = env::var("TURBO_ENGINE_VERSION") {
         return Ok(base_path.join(version));
     }
     // Database versioning. Pass `TURBO_ENGINE_IGNORE_DIRTY` at runtime to ignore a
     // dirty git repository. Pass `TURBO_ENGINE_DISABLE_VERSIONING` at runtime to disable
     // versioning and always use the same database.
+    let (version_info, git_dirty) = if let Some(version_info) = version_info.strip_suffix("-dirty")
+    {
+        (version_info, true)
+    } else {
+        (version_info, false)
+    };
     let ignore_dirty = env::var("TURBO_ENGINE_IGNORE_DIRTY").ok().is_some();
     let disabled_versioning = env::var("TURBO_ENGINE_DISABLE_VERSIONING").ok().is_some();
     let version = if disabled_versioning {
@@ -40,14 +33,14 @@ pub fn handle_db_versioning(base_path: &Path, version_info: &GitVersionInfo) ->
              caching database might be required."
         );
         Some("unversioned")
-    } else if !version_info.dirty {
-        Some(version_info.describe)
+    } else if !git_dirty {
+        Some(version_info)
     } else if ignore_dirty {
         println!(
             "WARNING: The git repository is dirty, but Persistent Caching is still enabled. \
              Manual removal of the persistent caching database might be required."
         );
-        Some(version_info.describe)
+        Some(version_info)
     } else {
         println!(
             "WARNING: The git repository is dirty: Persistent Caching is disabled. Use \

turbopack/crates/turbo-tasks-backend/src/database/db_versioning.rs

@@ -15,14 +15,13 @@ use std::path::Path;
 
 use anyhow::Result;
 
-use crate::database::{
-    db_versioning::handle_db_versioning, noop_kv::NoopKvDb, turbo::TurboKeyValueDatabase,
-};
-pub use crate::{
+pub use self::{
     backend::{BackendOptions, StorageMode, TurboTasksBackend},
-    database::db_versioning::GitVersionInfo,
     kv_backing_storage::KeyValueDatabaseBackingStorage,
 };
+use crate::database::{
+    db_versioning::handle_db_versioning, noop_kv::NoopKvDb, turbo::TurboKeyValueDatabase,
+};
 
 #[cfg(feature = "lmdb")]
 pub type LmdbBackingStorage = KeyValueDatabaseBackingStorage<
@@ -36,10 +35,7 @@ pub type LmdbBackingStorage = KeyValueDatabaseBackingStorage<
 >;
 
 #[cfg(feature = "lmdb")]
-pub fn lmdb_backing_storage(
-    path: &Path,
-    version_info: &GitVersionInfo,
-) -> Result<LmdbBackingStorage> {
+pub fn lmdb_backing_storage(path: &Path, version_info: &str) -> Result<LmdbBackingStorage> {
     use crate::database::{
         fresh_db_optimization::{is_fresh, FreshDbOptimization},
         read_transaction_cache::ReadTransactionCache,
@@ -57,10 +53,7 @@ pub fn lmdb_backing_storage(
 
 pub type TurboBackingStorage = KeyValueDatabaseBackingStorage<TurboKeyValueDatabase>;
 
-pub fn turbo_backing_storage(
-    path: &Path,
-    version_info: &GitVersionInfo,
-) -> Result<TurboBackingStorage> {
+pub fn turbo_backing_storage(path: &Path, version_info: &str) -> Result<TurboBackingStorage> {
     let path = handle_db_versioning(path, version_info)?;
     let database = TurboKeyValueDatabase::new(path)?;
     Ok(KeyValueDatabaseBackingStorage::new(database))
@@ -76,20 +69,14 @@ pub fn noop_backing_storage() -> NoopBackingStorage {
 pub type DefaultBackingStorage = LmdbBackingStorage;
 
 #[cfg(feature = "lmdb")]
-pub fn default_backing_storage(
-    path: &Path,
-    version_info: &GitVersionInfo,
-) -> Result<DefaultBackingStorage> {
+pub fn default_backing_storage(path: &Path, version_info: &str) -> Result<DefaultBackingStorage> {
     lmdb_backing_storage(path, version_info)
 }
 
 #[cfg(not(feature = "lmdb"))]
 pub type DefaultBackingStorage = TurboBackingStorage;
 
 #[cfg(not(feature = "lmdb"))]
-pub fn default_backing_storage(
-    path: &Path,
-    version_info: &GitVersionInfo,
-) -> Result<DefaultBackingStorage> {
+pub fn default_backing_storage(path: &Path, version_info: &str) -> Result<DefaultBackingStorage> {
     turbo_backing_storage(path, version_info)
 }