[Turbopack] move git version info to top-level crate to fix caching i…

…ssues (#74015)

### What?

crate level caching caused a Persisting Caching bug when upgrading the next.js version.

This moves the version info to the top-level crate to avoid this issue.

crates/napi/Cargo.toml

@@ -135,8 +135,8 @@ tokio = { workspace = true, features = ["full"] }
 
 [build-dependencies]
 napi-build = "2"
-serde = "1"
-serde_json = "1"
+serde = { workspace = true }
+serde_json = { workspace = true }
 # It is not a mistake this dependency is specified in dep / build-dep both.
 shadow-rs = { workspace = true }
 

crates/napi/src/next_api/utils.rs

@@ -127,6 +127,25 @@ pub fn create_turbo_tasks(
     memory_limit: usize,
 ) -> Result<NextTurboTasks> {
     Ok(if persistent_caching {
+        let dirty_suffix = if crate::build::GIT_CLEAN || !env!("CI").is_empty() {
+            ""
+        } else {
+            "-dirty"
+        };
+        #[allow(
+            clippy::const_is_empty,
+            reason = "LAST_TAG might be empty if the tag can't be determined"
+        )]
+        let version_info = if crate::build::LAST_TAG.is_empty() {
+            format!("{}{}", crate::build::SHORT_COMMIT, dirty_suffix)
+        } else {
+            format!(
+                "{}-{}{}",
+                crate::build::LAST_TAG,
+                crate::build::SHORT_COMMIT,
+                dirty_suffix
+            )
+        };
         NextTurboTasks::PersistentCaching(TurboTasks::new(
             turbo_tasks_backend::TurboTasksBackend::new(
                 turbo_tasks_backend::BackendOptions {
@@ -137,7 +156,7 @@ pub fn create_turbo_tasks(
                     }),
                     ..Default::default()
                 },
-                default_backing_storage(&output_path.join("cache/turbopack"))?,
+                default_backing_storage(&output_path.join("cache/turbopack"), &version_info)?,
             ),
         ))
     } else {

turbopack/crates/turbo-tasks-backend/Cargo.toml

@@ -55,9 +55,7 @@ turbo-tasks-testing = { workspace = true }
 criterion = { workspace = true, features = ["async_tokio"] }
 
 [build-dependencies]
-anyhow = { workspace = true }
 turbo-tasks-build = { workspace = true }
-vergen-gitcl = { version = "1.0.1" }
 
 [[bench]]
 name = "mod"

turbopack/crates/turbo-tasks-backend/build.rs

@@ -1,16 +1,5 @@
-use anyhow::Result;
 use turbo_tasks_build::generate_register;
-use vergen_gitcl::{Emitter, GitclBuilder};
-
-fn generate_version_info() -> Result<()> {
-    let git = GitclBuilder::default()
-        .describe(false, option_env!("CI").is_none(), None)
-        .build()?;
-    Emitter::default().add_instructions(&git)?.emit()?;
-    Ok(())
-}
 
 fn main() {
     generate_register();
-    generate_version_info().unwrap();
 }

turbopack/crates/turbo-tasks-backend/src/database/db_versioning.rs

@@ -12,14 +12,13 @@ use anyhow::Result;
 /// the current one and two older/newer ones.
 const MAX_OTHER_DB_VERSIONS: usize = 2;
 
-pub fn handle_db_versioning(base_path: &Path) -> Result<PathBuf> {
+pub fn handle_db_versioning(base_path: &Path, version_info: &str) -> Result<PathBuf> {
     if let Ok(version) = env::var("TURBO_ENGINE_VERSION") {
         return Ok(base_path.join(version));
     }
     // Database versioning. Pass `TURBO_ENGINE_IGNORE_DIRTY` at runtime to ignore a
     // dirty git repository. Pass `TURBO_ENGINE_DISABLE_VERSIONING` at runtime to disable
     // versioning and always use the same database.
-    let version_info = env!("VERGEN_GIT_DESCRIBE");
     let (version_info, git_dirty) = if let Some(version_info) = version_info.strip_suffix("-dirty")
     {
         (version_info, true)

turbopack/crates/turbo-tasks-backend/src/lib.rs

@@ -33,14 +33,14 @@ pub type LmdbBackingStorage = KeyValueDatabaseBackingStorage<
 >;
 
 #[cfg(feature = "lmdb")]
-pub fn lmdb_backing_storage(path: &Path) -> Result<LmdbBackingStorage> {
+pub fn lmdb_backing_storage(path: &Path, version_info: &str) -> Result<LmdbBackingStorage> {
     use crate::database::{
         fresh_db_optimization::{is_fresh, FreshDbOptimization},
         read_transaction_cache::ReadTransactionCache,
         startup_cache::StartupCacheLayer,
     };
 
-    let path = handle_db_versioning(path)?;
+    let path = handle_db_versioning(path, version_info)?;
     let fresh_db = is_fresh(&path);
     let database = crate::database::lmdb::LmbdKeyValueDatabase::new(&path)?;
     let database = FreshDbOptimization::new(database, fresh_db);
@@ -51,8 +51,8 @@ pub fn lmdb_backing_storage(path: &Path) -> Result<LmdbBackingStorage> {
 
 pub type TurboBackingStorage = KeyValueDatabaseBackingStorage<TurboKeyValueDatabase>;
 
-pub fn turbo_backing_storage(path: &Path) -> Result<TurboBackingStorage> {
-    let path = handle_db_versioning(path)?;
+pub fn turbo_backing_storage(path: &Path, version_info: &str) -> Result<TurboBackingStorage> {
+    let path = handle_db_versioning(path, version_info)?;
     let database = TurboKeyValueDatabase::new(path)?;
     Ok(KeyValueDatabaseBackingStorage::new(database))
 }
@@ -67,14 +67,14 @@ pub fn noop_backing_storage() -> NoopBackingStorage {
 pub type DefaultBackingStorage = LmdbBackingStorage;
 
 #[cfg(feature = "lmdb")]
-pub fn default_backing_storage(path: &Path) -> Result<DefaultBackingStorage> {
-    lmdb_backing_storage(path)
+pub fn default_backing_storage(path: &Path, version_info: &str) -> Result<DefaultBackingStorage> {
+    lmdb_backing_storage(path, version_info)
 }
 
 #[cfg(not(feature = "lmdb"))]
 pub type DefaultBackingStorage = TurboBackingStorage;
 
 #[cfg(not(feature = "lmdb"))]
-pub fn default_backing_storage(path: &Path) -> Result<DefaultBackingStorage> {
-    turbo_backing_storage(path)
+pub fn default_backing_storage(path: &Path, version_info: &str) -> Result<DefaultBackingStorage> {
+    turbo_backing_storage(path, version_info)
 }

turbopack/crates/turbo-tasks-backend/tests/test_config.trs

@@ -11,7 +11,8 @@
     turbo_tasks_backend::TurboTasksBackend::new(
       turbo_tasks_backend::BackendOptions::default(),
       turbo_tasks_backend::default_backing_storage(
-        path.as_path()
+        path.as_path(),
+        "test"
       ).unwrap()
     )
   )