Turbopack: avoid using rayon in favor of tokio tasks (#82256)

### What?

Avoid mixing rayon and tokio. Only use tokio. Implemented parallel for_each and map on top of tokio.

Closes PACK-5158

Author: sokra

Cargo.lock

@@ -3,7 +3,7 @@
 use std::path::PathBuf;
 
 use anyhow::{Context, Result, bail};
-use turbo_persistence::{MetaFileEntryInfo, TurboPersistence};
+use turbo_persistence::{MetaFileEntryInfo, SerialScheduler, TurboPersistence};
 
 fn main() -> Result<()> {
     // Get CLI argument
@@ -16,7 +16,7 @@ fn main() -> Result<()> {
         bail!("The provided path does not exist: {}", path.display());
     }
 
-    let db = TurboPersistence::open_read_only(path)?;
+    let db: TurboPersistence<SerialScheduler> = TurboPersistence::open_read_only(path)?;
     let meta_info = db
         .meta_info()
         .context("Failed to retrieve meta information")?;

turbopack/crates/turbo-persistence-tools/src/main.rs

@@ -22,7 +22,6 @@ memmap2 = "0.9.5"
 parking_lot = { workspace = true }
 qfilter = { version = "0.2.4", features = ["serde"] }
 quick_cache = { workspace = true }
-rayon = { workspace = true }
 rustc-hash = { workspace = true }
 smallvec = { workspace = true }
 thread_local = { workspace = true }
@@ -32,6 +31,7 @@ zstd = { version = "0.13.2", features = ["zdict_builder"] }
 
 [dev-dependencies]
 rand = { workspace = true, features = ["small_rng"] }
+rayon = { workspace = true }
 tempfile = { workspace = true }
 
 [lints]

turbopack/crates/turbo-persistence/Cargo.toml

@@ -1,3 +1,5 @@
+use std::mem::take;
+
 use crate::{
     ValueBuffer,
     collector_entry::{CollectorEntry, CollectorEntryValue, EntryKey},
@@ -111,4 +113,11 @@ impl<K: StoreKey, const SIZE_SHIFT: usize> Collector<K, SIZE_SHIFT> {
         self.total_value_size = 0;
         self.entries.drain(..)
     }
+
+    /// Clears the collector and drops the capacity
+    pub fn drop_contents(&mut self) {
+        drop(take(&mut self.entries));
+        self.total_key_size = 0;
+        self.total_value_size = 0;
+    }
 }