feat: support invalidate module (#2005)

* feat: support invalidate module

* feat: remove unless code

* feat: clear expired module

---------

Co-authored-by: ADNY <66500121+ErKeLost@users.noreply.github.com>
Co-authored-by: brightwu <1521488775@qq.com>

.changeset/spicy-snakes-pay.md

@@ -0,0 +1,5 @@
+---
+"@farmfe/core": patch
+---
+
+feat: support invalidateModule

crates/compiler/src/build/module_cache.rs

@@ -234,6 +234,7 @@ pub fn set_module_graph_cache(module_ids: Vec<ModuleId>, context: &Arc<Compilati
             }
           })
           .collect(),
+        is_expired: false,
       };
 
       context

crates/core/src/cache/cache_store.rs

@@ -9,7 +9,7 @@ use std::{
 
 use crate::config::Mode;
 
-const FARM_CACHE_VERSION: &str = "0.4.10";
+const FARM_CACHE_VERSION: &str = "0.4.20";
 const FARM_CACHE_MANIFEST_FILE: &str = "farm-cache.json";
 
 // TODO make CacheStore a trait and implement DiskCacheStore or RemoteCacheStore or more.
@@ -174,6 +174,19 @@ impl CacheStore {
 
     None
   }
+
+  pub fn remove_cache(&self, name: &str) {
+    if !self.manifest.contains_key(name) {
+      return;
+    }
+
+    let (_, cache_key) = self.manifest.remove(name).unwrap();
+    let cache_file = self.cache_dir.join(cache_key);
+
+    if cache_file.exists() && cache_file.is_file() {
+      std::fs::remove_file(cache_file).ok();
+    }
+  }
 }
 
 /// Cache key of the store, it's a pair of (name, cache_key), a name should only be related to one cache key.

crates/core/src/cache/module_cache.rs

@@ -42,6 +42,14 @@ pub struct CachedModule {
   pub module: Module,
   pub dependencies: Vec<CachedModuleDependency>,
   pub watch_dependencies: Vec<CachedWatchDependency>,
+  ///
+  /// `default`: false
+  ///
+  /// true: it makes the cache expire.
+  ///
+  /// when writing to the cache next time, it will be cleared from memory.
+  ///
+  pub is_expired: bool,
 }
 
 impl CachedModule {

crates/core/src/cache/module_cache/immutable_modules.rs

@@ -104,15 +104,7 @@ impl ImmutableModulesMemoryStore {
 
 impl ModuleMemoryStore for ImmutableModulesMemoryStore {
   fn has_cache(&self, key: &crate::module::ModuleId) -> bool {
-    if self.cached_modules.contains_key(key) {
-      return true;
-    }
-
-    if let Some(package_key) = self.manifest.get(key) {
-      return self.store.has_cache(package_key.value());
-    }
-
-    false
+    self.get_cache_ref(key).is_some_and(|m| !m.is_expired)
   }
 
   fn set_cache(&self, key: crate::module::ModuleId, module: super::CachedModule) {
@@ -179,18 +171,45 @@ impl ModuleMemoryStore for ImmutableModulesMemoryStore {
 
   fn write_cache(&self) {
     let mut packages = HashMap::new();
+    let mut pending_remove_modules = HashSet::new();
+    let mut maybe_remove_package = HashSet::new();
 
     for item in self.cached_modules.iter() {
       let module = item.value();
+
       let package_key =
         CachedPackage::gen_key(&module.module.package_name, &module.module.package_version);
 
+      if module.is_expired {
+        pending_remove_modules.insert(item.key().clone());
+        maybe_remove_package.insert(package_key);
+        continue;
+      }
+
       let package = packages.entry(package_key.clone()).or_insert_with(Vec::new);
 
       package.push(item.key().clone());
       self.manifest.insert(item.key().clone(), package_key);
     }
 
+    for key in pending_remove_modules {
+      self.cached_modules.remove(&key);
+      self.manifest.remove(&key);
+      self.manifest_reversed.iter_mut().for_each(|mut item| {
+        if item.value_mut().contains(&key) {
+          item.value_mut().remove(&key);
+        }
+      })
+    }
+
+    for package in maybe_remove_package {
+      if packages.contains_key(&package) {
+        return;
+      }
+
+      self.store.remove_cache(&package);
+    }
+
     let manifest = self
       .manifest
       .iter()
@@ -283,13 +302,9 @@ impl ModuleMemoryStore for ImmutableModulesMemoryStore {
   }
 
   fn invalidate_cache(&self, key: &ModuleId) {
-    self.cached_modules.remove(key);
-    self.manifest.remove(key);
-    self.manifest_reversed.iter_mut().for_each(|mut item| {
-      if item.value_mut().contains(key) {
-        item.value_mut().remove(key);
-      }
-    })
+    if let Some(mut m) = self.get_cache_mut_ref(key) {
+      m.is_expired = true;
+    }
   }
 
   fn is_cache_changed(&self, module: &crate::module::Module) -> bool {

crates/core/src/cache/module_cache/mutable_modules.rs

@@ -22,7 +22,7 @@ pub struct MutableModulesMemoryStore {
   /// ModuleId -> Cached Module
   cached_modules: DashMap<ModuleId, CachedModule>,
 }
-
+// TODO: cache unit test
 impl MutableModulesMemoryStore {
   pub fn new(cache_dir_str: &str, namespace: &str, mode: Mode) -> Self {
     Self {
@@ -58,11 +58,7 @@ impl MutableModulesMemoryStore {
 
 impl ModuleMemoryStore for MutableModulesMemoryStore {
   fn has_cache(&self, key: &ModuleId) -> bool {
-    if self.cached_modules.contains_key(key) {
-      return true;
-    }
-
-    self.store.has_cache(&key.to_string())
+    self.get_cache_ref(key).is_some_and(|m| !m.is_expired)
   }
 
   fn set_cache(&self, key: ModuleId, module: CachedModule) {
@@ -125,9 +121,14 @@ impl ModuleMemoryStore for MutableModulesMemoryStore {
 
   fn write_cache(&self) {
     let mut cache_map = HashMap::new();
-
+    let mut pending_removed_modules = vec![];
     for entry in self.cached_modules.iter() {
       let module = entry.value();
+      if module.is_expired {
+        pending_removed_modules.push(module.module.id.clone());
+        continue;
+      }
+
       let store_key = self.gen_cache_store_key(&module.module);
 
       if self.store.is_cache_changed(&store_key) {
@@ -141,10 +142,17 @@ impl ModuleMemoryStore for MutableModulesMemoryStore {
       .collect::<HashMap<_, _>>();
 
     self.store.write_cache(cache_map);
+
+    for module_id in pending_removed_modules {
+      self.cached_modules.remove(&module_id);
+      self.store.remove_cache(&module_id.to_string());
+    }
   }
 
   fn invalidate_cache(&self, key: &ModuleId) {
-    self.cached_modules.remove(key);
+    if let Some(mut m) = self.get_cache_mut_ref(key) {
+      m.is_expired = true;
+    };
   }
 
   fn is_cache_changed(&self, module: &crate::module::Module) -> bool {

crates/core/src/context/mod.rs

@@ -164,6 +164,10 @@ impl CompilationContext {
     resolve_cache.insert(param, result);
   }
 
+  pub fn invalidate_module(&self, module_id: &ModuleId) {
+    self.cache_manager.module_cache.invalidate_cache(module_id);
+  }
+
   pub fn clear_log_store(&self) {
     let mut log_store = self.log_store.lock();
     log_store.clear();

crates/node/src/lib.rs

@@ -475,6 +475,18 @@ impl JsCompiler {
     let context = self.compiler.context();
     context.record_manager.to_string()
   }
+
+  #[napi]
+  pub fn invalidate_module(&self, module_id: String) {
+    invalidate_module(self, module_id);
+  }
+}
+
+fn invalidate_module(js_compiler: &JsCompiler, module_id: String) {
+  let context = js_compiler.compiler.context();
+  let module_id = ModuleId::new(&module_id, "", &context.config.root);
+
+  context.invalidate_module(&module_id);
 }
 
 #[cfg(feature = "file_watcher")]

packages/core/binding/binding.d.ts

@@ -78,4 +78,5 @@ export declare class Compiler {
   relativeModulePaths(): Array<string>
   resource(name: string): Buffer | null
   stats(): string
+  invalidateModule(moduleId: string): void
 }

packages/core/src/compiler/index.ts

@@ -240,4 +240,8 @@ export class Compiler {
   stats() {
     return this._bindingCompiler.stats();
   }
+
+  invalidateModule(moduleId: string) {
+    this._bindingCompiler.invalidateModule(moduleId);
+  }
 }