feat(vendor): support modifying remote files in vendor folder without checksum errors

Cargo.toml

@@ -47,7 +47,7 @@ deno_ast = { version = "=0.38.2", features = ["transpiling"] }
 deno_core = { version = "0.283.0" }
 
 deno_bench_util = { version = "0.147.0", path = "./bench_util" }
-deno_lockfile = "0.19.0"
+deno_lockfile = "0.20.0"
 deno_media_type = { version = "0.1.4", features = ["module_specifier"] }
 deno_permissions = { version = "0.13.0", path = "./runtime/permissions" }
 deno_runtime = { version = "0.161.0", path = "./runtime" }
@@ -100,7 +100,7 @@ chrono = { version = "0.4", default-features = false, features = ["std", "serde"
 console_static_text = "=0.8.1"
 data-encoding = "2.3.3"
 data-url = "=0.3.0"
-deno_cache_dir = "=0.7.1"
+deno_cache_dir = "=0.10.0"
 dlopen2 = "0.6.1"
 ecb = "=0.1.2"
 elliptic-curve = { version = "0.13.4", features = ["alloc", "arithmetic", "ecdh", "std", "pem"] }

cli/Cargo.toml

@@ -67,17 +67,17 @@ deno_ast = { workspace = true, features = ["bundler", "cjs", "codegen", "proposa
 deno_cache_dir = { workspace = true }
 deno_config = "=0.16.3"
 deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] }
-deno_doc = { version = "=0.135.0", features = ["html", "syntect"] }
-deno_emit = "=0.40.3"
-deno_graph = { version = "=0.75.2", features = ["tokio_executor"] }
+deno_doc = { version = "=0.137.0", features = ["html", "syntect"] }
+deno_emit = "=0.41.0"
+deno_graph = { version = "=0.77.0", features = ["tokio_executor"] }
 deno_lint = { version = "=0.58.4", features = ["docs"] }
 deno_lockfile.workspace = true
-deno_npm = "=0.20.2"
+deno_npm = "=0.21.0"
 deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting"] }
 deno_semver = "=0.5.4"
 deno_task_shell = "=0.16.1"
 deno_terminal.workspace = true
-eszip = "=0.69.0"
+eszip = "=0.70.1"
 napi_sym.workspace = true
 
 async-trait.workspace = true

cli/args/lockfile.rs

@@ -2,18 +2,20 @@
 
 use std::path::PathBuf;
 
+use deno_core::anyhow::Context;
 use deno_core::error::AnyError;
 use deno_runtime::deno_node::PackageJson;
 
 use crate::args::ConfigFile;
+use crate::cache;
+use crate::util::fs::atomic_write_file;
 use crate::Flags;
 
 use super::DenoSubcommand;
 use super::InstallFlags;
 use super::InstallKind;
 
 pub use deno_lockfile::Lockfile;
-pub use deno_lockfile::LockfileError;
 
 pub fn discover(
   flags: &Flags,
@@ -54,6 +56,34 @@ pub fn discover(
     },
   };
 
-  let lockfile = Lockfile::new(filename, flags.lock_write)?;
+  let lockfile = if flags.lock_write {
+    Lockfile::new_empty(filename, true)
+  } else {
+    read_lockfile_at_path(filename)?
+  };
   Ok(Some(lockfile))
 }
+
+pub fn read_lockfile_at_path(filename: PathBuf) -> Result<Lockfile, AnyError> {
+  match std::fs::read_to_string(&filename) {
+    Ok(text) => Ok(Lockfile::with_lockfile_content(filename, &text, false)?),
+    Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
+      Ok(Lockfile::new_empty(filename, false))
+    }
+    Err(err) => Err(err).with_context(|| {
+      format!("Failed reading lockfile '{}'", filename.display())
+    }),
+  }
+}
+
+pub fn write_lockfile_if_has_changes(
+  lockfile: &Lockfile,
+) -> Result<(), AnyError> {
+  let Some(bytes) = lockfile.resolve_write_bytes() else {
+    return Ok(()); // nothing to do
+  };
+  // do an atomic write to reduce the chance of multiple deno
+  // processes corrupting the file
+  atomic_write_file(&lockfile.filename, bytes, cache::CACHE_PERM)
+    .context("Failed writing lockfile.")
+}

cli/args/mod.rs

@@ -34,8 +34,9 @@ pub use deno_config::TsConfigType;
 pub use deno_config::TsTypeLib;
 pub use deno_config::WorkspaceConfig;
 pub use flags::*;
+pub use lockfile::read_lockfile_at_path;
+pub use lockfile::write_lockfile_if_has_changes;
 pub use lockfile::Lockfile;
-pub use lockfile::LockfileError;
 pub use package_json::PackageJsonDepsProvider;
 
 use deno_ast::ModuleSpecifier;

cli/cache/mod.rs

@@ -228,7 +228,7 @@ impl Loader for FetchCacher {
         LoaderCacheSetting::Reload => {
           if matches!(file_fetcher.cache_setting(), CacheSetting::Only) {
             return Err(deno_core::anyhow::anyhow!(
-              "Failed to resolve version constraint. Try running again without --cached-only"
+              "Could not resolve version constraint using only cached data. Try running again without --cached-only"
             ));
           }
           Some(CacheSetting::ReloadAll)

cli/cache/module_info.rs

@@ -150,8 +150,9 @@ pub struct ModuleInfoCacheModuleAnalyzer<'a> {
   parser: &'a dyn ModuleParser,
 }
 
+#[async_trait::async_trait(?Send)]
 impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> {
-  fn analyze(
+  async fn analyze(
     &self,
     specifier: &ModuleSpecifier,
     source: Arc<str>,
@@ -176,8 +177,9 @@ impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> {
     }
 
     // otherwise, get the module info from the parsed source cache
+    // todo(23858): take advantage of this being async
     let analyzer = ParserModuleAnalyzer::new(self.parser);
-    let module_info = analyzer.analyze(specifier, source, media_type)?;
+    let module_info = analyzer.analyze(specifier, source, media_type).await?;
 
     // then attempt to cache it
     if let Err(err) = self.module_info_cache.set_module_info(

cli/errors.rs

@@ -14,6 +14,7 @@ use deno_core::error::AnyError;
 use deno_graph::source::ResolveError;
 use deno_graph::ModuleError;
 use deno_graph::ModuleGraphError;
+use deno_graph::ModuleLoadError;
 use deno_graph::ResolutionError;
 use import_map::ImportMapError;
 use std::fmt::Write;
@@ -27,24 +28,56 @@ fn get_diagnostic_class(_: &ParseDiagnostic) -> &'static str {
 }
 
 fn get_module_graph_error_class(err: &ModuleGraphError) -> &'static str {
+  use deno_graph::JsrLoadError;
+  use deno_graph::NpmLoadError;
+  use deno_graph::WorkspaceLoadError;
+
   match err {
+    ModuleGraphError::ResolutionError(err)
+    | ModuleGraphError::TypesResolutionError(err) => {
+      get_resolution_error_class(err)
+    }
     ModuleGraphError::ModuleError(err) => match err {
-      ModuleError::LoadingErr(_, _, err) => get_error_class_name(err.as_ref()),
       ModuleError::InvalidTypeAssertion { .. } => "SyntaxError",
       ModuleError::ParseErr(_, diagnostic) => get_diagnostic_class(diagnostic),
       ModuleError::UnsupportedMediaType { .. }
       | ModuleError::UnsupportedImportAttributeType { .. } => "TypeError",
-      ModuleError::Missing(_, _)
-      | ModuleError::MissingDynamic(_, _)
-      | ModuleError::MissingWorkspaceMemberExports { .. }
-      | ModuleError::UnknownExport { .. }
-      | ModuleError::UnknownPackage { .. }
-      | ModuleError::UnknownPackageReq { .. } => "NotFound",
+      ModuleError::Missing(_, _) | ModuleError::MissingDynamic(_, _) => {
+        "NotFound"
+      }
+      ModuleError::LoadingErr(_, _, err) => match err {
+        ModuleLoadError::Loader(err) => get_error_class_name(err.as_ref()),
+        ModuleLoadError::HttpsChecksumIntegrity(_)
+        | ModuleLoadError::TooManyRedirects => "Error",
+        ModuleLoadError::NodeUnknownBuiltinModule(_) => "NotFound",
+        ModuleLoadError::Decode(_) => "TypeError",
+        ModuleLoadError::Npm(err) => match err {
+          NpmLoadError::NotSupportedEnvironment
+          | NpmLoadError::PackageReqResolution(_)
+          | NpmLoadError::RegistryInfo(_) => "Error",
+          NpmLoadError::PackageReqReferenceParse(_) => "TypeError",
+        },
+        ModuleLoadError::Jsr(err) => match err {
+          JsrLoadError::UnsupportedManifestChecksum
+          | JsrLoadError::PackageFormat(_) => "TypeError",
+          JsrLoadError::ContentLoadExternalSpecifier
+          | JsrLoadError::ContentLoad(_)
+          | JsrLoadError::ContentChecksumIntegrity(_)
+          | JsrLoadError::PackageManifestLoad(_, _)
+          | JsrLoadError::PackageVersionManifestChecksumIntegrity(..)
+          | JsrLoadError::PackageVersionManifestLoad(_, _)
+          | JsrLoadError::RedirectInPackage(_) => "Error",
+          JsrLoadError::PackageNotFound(_)
+          | JsrLoadError::PackageReqNotFound(_)
+          | JsrLoadError::PackageVersionNotFound(_)
+          | JsrLoadError::UnknownExport { .. } => "NotFound",
+        },
+        ModuleLoadError::Workspace(err) => match err {
+          WorkspaceLoadError::MemberInvalidExportPath { .. } => "TypeError",
+          WorkspaceLoadError::MissingMemberExports { .. } => "NotFound",
+        },
+      },
     },
-    ModuleGraphError::ResolutionError(err)
-    | ModuleGraphError::TypesResolutionError(err) => {
-      get_resolution_error_class(err)
-    }
   }
 }
 

cli/factory.rs

@@ -665,7 +665,6 @@ impl CliFactory {
           self.options.clone(),
           self.npm_resolver().await?.clone(),
           self.module_graph_builder().await?.clone(),
-          self.maybe_lockfile().clone(),
           self.type_checker().await?.clone(),
         )))
       })

cli/file_fetcher.rs

@@ -253,15 +253,30 @@ impl FileFetcher {
         deno_core::resolve_import(redirect_to, specifier.as_str())?;
       return Ok(Some(FileOrRedirect::Redirect(redirect)));
     }
-    let Some(bytes) = self.http_cache.read_file_bytes(
+    let result = self.http_cache.read_file_bytes(
       &cache_key,
       maybe_checksum
         .as_ref()
         .map(|c| deno_cache_dir::Checksum::new(c.as_str())),
       deno_cache_dir::GlobalToLocalCopy::Allow,
-    )?
-    else {
-      return Ok(None);
+    );
+    let bytes = match result {
+      Ok(Some(bytes)) => bytes,
+      Ok(None) => return Ok(None),
+      Err(err) => match err {
+        deno_cache_dir::CacheReadFileError::Io(err) => return Err(err.into()),
+        deno_cache_dir::CacheReadFileError::ChecksumIntegrity(err) => {
+          // convert to the equivalent deno_graph error so that it
+          // enhances it if this is passed to deno_graph
+          return Err(
+            deno_graph::source::ChecksumIntegrityError {
+              actual: err.actual,
+              expected: err.expected,
+            }
+            .into(),
+          );
+        }
+      },
     };
 
     Ok(Some(FileOrRedirect::File(File {