diff --git a/Cargo.lock b/Cargo.lock index 6f3efa049..8f61a4c50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -280,6 +280,7 @@ dependencies = [ "regex", "serde", "serde_json", + "sha2", "tempfile", "thiserror", "tokio", @@ -1071,6 +1072,17 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "siphasher" version = "0.3.11" diff --git a/Cargo.toml b/Cargo.toml index c04a7324f..163536b43 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ parking_lot = "0.12.0" regex = "1.10.2" serde = { version = "1.0.130", features = ["derive", "rc"] } serde_json = { version = "1.0.67", features = ["preserve_order"] } +sha2 = "^0.10.0" thiserror = "1.0.24" url = { version = "2.2.2", features = ["serde"] } diff --git a/js/mod.ts b/js/mod.ts index 573dcd291..1bc625e8a 100644 --- a/js/mod.ts +++ b/js/mod.ts @@ -61,6 +61,7 @@ export interface CreateGraphOptions { specifier: string, isDynamic: boolean, cacheSetting: CacheSetting, + checksum: string | undefined, ): Promise; /** The type of graph to build. `"all"` includes all dependencies of the * roots. `"typesOnly"` skips any code only dependencies that do not impact @@ -143,10 +144,18 @@ export async function createGraph( rootSpecifiers, async ( specifier: string, - isDynamic: boolean, - cacheSetting: CacheSetting, + options: { + isDynamic: boolean; + cacheSetting: CacheSetting; + checksum: string | undefined; + }, ) => { - const result = await load(specifier, isDynamic, cacheSetting); + const result = await load( + specifier, + options.isDynamic, + options.cacheSetting, + options.checksum, + ); if (result?.kind === "module") { if (typeof result.content === "string") { result.content = encoder.encode(result.content); diff --git a/lib/lib.rs b/lib/lib.rs index 3742f285a..5e9ec7d09 100644 --- a/lib/lib.rs +++ b/lib/lib.rs @@ -10,8 +10,8 @@ use std::collections::HashMap; use deno_graph::resolve_import; use deno_graph::source::load_data_url; use deno_graph::source::CacheInfo; -use deno_graph::source::CacheSetting; use deno_graph::source::LoadFuture; +use deno_graph::source::LoadOptions; use deno_graph::source::Loader; use deno_graph::source::NullFileSystem; use deno_graph::source::ResolutionMode; @@ -65,18 +65,28 @@ impl Loader for JsLoader { fn load( &mut self, specifier: &ModuleSpecifier, - is_dynamic: bool, - cache_setting: CacheSetting, + options: LoadOptions, ) -> LoadFuture { + #[derive(Serialize)] + struct JsLoadOptions { + is_dynamic: bool, + cache_setting: &'static str, + maybe_checksum: Option, + } + if specifier.scheme() == "data" { Box::pin(future::ready(load_data_url(specifier))) } else { let specifier = specifier.clone(); let context = JsValue::null(); let arg1 = JsValue::from(specifier.to_string()); - let arg2 = JsValue::from(is_dynamic); - let arg3 = JsValue::from(cache_setting.as_js_str()); - let result = self.load.call3(&context, &arg1, &arg2, &arg3); + let arg2 = serde_wasm_bindgen::to_value(&JsLoadOptions { + is_dynamic: options.is_dynamic, + cache_setting: options.cache_setting.as_js_str(), + maybe_checksum: options.maybe_checksum.map(|c| c.into_string()), + }) + .unwrap(); + let result = self.load.call2(&context, &arg1, &arg2); let f = async move { let response = match result { Ok(result) => { diff --git a/src/graph.rs b/src/graph.rs index afbbeafb9..39971b7ca 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -2574,6 +2574,15 @@ enum PendingInfoResponse { }, } +impl PendingInfoResponse { + fn specifier(&self) -> &ModuleSpecifier { + match self { + Self::External { specifier } => specifier, + Self::Module { specifier, .. } => specifier, + } + } +} + impl From for PendingInfoResponse { fn from(load_response: LoadResponse) -> Self { match load_response { @@ -2641,6 +2650,12 @@ struct PendingContentLoadItem { module_info: ModuleInfo, } +#[derive(Clone)] +struct PendingJsrPackageVersionInfoLoadItem { + checksum: String, + info: Arc, +} + type PendingResult = Shared>>>; @@ -2649,7 +2664,7 @@ struct PendingJsrState { pending_package_info_loads: HashMap>>>, pending_package_version_info_loads: - HashMap>>, + HashMap>, pending_resolutions: Vec, pending_content_loads: FuturesUnordered>, @@ -2670,7 +2685,11 @@ struct PendingState { #[derive(Clone)] enum ContentOrModuleInfo { Content(Arc<[u8]>), - ModuleInfo(ModuleInfo), + ModuleInfo { + info: ModuleInfo, + /// The checksum to use when loading the content + checksum: LoaderChecksum, + }, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -2829,16 +2848,42 @@ impl<'a, 'graph> Builder<'a, 'graph> { maybe_version_info, }) => match result { Ok(Some(response)) => { - let assert_types = - self.state.pending_specifiers.remove(&specifier).unwrap(); - for maybe_assert_type in assert_types { - self.visit( - &specifier, - &response, - maybe_assert_type, - maybe_range.clone(), - maybe_version_info.as_ref(), - ) + if maybe_version_info.is_none() + && self + .loader + .registry_package_url_to_nv(response.specifier()) + .is_some() + { + self.graph.module_slots.insert( + specifier.clone(), + ModuleSlot::Err(ModuleError::LoadingErr( + specifier.clone(), + maybe_range, + // Two tasks we need to do before removing this error message: + // 1. If someone imports a package via an HTTPS URL then we should probably + // bail completely on fast check because it could expose additional types + // not found in fast check, which might cause strange behaviour. + // 2. For HTTPS URLS imported from the registry, we should probably still + // compare it against the checksums found in the registry otherwise it might + // not end up in the lockfile causing a security issue. + Arc::new(anyhow!(concat!( + "Importing a JSR package via an HTTPS URL is not implemented. ", + "Use a jsr: specifier instead for the time being." + )), + ))), + ); + } else { + let assert_types = + self.state.pending_specifiers.remove(&specifier).unwrap(); + for maybe_assert_type in assert_types { + self.visit( + &specifier, + &response, + maybe_assert_type, + maybe_range.clone(), + maybe_version_info.as_ref(), + ) + } } Some(specifier) } @@ -3013,13 +3058,18 @@ impl<'a, 'graph> Builder<'a, 'graph> { .await .map(|info| (info, resolution_item.package_ref.sub_path())); match version_info_result { - Ok((version_info, sub_path)) => { + Ok((version_info_load_item, sub_path)) => { + let version_info = version_info_load_item.info; + self + .graph + .packages + .ensure_package(nv.clone(), version_info_load_item.checksum); let base_url = self.loader.registry_package_url(&nv); let export_name = normalize_export_name(sub_path); match version_info.export(&export_name) { Some(export_value) => { self.graph.packages.add_export( - nv.clone(), + &nv, ( normalize_export_name(resolution_item.package_ref.sub_path()) .to_string(), @@ -3311,14 +3361,48 @@ impl<'a, 'graph> Builder<'a, 'graph> { let base_url = version_info.base_url.as_str(); let base_url = base_url.strip_suffix('/').unwrap_or(base_url); if let Some(sub_path) = specifier.as_str().strip_prefix(base_url) { + let checksum = match version_info.inner.manifest.get(sub_path) { + Some(manifest_entry) => { + match manifest_entry.checksum.strip_prefix("sha256-") { + Some(checksum) => checksum.to_string(), + None => { + self.graph.module_slots.insert( + specifier.clone(), + ModuleSlot::Err(ModuleError::LoadingErr( + specifier.clone(), + maybe_range.cloned(), + Arc::new(anyhow!( + "Unsupported checksum in manifest. Maybe try upgrading deno?", + )), + )), + ); + return; + } + } + } + None => { + self.graph.module_slots.insert( + specifier.clone(), + ModuleSlot::Err(ModuleError::Missing( + specifier.clone(), + maybe_range.cloned(), + )), + ); + return; + } + }; + let checksum = LoaderChecksum::new(checksum.clone()); if let Some(module_info) = version_info.inner.module_info(sub_path) { // Check if this specifier is in the cache. If it is, then // don't use the module information as it may be out of date // with what's in the cache let fut = self.loader.load( specifier, - self.in_dynamic_branch, - CacheSetting::Only, + LoadOptions { + is_dynamic: self.in_dynamic_branch, + cache_setting: CacheSetting::Only, + maybe_checksum: Some(checksum.clone()), + }, ); self.state.pending.push_back({ let specifier = specifier.clone(); @@ -3331,9 +3415,10 @@ impl<'a, 'graph> Builder<'a, 'graph> { specifier: specifier.clone(), maybe_range, result: Ok(Some(PendingInfoResponse::Module { - content_or_module_info: ContentOrModuleInfo::ModuleInfo( - module_info, - ), + content_or_module_info: ContentOrModuleInfo::ModuleInfo { + info: module_info, + checksum, + }, specifier, maybe_headers: None, })), @@ -3354,6 +3439,16 @@ impl<'a, 'graph> Builder<'a, 'graph> { .module_slots .insert(specifier.clone(), ModuleSlot::Pending); return; + } else { + self.load_pending_module( + specifier.clone(), + maybe_range.map(ToOwned::to_owned), + specifier.clone(), + is_dynamic, + Some(checksum), + Some(version_info.clone()), + ); + return; } } } @@ -3397,8 +3492,14 @@ impl<'a, 'graph> Builder<'a, 'graph> { } } - let specifier = specifier.clone(); - self.load_pending_module(&specifier, maybe_range, &specifier, is_dynamic); + self.load_pending_module( + specifier.clone(), + maybe_range, + specifier.clone(), + is_dynamic, + None, + None, + ); } fn load_jsr_specifier( @@ -3414,7 +3515,7 @@ impl<'a, 'graph> Builder<'a, 'graph> { self.loader.registry_package_url_to_nv(&range.specifier) { self.graph.packages.add_dependency( - nv, + &nv, JsrDepPackageReq::jsr(package_ref.req().clone()), ); } @@ -3435,10 +3536,12 @@ impl<'a, 'graph> Builder<'a, 'graph> { match result { Ok(load_specifier) => { self.load_pending_module( - &specifier, + specifier.clone(), maybe_range.clone(), - &load_specifier, + load_specifier, is_dynamic, + None, + None, ); } Err(err) => { @@ -3543,7 +3646,7 @@ impl<'a, 'graph> Builder<'a, 'graph> { self.loader.registry_package_url_to_nv(&range.specifier) { self.graph.packages.add_dependency( - nv, + &nv, JsrDepPackageReq::npm(package_ref.req().clone()), ); } @@ -3592,25 +3695,32 @@ impl<'a, 'graph> Builder<'a, 'graph> { fn load_pending_module( &mut self, - requested_specifier: &Url, + requested_specifier: Url, maybe_range: Option, - load_specifier: &Url, + load_specifier: Url, is_dynamic: bool, + maybe_checksum: Option, + maybe_version_info: Option, ) { self .graph .module_slots .insert(requested_specifier.clone(), ModuleSlot::Pending); - let load_specifier = load_specifier.clone(); - let requested_specifier = requested_specifier.clone(); let fut = self .loader - .load(&load_specifier, is_dynamic, CacheSetting::Use) + .load( + &load_specifier, + LoadOptions { + is_dynamic, + cache_setting: CacheSetting::Use, + maybe_checksum, + }, + ) .map(move |result| PendingInfo { specifier: requested_specifier, maybe_range, result: result.map(|r| r.map(Into::into)), - maybe_version_info: None, + maybe_version_info, }); self.state.pending.push_back(Box::pin(fut)); } @@ -3633,8 +3743,11 @@ impl<'a, 'graph> Builder<'a, 'graph> { .unwrap(); let fut = self.loader.load( &specifier, - false, - self.fill_pass_mode.to_cache_setting(), + LoadOptions { + is_dynamic: false, + cache_setting: self.fill_pass_mode.to_cache_setting(), + maybe_checksum: None, + }, ); let fut = async move { let data = fut.await.map_err(Arc::new)?; @@ -3673,18 +3786,35 @@ impl<'a, 'graph> Builder<'a, 'graph> { package_nv.name, package_nv.version )) .unwrap(); + let maybe_expected_checksum = self + .graph + .packages + .get_manifest_checksum(package_nv) + .map(|checksum| LoaderChecksum::new(checksum.clone())); let fut = self.loader.load( &specifier, - false, - self.fill_pass_mode.to_cache_setting(), + LoadOptions { + is_dynamic: false, + cache_setting: self.fill_pass_mode.to_cache_setting(), + // we won't have a checksum when loading this the + // first time or when not using a lockfile + maybe_checksum: maybe_expected_checksum.clone(), + }, ); let fut = async move { let data = fut.await.map_err(Arc::new)?; match data { Some(LoadResponse::Module { content, .. }) => { + // if we have the expected checksum, then we can re-use that here + let checksum = maybe_expected_checksum + .map(|c| c.into_string()) + .unwrap_or_else(|| LoaderChecksum::gen(&content)); let version_info: JsrPackageVersionInfo = serde_json::from_slice(&content).map_err(|e| Arc::new(e.into()))?; - Ok(Arc::new(version_info)) + Ok(PendingJsrPackageVersionInfoLoadItem { + checksum, + info: Arc::new(version_info), + }) } _ => Err(Arc::new(anyhow!("Not found: {}", specifier))), } @@ -3776,15 +3906,18 @@ impl<'a, 'graph> Builder<'a, 'graph> { let (content, maybe_module_analyzer) = match content_or_module_info { ContentOrModuleInfo::Content(content) => (content, None), - ContentOrModuleInfo::ModuleInfo(info) => { + ContentOrModuleInfo::ModuleInfo { info, checksum } => { self.state.jsr.pending_content_loads.push({ let specifier = specifier.clone(); let maybe_range = maybe_referrer.clone(); let module_info = info.clone(); let fut = self.loader.load( &specifier, - self.in_dynamic_branch, - CacheSetting::Use, + LoadOptions { + is_dynamic: self.in_dynamic_branch, + cache_setting: CacheSetting::Use, + maybe_checksum: Some(checksum), + }, ); async move { let result = fut.await; @@ -4446,13 +4579,12 @@ mod tests { fn load( &mut self, specifier: &ModuleSpecifier, - is_dynamic: bool, - _cache_setting: CacheSetting, + options: LoadOptions, ) -> LoadFuture { let specifier = specifier.clone(); match specifier.as_str() { "file:///foo.js" => { - assert!(!is_dynamic); + assert!(!options.is_dynamic); self.loaded_foo = true; Box::pin(async move { Ok(Some(LoadResponse::Module { @@ -4463,7 +4595,7 @@ mod tests { }) } "file:///bar.js" => { - assert!(is_dynamic); + assert!(options.is_dynamic); self.loaded_bar = true; Box::pin(async move { Ok(Some(LoadResponse::Module { @@ -4474,7 +4606,7 @@ mod tests { }) } "file:///baz.js" => { - assert!(is_dynamic); + assert!(options.is_dynamic); self.loaded_baz = true; Box::pin(async move { Ok(Some(LoadResponse::Module { @@ -4515,8 +4647,7 @@ mod tests { fn load( &mut self, specifier: &ModuleSpecifier, - _is_dynamic: bool, - _cache_setting: CacheSetting, + _options: LoadOptions, ) -> LoadFuture { let specifier = specifier.clone(); match specifier.as_str() { @@ -4602,8 +4733,7 @@ mod tests { fn load( &mut self, specifier: &ModuleSpecifier, - _is_dynamic: bool, - _cache_setting: CacheSetting, + _options: LoadOptions, ) -> LoadFuture { let specifier = specifier.clone(); match specifier.as_str() { @@ -4670,8 +4800,7 @@ mod tests { fn load( &mut self, specifier: &ModuleSpecifier, - _is_dynamic: bool, - _cache_setting: CacheSetting, + _options: LoadOptions, ) -> LoadFuture { let specifier = specifier.clone(); match specifier.as_str() { @@ -4795,8 +4924,7 @@ mod tests { fn load( &mut self, specifier: &ModuleSpecifier, - is_dynamic: bool, - _cache_setting: CacheSetting, + options: LoadOptions, ) -> LoadFuture { let specifier = specifier.clone(); match specifier.as_str() { @@ -4811,7 +4939,7 @@ mod tests { })) }), "file:///bar.js" => { - assert!(!is_dynamic); + assert!(!options.is_dynamic); self.loaded_bar = true; Box::pin(async move { Ok(Some(LoadResponse::Module { @@ -4844,8 +4972,7 @@ mod tests { fn load( &mut self, specifier: &ModuleSpecifier, - is_dynamic: bool, - _cache_setting: CacheSetting, + options: LoadOptions, ) -> LoadFuture { let specifier = specifier.clone(); match specifier.as_str() { @@ -4869,7 +4996,7 @@ mod tests { })) }), "file:///bar.ts" => { - assert!(!is_dynamic); + assert!(!options.is_dynamic); Box::pin(async move { Ok(Some(LoadResponse::Module { specifier: specifier.clone(), @@ -4879,7 +5006,7 @@ mod tests { }) } "file:///baz.json" => { - assert!(!is_dynamic); + assert!(!options.is_dynamic); Box::pin(async move { Ok(Some(LoadResponse::Module { specifier: specifier.clone(), diff --git a/src/packages.rs b/src/packages.rs index d3514ec7b..0b9b9c88d 100644 --- a/src/packages.rs +++ b/src/packages.rs @@ -25,6 +25,11 @@ pub struct JsrPackageInfoVersion { // no used fields yet } +#[derive(Debug, Serialize, Deserialize, Clone, Default)] +pub struct JsrPackageVersionManifestEntry { + pub checksum: String, +} + #[derive(Debug, Serialize, Deserialize, Clone, Default)] pub struct JsrPackageVersionInfo { // ensure the fields on here are resilient to change @@ -32,6 +37,7 @@ pub struct JsrPackageVersionInfo { pub exports: serde_json::Value, #[serde(rename = "moduleGraph1")] pub module_graph: Option, + pub manifest: HashMap, } impl JsrPackageVersionInfo { @@ -80,13 +86,21 @@ impl JsrPackageVersionInfo { } } -#[derive(Default, Debug, Clone)] +#[derive(Debug, Clone)] struct PackageNvInfo { + manifest_checksum: String, /// Collection of exports used. exports: IndexMap, found_dependencies: HashSet, } +#[derive(Debug, Clone)] +pub struct PackageManifestIntegrityError { + pub nv: PackageNv, + pub actual: String, + pub expected: String, +} + #[derive(Debug, Clone, Default, Serialize)] pub struct PackageSpecifiers { #[serde(flatten)] @@ -111,40 +125,90 @@ impl PackageSpecifiers { nvs.push(nv.clone()); } self.package_reqs.insert(package_req, nv.clone()); - // always create an entry because this is used in the lockfile - // todo(dsherret): add integrity for the package here - self.packages.entry(nv.clone()).or_default(); + } + + pub(crate) fn ensure_package( + &mut self, + nv: PackageNv, + manifest_checksum: String, + ) { + self.packages.entry(nv).or_insert_with(|| PackageNvInfo { + manifest_checksum, + exports: Default::default(), + found_dependencies: Default::default(), + }); + } + + pub(crate) fn get_manifest_checksum( + &self, + nv: &PackageNv, + ) -> Option<&String> { + self.packages.get(nv).map(|p| &p.manifest_checksum) + } + + pub fn add_manifest_checksum( + &mut self, + nv: PackageNv, + checksum: String, + ) -> Result<(), Box> { + let package = self.packages.get(&nv); + if let Some(package) = package { + if package.manifest_checksum != checksum { + Err(Box::new(PackageManifestIntegrityError { + nv, + actual: checksum, + expected: package.manifest_checksum.clone(), + })) + } else { + Ok(()) + } + } else { + self.packages.insert( + nv, + PackageNvInfo { + manifest_checksum: checksum, + exports: Default::default(), + found_dependencies: Default::default(), + }, + ); + Ok(()) + } } /// Gets the dependencies (package constraints) of JSR packages found in the graph. - pub fn package_deps( + pub fn packages_with_checksum_and_deps( &self, - ) -> impl Iterator)> - { + ) -> impl Iterator< + Item = (&PackageNv, &String, impl Iterator), + > { self.packages.iter().map(|(nv, info)| { let deps = info.found_dependencies.iter(); - (nv, deps) + (nv, &info.manifest_checksum, deps) }) } pub(crate) fn add_dependency( &mut self, - nv: PackageNv, + nv: &PackageNv, dep: JsrDepPackageReq, ) { self .packages - .entry(nv.clone()) - .or_default() + .get_mut(nv) + .unwrap() .found_dependencies .insert(dep); } - pub(crate) fn add_export(&mut self, nv: PackageNv, export: (String, String)) { + pub(crate) fn add_export( + &mut self, + nv: &PackageNv, + export: (String, String), + ) { self .packages - .entry(nv.clone()) - .or_default() + .get_mut(nv) + .unwrap() .exports .insert(export.0, export.1); } diff --git a/src/source/mod.rs b/src/source/mod.rs index 834dc4a88..02b7af723 100644 --- a/src/source/mod.rs +++ b/src/source/mod.rs @@ -109,6 +109,68 @@ impl CacheSetting { pub static DEFAULT_DENO_REGISTRY_URL: Lazy = Lazy::new(|| Url::parse("https://jsr.io").unwrap()); +#[derive(Debug, Error)] +#[error("Integrity check failed.\n\nActual: {}\nExpected: {}", .actual, .expected)] +pub struct ChecksumIntegrityError { + pub actual: String, + pub expected: String, +} + +/// A SHA-256 checksum to verify the contents of a module +/// with while loading. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct LoaderChecksum(String); + +impl LoaderChecksum { + pub fn new(checksum: String) -> Self { + Self(checksum) + } + + pub fn into_string(self) -> String { + self.0 + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn check_source( + &self, + source: &[u8], + ) -> Result<(), ChecksumIntegrityError> { + let actual_checksum = Self::gen(source); + if self.0 == actual_checksum { + Ok(()) + } else { + Err(ChecksumIntegrityError { + actual: actual_checksum, + expected: self.0.to_string(), + }) + } + } + + pub fn gen(source: &[u8]) -> String { + use sha2::Digest; + use sha2::Sha256; + let mut hasher = Sha256::new(); + hasher.update(source); + format!("{:x}", hasher.finalize()) + } +} + +#[derive(Debug, Clone)] +pub struct LoadOptions { + pub is_dynamic: bool, + pub cache_setting: CacheSetting, + /// It is the loader's responsibility to verify the provided checksum if it + /// exists because in the CLI we only verify the checksum of the source when + /// it is loaded from the global cache. We don't verify it when loaded from + /// the vendor folder. + /// + /// The source may be verified by running `checksum.check_source(content)?`. + pub maybe_checksum: Option, +} + /// A trait which allows asynchronous loading of source files into a module /// graph in a thread safe way as well as a way to provide additional meta data /// about any cached resources. @@ -135,8 +197,7 @@ pub trait Loader { fn load( &mut self, specifier: &ModuleSpecifier, - is_dynamic: bool, - cache_setting: CacheSetting, + options: LoadOptions, ) -> LoadFuture; /// Cache the module info for the provided specifier if the loader @@ -502,8 +563,7 @@ impl Loader for MemoryLoader { fn load( &mut self, specifier: &ModuleSpecifier, - _is_dynamic: bool, - _cache_setting: CacheSetting, + _options: LoadOptions, ) -> LoadFuture { let response = match self.sources.get(specifier) { Some(Ok(response)) => Ok(Some(response.clone())), diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs index 604a766ac..3f14cffa6 100644 --- a/tests/helpers/mod.rs +++ b/tests/helpers/mod.rs @@ -1,11 +1,17 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::collections::HashMap; use std::path::Path; use std::path::PathBuf; mod test_builder; +use deno_graph::source::recommended_registry_package_url; +use deno_graph::source::recommended_registry_package_url_to_nv; +use deno_graph::source::LoaderChecksum; +use deno_graph::source::DEFAULT_DENO_REGISTRY_URL; use deno_graph::WorkspaceMember; +use deno_semver::package::PackageNv; use indexmap::IndexMap; use serde::de::DeserializeOwned; use serde::Deserialize; @@ -58,12 +64,82 @@ impl Spec { } text } + + /// Fills the `manifest` field in the `_meta.json` files with the checksums + /// so that we don't need to bother having them in the tests. + pub fn fill_jsr_meta_files_with_checksums(&mut self) { + for (nv, checksums_by_files) in self.get_jsr_checksums() { + let base_specifier = + recommended_registry_package_url(&DEFAULT_DENO_REGISTRY_URL, &nv); + let meta_file = base_specifier + .join(&format!("../{}_meta.json", nv.version)) + .unwrap(); + + let meta_file = self + .files + .iter_mut() + .find(|f| f.url() == meta_file) + .unwrap_or_else(|| panic!("Could not find in specs: {}", meta_file)); + let mut meta_value = serde_json::from_str::< + HashMap, + >(&meta_file.text) + .unwrap(); + let manifest = meta_value + .entry("manifest".to_string()) + .or_insert_with(|| serde_json::Value::Object(Default::default())) + .as_object_mut() + .unwrap(); + for (file, checksum) in checksums_by_files { + if !manifest.contains_key(&file) { + manifest.insert(file, checksum); + } + } + // use the original text as the emit text so we don't + // end up with these hashes in the output + meta_file.emit_text = Some(std::mem::take(&mut meta_file.text)); + meta_file.text = serde_json::to_string_pretty(&meta_value).unwrap(); + } + } + + pub fn get_jsr_checksums( + &self, + ) -> HashMap> { + let mut checksums_by_package: HashMap< + PackageNv, + HashMap, + > = Default::default(); + for file in &self.files { + if let Some(nv) = recommended_registry_package_url_to_nv( + &DEFAULT_DENO_REGISTRY_URL, + &file.url(), + ) { + let base_specifier = + recommended_registry_package_url(&DEFAULT_DENO_REGISTRY_URL, &nv); + let relative_url = file + .url() + .to_string() + .strip_prefix(base_specifier.to_string().strip_suffix('/').unwrap()) + .unwrap() + .to_string(); + checksums_by_package.entry(nv.clone()).or_default().insert( + relative_url, + serde_json::json!({ + "size": file.text.len(), + "checksum": format!("sha256-{}", LoaderChecksum::gen(file.text.as_bytes())), + }), + ); + } + } + checksums_by_package + } } #[derive(Debug)] pub struct SpecFile { pub specifier: String, pub text: String, + /// Text to use when emitting the spec file. + pub emit_text: Option, pub headers: IndexMap, } @@ -76,7 +152,7 @@ impl SpecFile { serde_json::to_string(&self.headers).unwrap() )); } - text.push_str(&self.text); + text.push_str(self.emit_text.as_ref().unwrap_or(&self.text)); text } @@ -119,7 +195,12 @@ pub fn get_specs_in_dir(path: &Path) -> Vec<(PathBuf, Spec)> { }; files .into_iter() - .map(|file| (file.path, parse_spec(file.text))) + .map(|file| { + let mut spec = parse_spec(file.text); + // always do this as we want this for the spec tests + spec.fill_jsr_meta_files_with_checksums(); + (file.path, spec) + }) .collect() } @@ -140,6 +221,7 @@ fn parse_spec(text: String) -> Spec { current_file = Some(SpecFile { specifier: specifier.to_string(), text: String::new(), + emit_text: None, headers: Default::default(), }); } else if let Some(headers) = line.strip_prefix("HEADERS: ") { diff --git a/tests/helpers/test_builder.rs b/tests/helpers/test_builder.rs index 3e6d45632..d6f3ac9a7 100644 --- a/tests/helpers/test_builder.rs +++ b/tests/helpers/test_builder.rs @@ -4,12 +4,14 @@ use deno_ast::ModuleSpecifier; use deno_graph::source::CacheInfo; use deno_graph::source::CacheSetting; use deno_graph::source::LoadFuture; +use deno_graph::source::LoadOptions; use deno_graph::source::Loader; use deno_graph::source::MemoryLoader; use deno_graph::BuildDiagnostic; use deno_graph::GraphKind; use deno_graph::ModuleGraph; use deno_graph::WorkspaceMember; +use futures::FutureExt; #[derive(Default)] pub struct TestLoader { @@ -25,22 +27,29 @@ impl Loader for TestLoader { fn load( &mut self, specifier: &ModuleSpecifier, - is_dynamic: bool, - cache_setting: CacheSetting, + options: LoadOptions, ) -> LoadFuture { - match cache_setting { + let checksum = options.maybe_checksum.clone(); + let future = match options.cache_setting { // todo(dsherret): in the future, actually make this use the cache - CacheSetting::Use => { - self.remote.load(specifier, is_dynamic, cache_setting) - } + CacheSetting::Use => self.remote.load(specifier, options), // todo(dsherret): in the future, make this update the cache - CacheSetting::Reload => { - self.remote.load(specifier, is_dynamic, cache_setting) - } - CacheSetting::Only => { - self.cache.load(specifier, is_dynamic, cache_setting) + CacheSetting::Reload => self.remote.load(specifier, options), + CacheSetting::Only => self.cache.load(specifier, options), + }; + async move { + let response = future.await?; + if let Some(deno_graph::source::LoadResponse::Module { + content, .. + }) = &response + { + if let Some(checksum) = checksum { + checksum.check_source(content)?; + } } + Ok(response) } + .boxed_local() } } diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 938df8b60..fd18ddc35 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -13,6 +13,7 @@ use anyhow::anyhow; use deno_ast::ModuleSpecifier; use deno_graph::source::CacheSetting; use deno_graph::source::LoadFuture; +use deno_graph::source::LoadOptions; use deno_graph::source::LoadResponse; use deno_graph::source::MemoryFileSystem; use deno_graph::source::MemoryLoader; @@ -69,16 +70,18 @@ async fn test_graph_specs() { } let result = builder.build().await; - let update_var = std::env::var("UPDATE"); let mut output_text = serde_json::to_string_pretty(&result.graph).unwrap(); output_text.push('\n'); // include the list of jsr dependencies let jsr_deps = result .graph .packages - .package_deps() - .map(|(k, v)| { - (k.to_string(), v.map(|v| v.to_string()).collect::>()) + .packages_with_checksum_and_deps() + .map(|(k, _checksum, deps)| { + ( + k.to_string(), + deps.map(|d| d.to_string()).collect::>(), + ) }) .filter(|(_, v)| !v.is_empty()) .collect::>(); @@ -135,7 +138,9 @@ async fn test_graph_specs() { .iter() .map(|d| serde_json::to_value(d.to_string()).unwrap()) .collect::>(); - let spec = if update_var.as_ref().map(|v| v.as_str()) == Ok("1") { + let update = + std::env::var("UPDATE").as_ref().map(|v| v.as_str()) == Ok("1"); + let spec = if update { let mut spec = spec; spec.output_file.text = output_text.clone(); spec.diagnostics = diagnostics.clone(); @@ -447,11 +452,12 @@ async fn test_jsr_version_not_found_then_found() { fn load( &mut self, specifier: &ModuleSpecifier, - is_dynamic: bool, - cache_setting: CacheSetting, + options: LoadOptions, ) -> LoadFuture { - assert!(!is_dynamic); - self.requests.push((specifier.to_string(), cache_setting)); + assert!(!options.is_dynamic); + self + .requests + .push((specifier.to_string(), options.cache_setting)); let specifier = specifier.clone(); match specifier.as_str() { "file:///main.ts" => Box::pin(async move { @@ -466,7 +472,7 @@ async fn test_jsr_version_not_found_then_found() { Ok(Some(LoadResponse::Module { specifier: specifier.clone(), maybe_headers: None, - content: match cache_setting { + content: match options.cache_setting { CacheSetting::Only | CacheSetting::Use => { // first time it won't have the version br#"{ "versions": { "1.0.0": {} } }"#.to_vec().into() @@ -483,7 +489,17 @@ async fn test_jsr_version_not_found_then_found() { Ok(Some(LoadResponse::Module { specifier: specifier.clone(), maybe_headers: None, - content: br#"{ "exports": { ".": "./mod.ts" } }"#.to_vec().into(), + content: br#"{ + "exports": { ".": "./mod.ts" }, + "manifest": { + "/mod.ts": { + "size": 123, + "checksum": "sha256-b8059cfb1ea623e79efbf432db31595df213c99c6534c58bec9d5f5e069344df" + } + } + }"# + .to_vec() + .into(), })) }), "https://jsr.io/@scope/a/1.2.0/mod.ts" => Box::pin(async move { diff --git a/tests/specs/graph/JsrSpecifiers_Checksum_Mismatch.txt b/tests/specs/graph/JsrSpecifiers_Checksum_Mismatch.txt new file mode 100644 index 000000000..2560481eb --- /dev/null +++ b/tests/specs/graph/JsrSpecifiers_Checksum_Mismatch.txt @@ -0,0 +1,68 @@ +# https://jsr.io/@scope/a/meta.json +{ + "versions": { + "1.0.0": {} + } +} + +# https://jsr.io/@scope/a/1.0.0_meta.json +{ + "exports": { + ".": "./mod.ts" + }, + "manifest": { + "/mod.ts": { + "size": 1, + "checksum": "sha256-somechecksum" + } + } +} + +# mod.ts +import 'jsr:@scope/a' + +# https://jsr.io/@scope/a/1.0.0/mod.ts +console.log('HI'); + +# output +{ + "roots": [ + "file:///mod.ts" + ], + "modules": [ + { + "kind": "esm", + "dependencies": [ + { + "specifier": "jsr:@scope/a", + "code": { + "specifier": "jsr:@scope/a", + "span": { + "start": { + "line": 0, + "character": 7 + }, + "end": { + "line": 0, + "character": 21 + } + } + } + } + ], + "size": 22, + "mediaType": "TypeScript", + "specifier": "file:///mod.ts" + }, + { + "specifier": "https://jsr.io/@scope/a/1.0.0/mod.ts", + "error": "Integrity check failed.\n\nActual: 6584ea1318267e53901c6b7b83d0d3c1d3e386faa7b93c8e1f7ca1a1ffc76a71\nExpected: somechecksum" + } + ], + "redirects": { + "jsr:@scope/a": "https://jsr.io/@scope/a/1.0.0/mod.ts" + }, + "packages": { + "@scope/a": "@scope/a@1.0.0" + } +} diff --git a/tests/specs/graph/JsrSpecifiers_Checksum_Unsupported.txt b/tests/specs/graph/JsrSpecifiers_Checksum_Unsupported.txt new file mode 100644 index 000000000..5cbce4afa --- /dev/null +++ b/tests/specs/graph/JsrSpecifiers_Checksum_Unsupported.txt @@ -0,0 +1,68 @@ +# https://jsr.io/@scope/a/meta.json +{ + "versions": { + "1.0.0": {} + } +} + +# https://jsr.io/@scope/a/1.0.0_meta.json +{ + "exports": { + ".": "./mod.ts" + }, + "manifest": { + "/mod.ts": { + "size": 1, + "checksum": "sha1-somechecksum" + } + } +} + +# mod.ts +import 'jsr:@scope/a' + +# https://jsr.io/@scope/a/1.0.0/mod.ts +console.log('HI'); + +# output +{ + "roots": [ + "file:///mod.ts" + ], + "modules": [ + { + "kind": "esm", + "dependencies": [ + { + "specifier": "jsr:@scope/a", + "code": { + "specifier": "jsr:@scope/a", + "span": { + "start": { + "line": 0, + "character": 7 + }, + "end": { + "line": 0, + "character": 21 + } + } + } + } + ], + "size": 22, + "mediaType": "TypeScript", + "specifier": "file:///mod.ts" + }, + { + "specifier": "https://jsr.io/@scope/a/1.0.0/mod.ts", + "error": "Unsupported checksum in manifest. Maybe try upgrading deno?" + } + ], + "redirects": { + "jsr:@scope/a": "https://jsr.io/@scope/a/1.0.0/mod.ts" + }, + "packages": { + "@scope/a": "@scope/a@1.0.0" + } +} diff --git a/tests/specs/graph/JsrSpecifiers_InvalidExport.txt b/tests/specs/graph/JsrSpecifiers_InvalidExport.txt index 808145e2d..d7d7b5239 100644 --- a/tests/specs/graph/JsrSpecifiers_InvalidExport.txt +++ b/tests/specs/graph/JsrSpecifiers_InvalidExport.txt @@ -7,6 +7,7 @@ # https://jsr.io/@scope/a/1.0.0_meta.json { + "manifest": {}, "exports": { ".": "./mod.ts", "./sub": "./sub_dir/sub.ts", diff --git a/tests/specs/graph/JsrSpecifiers_WorkspaceNotMatched.txt b/tests/specs/graph/JsrSpecifiers_WorkspaceNotMatched.txt index 5d7ff155c..5f369cee6 100644 --- a/tests/specs/graph/JsrSpecifiers_WorkspaceNotMatched.txt +++ b/tests/specs/graph/JsrSpecifiers_WorkspaceNotMatched.txt @@ -25,6 +25,7 @@ # https://jsr.io/@scope/b/2.0.3_meta.json { + "manifest": {}, "exports": { ".": "./mod.ts" } diff --git a/tests/specs/graph/Jsr_ImportViaHttps.txt b/tests/specs/graph/Jsr_ImportViaHttps.txt new file mode 100644 index 000000000..13cd6e7b4 --- /dev/null +++ b/tests/specs/graph/Jsr_ImportViaHttps.txt @@ -0,0 +1,65 @@ +# https://jsr.io/@scope/a/meta.json +{"versions": { "1.0.0": {} } } + +# https://jsr.io/@scope/a/1.0.0_meta.json +{ + "checksums": { + "mod.ts": { + "size": 21, + "checksum": "151a3a3f4587a29c7b3449a3635fed35c4e88a3a773b3bf296804f1a4e1ab86d" + } + }, + "exports": { + ".": "./mod.ts" + }, + "manifest": { + "/mod.ts": { + "size": 21, + "checksum": "sha256-151a3a3f4587a29c7b3449a3635fed35c4e88a3a773b3bf296804f1a4e1ab86d" + } + } +} +# https://jsr.io/@scope/a/1.0.0/mod.ts +export class Test {} + +# mod.ts +import { Test } from "https://jsr.io/@scope/a/1.0.0/mod.ts"; +console.log(Test); + +# output +{ + "roots": [ + "file:///mod.ts" + ], + "modules": [ + { + "kind": "esm", + "dependencies": [ + { + "specifier": "https://jsr.io/@scope/a/1.0.0/mod.ts", + "code": { + "specifier": "https://jsr.io/@scope/a/1.0.0/mod.ts", + "span": { + "start": { + "line": 0, + "character": 21 + }, + "end": { + "line": 0, + "character": 59 + } + } + } + } + ], + "size": 80, + "mediaType": "TypeScript", + "specifier": "file:///mod.ts" + }, + { + "specifier": "https://jsr.io/@scope/a/1.0.0/mod.ts", + "error": "Importing a JSR package via an HTTPS URL is not implemented. Use a jsr: specifier instead for the time being." + } + ], + "redirects": {} +}