diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 50685aa..a766d10 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,8 +4,8 @@ on: workflow_dispatch: inputs: releaseKind: - description: 'Kind of release' - default: 'minor' + description: "Kind of release" + default: "minor" type: choice options: - patch @@ -36,4 +36,4 @@ jobs: run: | git config user.email "denobot@users.noreply.github.com" git config user.name "denobot" - deno run -A https://raw.githubusercontent.com/denoland/automation/0.14.2/tasks/publish_release.ts --${{github.event.inputs.releaseKind}} deno_cache_dir + deno run -A https://raw.githubusercontent.com/denoland/automation/0.20.0/tasks/publish_release.ts --${{github.event.inputs.releaseKind}} deno_cache_dir diff --git a/Cargo.lock b/Cargo.lock index 2474881..7f982a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base32" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "022dfe9eb35f19ebbcb51e0b40a5ab759f46ad60cadf7297e0bd085afb50e076" + [[package]] name = "bitflags" version = "1.3.2" @@ -89,6 +95,7 @@ checksum = "8d7439c3735f405729d52c3fbbe4de140eaf938a1fe47d227c27f8254d4302a5" name = "deno_cache_dir" version = "0.11.1" dependencies = [ + "base32", "console_error_panic_hook", "deno_media_type", "indexmap", diff --git a/rs_lib/Cargo.toml b/rs_lib/Cargo.toml index cebe97f..c85ddc1 100644 --- a/rs_lib/Cargo.toml +++ b/rs_lib/Cargo.toml @@ -13,6 +13,7 @@ crate-type = ["cdylib", "lib"] wasm = ["console_error_panic_hook", "js-sys", "serde-wasm-bindgen", "wasm-bindgen"] [dependencies] +base32 = "=0.5.1" deno_media_type = "0.1.1" indexmap = { version = "2.0.0", features = ["serde"] } log = "0.4.19" diff --git a/rs_lib/fs.js b/rs_lib/fs.js index 4b622ed..1a66090 100644 --- a/rs_lib/fs.js +++ b/rs_lib/fs.js @@ -10,6 +10,22 @@ export function read_file_bytes(path) { } } +export function canonicalize_path(path) { + try { + return Deno.realPathSync(path); + } catch (err) { + if (err instanceof Deno.errors.NotFound) { + return undefined; + } else { + throw err; + } + } +} + +export function create_dir_all(path) { + Deno.mkdirSync(path, { recursive: true }); +} + export function remove_file(path) { Deno.removeSync(path); } diff --git a/rs_lib/src/cache.rs b/rs_lib/src/cache.rs index 606c776..5423267 100644 --- a/rs_lib/src/cache.rs +++ b/rs_lib/src/cache.rs @@ -1,4 +1,4 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +// Copyright 2018-2024 the Deno authors. MIT license. use serde::Deserialize; use serde::Serialize; diff --git a/rs_lib/src/common.rs b/rs_lib/src/common.rs index 8ae98d9..8133e02 100644 --- a/rs_lib/src/common.rs +++ b/rs_lib/src/common.rs @@ -1,7 +1,8 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +// Copyright 2018-2024 the Deno authors. MIT license. use std::borrow::Cow; use std::collections::HashMap; +use std::path::Path; use url::Url; @@ -48,6 +49,79 @@ pub fn checksum(v: &[u8]) -> String { format!("{:x}", hasher.finalize()) } +pub fn url_from_directory_path(path: &Path) -> Result { + #[cfg(any(unix, windows, target_os = "redox", target_os = "wasi"))] + return Url::from_directory_path(path); + #[cfg(not(any(unix, windows, target_os = "redox", target_os = "wasi")))] + url_from_directory_path_wasm(path) +} + +#[cfg(any( + test, + not(any(unix, windows, target_os = "redox", target_os = "wasi")) +))] +fn url_from_directory_path_wasm(path: &Path) -> Result { + let mut url = url_from_file_path_wasm(path)?; + url.path_segments_mut().unwrap().push(""); + Ok(url) +} + +#[cfg(any( + test, + not(any(unix, windows, target_os = "redox", target_os = "wasi")) +))] +fn url_from_file_path_wasm(path: &Path) -> Result { + use std::path::Component; + + let original_path = path.to_string_lossy(); + let mut path_str = original_path; + // assume paths containing backslashes are windows paths + if path_str.contains('\\') { + let mut url = Url::parse("file://").unwrap(); + if let Some(next) = path_str.strip_prefix(r#"\\?\UNC\"#) { + if let Some((host, rest)) = next.split_once('\\') { + if url.set_host(Some(host)).is_ok() { + path_str = rest.to_string().into(); + } + } + } else if let Some(next) = path_str.strip_prefix(r#"\\?\"#) { + path_str = next.to_string().into(); + } else if let Some(next) = path_str.strip_prefix(r#"\\"#) { + if let Some((host, rest)) = next.split_once('\\') { + if url.set_host(Some(host)).is_ok() { + path_str = rest.to_string().into(); + } + } + } + + for component in path_str.split('\\') { + url.path_segments_mut().unwrap().push(component); + } + + Ok(url) + } else { + let mut url = Url::parse("file://").unwrap(); + for component in path.components() { + match component { + Component::RootDir => { + url.path_segments_mut().unwrap().push(""); + } + Component::Normal(segment) => { + url + .path_segments_mut() + .unwrap() + .push(&segment.to_string_lossy()); + } + Component::Prefix(_) | Component::CurDir | Component::ParentDir => { + return Err(()); + } + } + } + + Ok(url) + } +} + #[cfg(test)] mod tests { use super::*; @@ -60,4 +134,55 @@ mod tests { "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" ); } + + #[test] + fn test_url_from_file_path_wasm() { + #[track_caller] + fn convert(path: &str) -> String { + url_from_file_path_wasm(Path::new(path)) + .unwrap() + .to_string() + } + + assert_eq!(convert("/a/b/c.json"), "file:///a/b/c.json"); + assert_eq!( + convert("D:\\test\\other.json"), + "file:///D:/test/other.json" + ); + assert_eq!( + convert("/path with spaces/and#special%chars!.json"), + "file:///path%20with%20spaces/and%23special%25chars!.json" + ); + assert_eq!( + convert("C:\\My Documents\\file.txt"), + "file:///C:/My%20Documents/file.txt" + ); + assert_eq!( + convert("/a/b/пример.txt"), + "file:///a/b/%D0%BF%D1%80%D0%B8%D0%BC%D0%B5%D1%80.txt" + ); + assert_eq!( + convert("\\\\server\\share\\folder\\file.txt"), + "file://server/share/folder/file.txt" + ); + assert_eq!(convert(r#"\\?\UNC\server\share"#), "file://server/share"); + assert_eq!( + convert(r"\\?\cat_pics\subfolder\file.jpg"), + "file:///cat_pics/subfolder/file.jpg" + ); + assert_eq!(convert(r"\\?\cat_pics"), "file:///cat_pics"); + } + + #[test] + fn test_url_from_directory_path_wasm() { + #[track_caller] + fn convert(path: &str) -> String { + url_from_directory_path_wasm(Path::new(path)) + .unwrap() + .to_string() + } + + assert_eq!(convert("/a/b/c"), "file:///a/b/c/"); + assert_eq!(convert("D:\\test\\other"), "file:///D:/test/other/"); + } } diff --git a/rs_lib/src/env.rs b/rs_lib/src/env.rs index 25e0c84..382d61a 100644 --- a/rs_lib/src/env.rs +++ b/rs_lib/src/env.rs @@ -1,12 +1,15 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +// Copyright 2018-2024 the Deno authors. MIT license. use std::path::Path; +use std::path::PathBuf; use std::time::SystemTime; pub trait DenoCacheEnv: Send + Sync + std::fmt::Debug + Clone { fn read_file_bytes(&self, path: &Path) -> std::io::Result>; fn atomic_write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()>; + fn canonicalize_path(&self, path: &Path) -> std::io::Result; + fn create_dir_all(&self, path: &Path) -> std::io::Result<()>; fn remove_file(&self, path: &Path) -> std::io::Result<()>; fn modified(&self, path: &Path) -> std::io::Result>; fn is_file(&self, path: &Path) -> bool; @@ -49,6 +52,14 @@ mod test_fs { } } + fn canonicalize_path(&self, path: &Path) -> std::io::Result { + path.canonicalize() + } + + fn create_dir_all(&self, path: &Path) -> std::io::Result<()> { + std::fs::create_dir_all(path) + } + fn remove_file(&self, path: &Path) -> std::io::Result<()> { std::fs::remove_file(path) } diff --git a/rs_lib/src/global/cache_file.rs b/rs_lib/src/global/cache_file.rs index d486d5e..0fce5d3 100644 --- a/rs_lib/src/global/cache_file.rs +++ b/rs_lib/src/global/cache_file.rs @@ -1,4 +1,4 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +// Copyright 2018-2024 the Deno authors. MIT license. use std::io::ErrorKind; use std::path::Path; diff --git a/rs_lib/src/global/mod.rs b/rs_lib/src/global/mod.rs index 046b334..26ccea9 100644 --- a/rs_lib/src/global/mod.rs +++ b/rs_lib/src/global/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +// Copyright 2018-2024 the Deno authors. MIT license. use std::path::Path; use std::path::PathBuf; diff --git a/rs_lib/src/lib.rs b/rs_lib/src/lib.rs index e882889..b478cb9 100644 --- a/rs_lib/src/lib.rs +++ b/rs_lib/src/lib.rs @@ -1,10 +1,11 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +// Copyright 2018-2024 the Deno authors. MIT license. mod cache; mod common; mod env; mod global; mod local; +pub mod npm; pub use cache::url_to_filename; pub use cache::CacheEntry; @@ -55,6 +56,10 @@ pub mod wasm { #[wasm_bindgen(catch)] fn atomic_write_file(path: &str, bytes: &[u8]) -> Result; #[wasm_bindgen(catch)] + fn canonicalize_path(path: &str) -> Result; + #[wasm_bindgen(catch)] + fn create_dir_all(path: &str) -> Result<(), JsValue>; + #[wasm_bindgen(catch)] fn modified_time(path: &str) -> Result, JsValue>; fn is_file(path: &str) -> bool; fn time_now() -> usize; @@ -84,6 +89,20 @@ pub mod wasm { Ok(()) } + fn canonicalize_path(&self, path: &Path) -> std::io::Result { + let js_value = + canonicalize_path(&path.to_string_lossy()).map_err(js_to_io_error)?; + if js_value.is_null() || js_value.is_undefined() { + Err(std::io::Error::new(ErrorKind::NotFound, "")) + } else { + Ok(PathBuf::from(js_value.as_string().unwrap())) + } + } + + fn create_dir_all(&self, path: &Path) -> std::io::Result<()> { + create_dir_all(&path.to_string_lossy()).map_err(js_to_io_error) + } + fn remove_file(&self, path: &Path) -> std::io::Result<()> { remove_file(&path.to_string_lossy()).map_err(js_to_io_error) } diff --git a/rs_lib/src/local.rs b/rs_lib/src/local.rs index 54ce3f6..5ff48f6 100644 --- a/rs_lib/src/local.rs +++ b/rs_lib/src/local.rs @@ -1,4 +1,4 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +// Copyright 2018-2024 the Deno authors. MIT license. use std::borrow::Cow; use std::collections::BTreeMap; diff --git a/rs_lib/src/npm.rs b/rs_lib/src/npm.rs new file mode 100644 index 0000000..618289e --- /dev/null +++ b/rs_lib/src/npm.rs @@ -0,0 +1,306 @@ +// Copyright 2018-2024 the Deno authors. MIT license. + +use std::io::ErrorKind; +use std::path::Path; +use std::path::PathBuf; + +use url::Url; + +use crate::common::url_from_directory_path; +use crate::DenoCacheEnv; + +pub struct NpmCacheFolderId { + /// Package name. + pub name: String, + /// Package version. + pub version: String, + /// Package copy index. + pub copy_index: u8, +} + +/// The global cache directory of npm packages. +#[derive(Clone, Debug)] +pub struct NpmCacheDir { + root_dir: PathBuf, + // cached url representation of the root directory + root_dir_url: Url, + // A list of all registry that were discovered via `.npmrc` files + // turned into a safe directory names. + known_registries_dirnames: Vec, +} + +impl NpmCacheDir { + pub fn new( + env: &Env, + root_dir: PathBuf, + known_registries_urls: Vec, + ) -> Self { + fn try_get_canonicalized_root_dir( + env: &Env, + root_dir: &Path, + ) -> Result { + match env.canonicalize_path(root_dir) { + Ok(path) => Ok(path), + Err(err) if err.kind() == ErrorKind::NotFound => { + env.create_dir_all(root_dir)?; + env.canonicalize_path(root_dir) + } + Err(err) => Err(err), + } + } + + // this may fail on readonly file systems, so just ignore if so + let root_dir = + try_get_canonicalized_root_dir(env, &root_dir).unwrap_or(root_dir); + let root_dir_url = url_from_directory_path(&root_dir).unwrap(); + + let known_registries_dirnames: Vec<_> = known_registries_urls + .into_iter() + .map(|url| { + root_url_to_safe_local_dirname(&url) + .to_string_lossy() + .replace('\\', "/") + }) + .collect(); + + Self { + root_dir, + root_dir_url, + known_registries_dirnames, + } + } + + pub fn root_dir(&self) -> &Path { + &self.root_dir + } + + pub fn root_dir_url(&self) -> &Url { + &self.root_dir_url + } + + pub fn package_folder_for_id( + &self, + package_name: &str, + package_version: &str, + package_copy_index: u8, + registry_url: &Url, + ) -> PathBuf { + if package_copy_index == 0 { + self + .package_name_folder(package_name, registry_url) + .join(package_version) + } else { + self + .package_name_folder(package_name, registry_url) + .join(format!("{}_{}", package_version, package_copy_index)) + } + } + + pub fn package_name_folder(&self, name: &str, registry_url: &Url) -> PathBuf { + let mut dir = self.registry_folder(registry_url); + if name.to_lowercase() != name { + let encoded_name = mixed_case_package_name_encode(name); + // Using the encoded directory may have a collision with an actual package name + // so prefix it with an underscore since npm packages can't start with that + dir.join(format!("_{encoded_name}")) + } else { + // ensure backslashes are used on windows + for part in name.split('/') { + dir = dir.join(part); + } + dir + } + } + + fn registry_folder(&self, registry_url: &Url) -> PathBuf { + self + .root_dir + .join(root_url_to_safe_local_dirname(registry_url)) + } + + pub fn resolve_package_folder_id_from_specifier( + &self, + specifier: &Url, + ) -> Option { + let mut maybe_relative_url = None; + + // Iterate through known registries and try to get a match. + for registry_dirname in &self.known_registries_dirnames { + let registry_root_dir = self + .root_dir_url + .join(&format!("{}/", registry_dirname)) + // this not succeeding indicates a fatal issue, so unwrap + .unwrap(); + + let Some(relative_url) = registry_root_dir.make_relative(specifier) + else { + continue; + }; + + if relative_url.starts_with("../") { + continue; + } + + maybe_relative_url = Some(relative_url); + break; + } + + let mut relative_url = maybe_relative_url?; + + // base32 decode the url if it starts with an underscore + // * Ex. _{base32(package_name)}/ + if let Some(end_url) = relative_url.strip_prefix('_') { + let mut parts = end_url + .split('/') + .map(ToOwned::to_owned) + .collect::>(); + match mixed_case_package_name_decode(&parts[0]) { + Some(part) => { + parts[0] = part; + } + None => return None, + } + relative_url = parts.join("/"); + } + + // examples: + // * chalk/5.0.1/ + // * @types/chalk/5.0.1/ + // * some-package/5.0.1_1/ -- where the `_1` (/_\d+/) is a copy of the folder for peer deps + let is_scoped_package = relative_url.starts_with('@'); + let mut parts = relative_url + .split('/') + .enumerate() + .take(if is_scoped_package { 3 } else { 2 }) + .map(|(_, part)| part) + .collect::>(); + if parts.len() < 2 { + return None; + } + let version_part = parts.pop().unwrap(); + let name = parts.join("/"); + let (version, copy_index) = + if let Some((version, copy_count)) = version_part.split_once('_') { + (version, copy_count.parse::().ok()?) + } else { + (version_part, 0) + }; + Some(NpmCacheFolderId { + name, + version: version.to_string(), + copy_index, + }) + } + + pub fn get_cache_location(&self) -> PathBuf { + self.root_dir.clone() + } +} + +pub fn mixed_case_package_name_encode(name: &str) -> String { + // use base32 encoding because it's reversible and the character set + // only includes the characters within 0-9 and A-Z so it can be lower cased + base32::encode( + base32::Alphabet::Rfc4648Lower { padding: false }, + name.as_bytes(), + ) + .to_lowercase() +} + +pub fn mixed_case_package_name_decode(name: &str) -> Option { + base32::decode(base32::Alphabet::Rfc4648Lower { padding: false }, name) + .and_then(|b| String::from_utf8(b).ok()) +} + +/// Gets a safe local directory name for the provided url. +/// +/// For example: +/// https://deno.land:8080/path -> deno.land_8080/path +fn root_url_to_safe_local_dirname(root: &Url) -> PathBuf { + fn sanitize_segment(text: &str) -> String { + text + .chars() + .map(|c| if is_banned_segment_char(c) { '_' } else { c }) + .collect() + } + + fn is_banned_segment_char(c: char) -> bool { + matches!(c, '/' | '\\') || is_banned_path_char(c) + } + + let mut result = String::new(); + if let Some(domain) = root.domain() { + result.push_str(&sanitize_segment(domain)); + } + if let Some(port) = root.port() { + if !result.is_empty() { + result.push('_'); + } + result.push_str(&port.to_string()); + } + let mut result = PathBuf::from(result); + if let Some(segments) = root.path_segments() { + for segment in segments.filter(|s| !s.is_empty()) { + result = result.join(sanitize_segment(segment)); + } + } + + result +} + +/// Gets if the provided character is not supported on all +/// kinds of file systems. +fn is_banned_path_char(c: char) -> bool { + matches!(c, '<' | '>' | ':' | '"' | '|' | '?' | '*') +} + +#[cfg(test)] +mod test { + use url::Url; + + use crate::TestRealDenoCacheEnv; + + use super::NpmCacheDir; + + #[test] + fn should_get_package_folder() { + #[allow(clippy::disallowed_methods)] + let root_dir = std::env::current_dir().unwrap().canonicalize().unwrap(); + let registry_url = Url::parse("https://registry.npmjs.org/").unwrap(); + let env = TestRealDenoCacheEnv; + let cache = + NpmCacheDir::new(&env, root_dir.clone(), vec![registry_url.clone()]); + + assert_eq!( + cache.package_folder_for_id("json", "1.2.5", 0, ®istry_url,), + root_dir + .join("registry.npmjs.org") + .join("json") + .join("1.2.5"), + ); + + assert_eq!( + cache.package_folder_for_id("json", "1.2.5", 1, ®istry_url,), + root_dir + .join("registry.npmjs.org") + .join("json") + .join("1.2.5_1"), + ); + + assert_eq!( + cache.package_folder_for_id("JSON", "2.1.5", 0, ®istry_url,), + root_dir + .join("registry.npmjs.org") + .join("_jjju6tq") + .join("2.1.5"), + ); + + assert_eq!( + cache.package_folder_for_id("@types/JSON", "2.1.5", 0, ®istry_url,), + root_dir + .join("registry.npmjs.org") + .join("_ib2hs4dfomxuuu2pjy") + .join("2.1.5"), + ); + } +} diff --git a/rs_lib/tests/integration_test.rs b/rs_lib/tests/integration_test.rs index fb79b80..0dec5c1 100644 --- a/rs_lib/tests/integration_test.rs +++ b/rs_lib/tests/integration_test.rs @@ -1,3 +1,7 @@ +// Copyright 2018-2024 the Deno authors. MIT license. + +#![allow(clippy::disallowed_methods)] + use std::collections::HashMap; use std::path::Path; use std::sync::Arc; diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 52cb204..95c5b61 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.80.1" +channel = "1.81.0" components = [ "clippy", "rustfmt" ]