From 3c6d458440c3d134b5279722e278a0e007d66f7c Mon Sep 17 00:00:00 2001 From: Michael Henry Date: Fri, 22 May 2020 11:15:24 -0400 Subject: [PATCH 1/2] Support `{prefix}` and `{lowerprefix}` markers in `config.json` `dl` key. These new markers allow Cargo to supply a directory name (similar to that used in crates.io-index) as part of a crate's download URL, enabling simpler hosting of crates. Previously, a `file` URL would need to put all crates into a single huge directory (such as `/srv/crates/`), e.g.: file:///srv/crates/{crate}/{crate}-{version}.crate With the `{prefix}` marker, a more efficient directory structure may be used, e.g.: file:///srv/crates/{prefix}/{crate}/{crate}-{version}.crate An example crate of `cargo-0.44.1.crate` would map to the path: /srv/crates/ca/rg/cargo/cargo-0.44.1.crate --- src/cargo/sources/registry/mod.rs | 12 ++++++--- src/cargo/sources/registry/remote.rs | 40 +++++++++++++++++++++++++--- src/doc/src/reference/registries.md | 22 +++++++++++++-- 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/src/cargo/sources/registry/mod.rs b/src/cargo/sources/registry/mod.rs index d49b46318e9..44328b29fbd 100644 --- a/src/cargo/sources/registry/mod.rs +++ b/src/cargo/sources/registry/mod.rs @@ -185,6 +185,8 @@ pub const CRATES_IO_INDEX: &str = "https://github.com/rust-lang/crates.io-index" pub const CRATES_IO_REGISTRY: &str = "crates-io"; const CRATE_TEMPLATE: &str = "{crate}"; const VERSION_TEMPLATE: &str = "{version}"; +const PREFIX_TEMPLATE: &str = "{prefix}"; +const LOWER_PREFIX_TEMPLATE: &str = "{lowerprefix}"; pub struct RegistrySource<'cfg> { source_id: SourceId, @@ -203,10 +205,14 @@ pub struct RegistryConfig { /// The string is a template which will generate the download URL for the /// tarball of a specific version of a crate. The substrings `{crate}` and /// `{version}` will be replaced with the crate's name and version - /// respectively. + /// respectively. The substring `{prefix}` will be replaced with the + /// crate's prefix directory name, and the substring `{lowerprefix}` will + /// be replaced with the crate's prefix directory name converted to + /// lowercase. /// - /// For backwards compatibility, if the string does not contain `{crate}` or - /// `{version}`, it will be extended with `/{crate}/{version}/download` to + /// For backwards compatibility, if the string does not contain any + /// markers (`{crate}`, `{version}`, `{prefix}`, or ``{lowerprefix}`), it + /// will be extended with `/{crate}/{version}/download` to /// support registries like crates.io which were created before the /// templating setup was created. pub dl: String, diff --git a/src/cargo/sources/registry/remote.rs b/src/cargo/sources/registry/remote.rs index 6dc4db00cc0..79c4a60d586 100644 --- a/src/cargo/sources/registry/remote.rs +++ b/src/cargo/sources/registry/remote.rs @@ -1,7 +1,10 @@ use crate::core::{InternedString, PackageId, SourceId}; use crate::sources::git; use crate::sources::registry::MaybeLock; -use crate::sources::registry::{RegistryConfig, RegistryData, CRATE_TEMPLATE, VERSION_TEMPLATE}; +use crate::sources::registry::{ + RegistryConfig, RegistryData, CRATE_TEMPLATE, LOWER_PREFIX_TEMPLATE, PREFIX_TEMPLATE, + VERSION_TEMPLATE, +}; use crate::util::errors::{CargoResult, CargoResultExt}; use crate::util::paths; use crate::util::{Config, Filesystem, Sha256}; @@ -16,6 +19,15 @@ use std::mem; use std::path::Path; use std::str; +fn make_crate_prefix(name: &str) -> String { + match name.len() { + 1 => format!("1"), + 2 => format!("2"), + 3 => format!("3/{}", &name[..1]), + _ => format!("{}/{}", &name[0..2], &name[2..4]), + } +} + pub struct RemoteRegistry<'cfg> { index_path: Filesystem, cache_path: Filesystem, @@ -250,12 +262,19 @@ impl<'cfg> RegistryData for RemoteRegistry<'cfg> { let config = self.config()?.unwrap(); let mut url = config.dl; - if !url.contains(CRATE_TEMPLATE) && !url.contains(VERSION_TEMPLATE) { + if !url.contains(CRATE_TEMPLATE) + && !url.contains(VERSION_TEMPLATE) + && !url.contains(PREFIX_TEMPLATE) + && !url.contains(LOWER_PREFIX_TEMPLATE) + { write!(url, "/{}/{}/download", CRATE_TEMPLATE, VERSION_TEMPLATE).unwrap(); } + let prefix = make_crate_prefix(&*pkg.name()); let url = url .replace(CRATE_TEMPLATE, &*pkg.name()) - .replace(VERSION_TEMPLATE, &pkg.version().to_string()); + .replace(VERSION_TEMPLATE, &pkg.version().to_string()) + .replace(PREFIX_TEMPLATE, &prefix) + .replace(LOWER_PREFIX_TEMPLATE, &prefix.to_lowercase()); Ok(MaybeLock::Download { url, @@ -314,3 +333,18 @@ impl<'cfg> Drop for RemoteRegistry<'cfg> { self.tree.borrow_mut().take(); } } + +#[cfg(test)] +mod tests { + use super::make_crate_prefix; + + #[test] + fn crate_prefix() { + assert_eq!(make_crate_prefix("a"), "1"); + assert_eq!(make_crate_prefix("ab"), "2"); + assert_eq!(make_crate_prefix("abc"), "3/a"); + assert_eq!(make_crate_prefix("Abc"), "3/A"); + assert_eq!(make_crate_prefix("AbCd"), "Ab/Cd"); + assert_eq!(make_crate_prefix("aBcDe"), "aB/cD"); + } +} diff --git a/src/doc/src/reference/registries.md b/src/doc/src/reference/registries.md index 33d4955bbe0..f0148ca67ee 100644 --- a/src/doc/src/reference/registries.md +++ b/src/doc/src/reference/registries.md @@ -128,8 +128,11 @@ looks like: The keys are: - `dl`: This is the URL for downloading crates listed in the index. The value may have the markers `{crate}` and `{version}` which are replaced with the - name and version of the crate to download. If the markers are not present, - then the value `/{crate}/{version}/download` is appended to the end. + name and version of the crate to download, or the marker `{prefix}` which is + replaced with the crate's prefix, or the marker `{lowerprefix}` which is + replaced with the crate's prefix converted to lowercase. If none of the + markers are present, then the value `/{crate}/{version}/download` is appended + to the end. See below for more about crate prefixes. - `api`: This is the base URL for the web API. This key is optional, but if it is not specified, commands such as [`cargo publish`] will not work. The web API is described below. @@ -159,6 +162,21 @@ directories: > package names in `Cargo.toml` and the index JSON data are case-sensitive and > may contain upper and lower case characters. +The directory name above is calculated based on the package name converted to +lowercase; it is represented by the marker `{lowerprefix}`. When the original +package name is used without case conversion, the resulting directory name is +represented by the marker `{prefix}`. For example, the package `MyCrate` would +have a `{prefix}` of `My/Cr` and a `{lowerprefix}` of `my/cr`. In general, +using `{prefix}` is recommended over `{lowerprefix}`, but there are pros and +cons to each choice. Using `{prefix}` on case-insensitive filesystems results +in (harmless-but-inelegant) directory aliasing. For example, `crate` and +`CrateTwo` have `{prefix}` values of `cr/at` and `Cr/at`; these are distinct on +Unix machines but alias to the same directory on Windows. Using directories +with normalized case avoids aliasing, but on case-sensitive filesystems it's +harder to suport older versions of Cargo that lack `{prefix}`/`{lowerprefix}`. +For example, nginx rewrite rules can easily construct `{prefix}` but can't +perform case-conversion to construct `{lowerprefix}`. + Registries should consider enforcing limitations on package names added to their index. Cargo itself allows names with any [alphanumeric], `-`, or `_` characters. [crates.io] imposes its own limitations, including the following: From b375bea147edda469b3442b9e89390180a442256 Mon Sep 17 00:00:00 2001 From: Michael Henry Date: Thu, 28 May 2020 16:24:16 -0400 Subject: [PATCH 2/2] Improve the wording of `dl` documentation. --- src/doc/src/reference/registries.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/doc/src/reference/registries.md b/src/doc/src/reference/registries.md index f0148ca67ee..6b49cd0dc05 100644 --- a/src/doc/src/reference/registries.md +++ b/src/doc/src/reference/registries.md @@ -127,12 +127,17 @@ looks like: The keys are: - `dl`: This is the URL for downloading crates listed in the index. The value - may have the markers `{crate}` and `{version}` which are replaced with the - name and version of the crate to download, or the marker `{prefix}` which is - replaced with the crate's prefix, or the marker `{lowerprefix}` which is - replaced with the crate's prefix converted to lowercase. If none of the - markers are present, then the value `/{crate}/{version}/download` is appended - to the end. See below for more about crate prefixes. + may have the following markers which will be replaced with their + corresponding value: + + - `{crate}`: The name of crate. + - `{version}`: The crate version. + - `{prefix}`: A directory prefix computed from the crate name. For example, + a crate named `cargo` has a prefix of `ca/rg`. See below for details. + - `{lowerprefix}`: Lowercase variant of `{prefix}`. + + If none of the markers are present, then the value + `/{crate}/{version}/download` is appended to the end. - `api`: This is the base URL for the web API. This key is optional, but if it is not specified, commands such as [`cargo publish`] will not work. The web API is described below.