Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add local registry overlays #13926

Merged
merged 3 commits into from
Jun 11, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions crates/cargo-test-support/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -877,6 +877,17 @@ impl Execs {
self
}

pub fn overlay_registry(&mut self, url: &Url, path: &str) -> &mut Self {
if let Some(ref mut p) = self.process_builder {
let env_value = format!("{}={}", url, path);
p.env(
"__CARGO_TEST_DEPENDENCY_CONFUSION_VULNERABILITY_DO_NOT_USE_THIS",
env_value,
);
}
self
}

pub fn enable_split_debuginfo_packed(&mut self) -> &mut Self {
self.env("CARGO_PROFILE_DEV_SPLIT_DEBUGINFO", "packed")
.env("CARGO_PROFILE_TEST_SPLIT_DEBUGINFO", "packed")
Expand Down
7 changes: 6 additions & 1 deletion crates/cargo-test-support/src/registry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1649,7 +1649,12 @@ impl Package {
/// Returns the path to the compressed package file.
pub fn archive_dst(&self) -> PathBuf {
if self.local {
registry_path().join(format!("{}-{}.crate", self.name, self.vers))
let path = if self.alternative {
jneem marked this conversation as resolved.
Show resolved Hide resolved
alt_registry_path()
} else {
registry_path()
};
path.join(format!("{}-{}.crate", self.name, self.vers))
} else if self.alternative {
alt_dl_path()
.join(&self.name)
Expand Down
8 changes: 4 additions & 4 deletions crates/xtask-bump-check/src/xtask.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ use std::fs;
use std::task;

use cargo::core::dependency::Dependency;
use cargo::core::registry::PackageRegistry;
use cargo::core::Package;
use cargo::core::Registry;
use cargo::core::SourceId;
Expand Down Expand Up @@ -137,7 +136,7 @@ fn bump_check(args: &clap::ArgMatches, gctx: &cargo::util::GlobalContext) -> Car

let mut needs_bump = Vec::new();

check_crates_io(gctx, &changed_members, &mut needs_bump)?;
check_crates_io(&ws, &changed_members, &mut needs_bump)?;

if let Some(referenced_commit) = referenced_commit.as_ref() {
status(&format!("compare against `{}`", referenced_commit.id()))?;
Expand Down Expand Up @@ -385,12 +384,13 @@ fn symmetric_diff<'a>(
///
/// Assumption: We always release a version larger than all existing versions.
fn check_crates_io<'a>(
gctx: &GlobalContext,
ws: &Workspace<'a>,
changed_members: &HashMap<&'a str, &'a Package>,
needs_bump: &mut Vec<&'a Package>,
) -> CargoResult<()> {
let gctx = ws.gctx();
let source_id = SourceId::crates_io(gctx)?;
let mut registry = PackageRegistry::new(gctx)?;
let mut registry = ws.package_registry()?;
let _lock = gctx.acquire_package_cache_lock(CacheLockMode::DownloadExclusive)?;
registry.lock_patches();
gctx.shell().status(
Expand Down
6 changes: 4 additions & 2 deletions src/cargo/core/registry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -194,8 +194,10 @@ pub struct LockedPatchDependency {
}

impl<'gctx> PackageRegistry<'gctx> {
pub fn new(gctx: &'gctx GlobalContext) -> CargoResult<PackageRegistry<'gctx>> {
let source_config = SourceConfigMap::new(gctx)?;
pub fn new_with_source_config(
gctx: &'gctx GlobalContext,
source_config: SourceConfigMap<'gctx>,
) -> CargoResult<PackageRegistry<'gctx>> {
Ok(PackageRegistry {
gctx,
sources: SourceMap::new(),
Expand Down
44 changes: 43 additions & 1 deletion src/cargo/core/workspace.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ use crate::core::{
};
use crate::core::{EitherManifest, Package, SourceId, VirtualManifest};
use crate::ops;
use crate::sources::{PathSource, CRATES_IO_INDEX, CRATES_IO_REGISTRY};
use crate::sources::{PathSource, SourceConfigMap, CRATES_IO_INDEX, CRATES_IO_REGISTRY};
use crate::util::edit_distance;
use crate::util::errors::{CargoResult, ManifestError};
use crate::util::interning::InternedString;
Expand Down Expand Up @@ -109,6 +109,9 @@ pub struct Workspace<'gctx> {

/// Workspace-level custom metadata
custom_metadata: Option<toml::Value>,

/// Local overlay configuration. See [`crate::sources::overlay`].
local_overlays: HashMap<SourceId, PathBuf>,
}

// Separate structure for tracking loaded packages (to avoid loading anything
Expand Down Expand Up @@ -237,6 +240,7 @@ impl<'gctx> Workspace<'gctx> {
resolve_behavior: ResolveBehavior::V1,
resolve_honors_rust_version: false,
custom_metadata: None,
local_overlays: HashMap::new(),
}
}

Expand Down Expand Up @@ -1674,6 +1678,44 @@ impl<'gctx> Workspace<'gctx> {
// Cargo to panic, see issue #10545.
self.is_member(&unit.pkg) && !(unit.target.for_host() || unit.pkg.proc_macro())
}

/// Adds a local package registry overlaying a `SourceId`.
///
/// See [`crate::sources::overlay::DependencyConfusionThreatOverlaySource`] for why you shouldn't use this.
pub fn add_local_overlay(&mut self, id: SourceId, registry_path: PathBuf) {
self.local_overlays.insert(id, registry_path);
}

/// Builds a package registry that reflects this workspace configuration.
pub fn package_registry(&self) -> CargoResult<PackageRegistry<'gctx>> {
let source_config =
SourceConfigMap::new_with_overlays(self.gctx(), self.local_overlays()?)?;
PackageRegistry::new_with_source_config(self.gctx(), source_config)
}

/// Returns all the configured local overlays, including the ones from our secret environment variable.
fn local_overlays(&self) -> CargoResult<impl Iterator<Item = (SourceId, SourceId)>> {
let mut ret = self
.local_overlays
.iter()
.map(|(id, path)| Ok((*id, SourceId::for_local_registry(path)?)))
.collect::<CargoResult<Vec<_>>>()?;

if let Ok(overlay) = self
.gctx
.get_env("__CARGO_TEST_DEPENDENCY_CONFUSION_VULNERABILITY_DO_NOT_USE_THIS")
{
let (url, path) = overlay.split_once('=').ok_or(anyhow::anyhow!(
"invalid overlay format. I won't tell you why; you shouldn't be using it anyway"
))?;
ret.push((
SourceId::from_url(url)?,
SourceId::for_local_registry(path.as_ref())?,
));
}

Ok(ret.into_iter())
}
}

impl<'gctx> Packages<'gctx> {
Expand Down
2 changes: 1 addition & 1 deletion src/cargo/ops/cargo_add/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ pub fn add(workspace: &Workspace<'_>, options: &AddOptions<'_>) -> CargoResult<(
);
}

let mut registry = PackageRegistry::new(options.gctx)?;
let mut registry = workspace.package_registry()?;

let deps = {
let _lock = options
Expand Down
4 changes: 2 additions & 2 deletions src/cargo/ops/cargo_package.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use std::task::Poll;
use crate::core::compiler::{BuildConfig, CompileMode, DefaultExecutor, Executor};
use crate::core::manifest::Target;
use crate::core::resolver::CliFeatures;
use crate::core::{registry::PackageRegistry, resolver::HasDevUnits};
use crate::core::resolver::HasDevUnits;
use crate::core::{Feature, PackageIdSpecQuery, Shell, Verbosity, Workspace};
use crate::core::{Package, PackageId, PackageSet, Resolve, SourceId};
use crate::sources::PathSource;
Expand Down Expand Up @@ -472,7 +472,7 @@ fn build_lock(ws: &Workspace<'_>, publish_pkg: &Package) -> CargoResult<String>
let orig_resolve = ops::load_pkg_lockfile(ws)?;

let tmp_ws = Workspace::ephemeral(publish_pkg.clone(), ws.gctx(), None, true)?;
let mut tmp_reg = PackageRegistry::new(ws.gctx())?;
let mut tmp_reg = ws.package_registry()?;
let mut new_resolve = ops::resolve_with_previous(
&mut tmp_reg,
&tmp_ws,
Expand Down
8 changes: 4 additions & 4 deletions src/cargo/ops/cargo_update.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ pub struct UpdateOptions<'a> {
}

pub fn generate_lockfile(ws: &Workspace<'_>) -> CargoResult<()> {
let mut registry = PackageRegistry::new(ws.gctx())?;
let mut registry = ws.package_registry()?;
let previous_resolve = None;
let mut resolve = ops::resolve_with_previous(
&mut registry,
Expand Down Expand Up @@ -73,7 +73,7 @@ pub fn update_lockfile(ws: &Workspace<'_>, opts: &UpdateOptions<'_>) -> CargoRes
// Precise option specified, so calculate a previous_resolve required
// by precise package update later.
Some(_) => {
let mut registry = PackageRegistry::new(opts.gctx)?;
let mut registry = ws.package_registry()?;
ops::resolve_with_previous(
&mut registry,
ws,
Expand All @@ -88,7 +88,7 @@ pub fn update_lockfile(ws: &Workspace<'_>, opts: &UpdateOptions<'_>) -> CargoRes
}
}
};
let mut registry = PackageRegistry::new(opts.gctx)?;
let mut registry = ws.package_registry()?;
let mut to_avoid = HashSet::new();

if opts.to_update.is_empty() {
Expand Down Expand Up @@ -226,7 +226,7 @@ pub fn upgrade_manifests(
// that we're synchronized against other Cargos.
let _lock = gctx.acquire_package_cache_lock(CacheLockMode::DownloadExclusive)?;

let mut registry = PackageRegistry::new(gctx)?;
let mut registry = ws.package_registry()?;
registry.lock_patches();

for member in ws.members_mut().sorted() {
Expand Down
4 changes: 2 additions & 2 deletions src/cargo/ops/resolve.rs
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ version. This may also occur with an optional dependency that is not enabled.";
/// This is a simple interface used by commands like `clean`, `fetch`, and
/// `package`, which don't specify any options or features.
pub fn resolve_ws<'a>(ws: &Workspace<'a>, dry_run: bool) -> CargoResult<(PackageSet<'a>, Resolve)> {
let mut registry = PackageRegistry::new(ws.gctx())?;
let mut registry = ws.package_registry()?;
let resolve = resolve_with_registry(ws, &mut registry, dry_run)?;
let packages = get_resolved_packages(&resolve, registry)?;
Ok((packages, resolve))
Expand All @@ -142,7 +142,7 @@ pub fn resolve_ws_with_opts<'gctx>(
force_all_targets: ForceAllTargets,
dry_run: bool,
) -> CargoResult<WorkspaceResolve<'gctx>> {
let mut registry = PackageRegistry::new(ws.gctx())?;
let mut registry = ws.package_registry()?;
let (resolve, resolved_with_overrides) = if ws.ignore_lock() {
let add_patches = true;
let resolve = None;
Expand Down
41 changes: 37 additions & 4 deletions src/cargo/sources/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
//! sources to one another via the `replace-with` key in `.cargo/config`.

use crate::core::{GitReference, PackageId, SourceId};
use crate::sources::overlay::DependencyConfusionThreatOverlaySource;
use crate::sources::source::Source;
use crate::sources::{ReplacedSource, CRATES_IO_REGISTRY};
use crate::util::context::{self, ConfigRelativePath, OptValue};
Expand All @@ -24,6 +25,8 @@ pub struct SourceConfigMap<'gctx> {
cfgs: HashMap<String, SourceConfig>,
/// Mapping of [`SourceId`] to the source name.
id2name: HashMap<SourceId, String>,
/// Mapping of sources to local registries that will be overlaid on them.
overlays: HashMap<SourceId, SourceId>,
gctx: &'gctx GlobalContext,
}

Expand Down Expand Up @@ -81,6 +84,18 @@ impl<'gctx> SourceConfigMap<'gctx> {
base.add_config(key, value)?;
}
}

Ok(base)
}

/// Like [`SourceConfigMap::new`] but includes sources from source
/// replacement configurations.
pub fn new_with_overlays(
gctx: &'gctx GlobalContext,
overlays: impl IntoIterator<Item = (SourceId, SourceId)>,
) -> CargoResult<SourceConfigMap<'gctx>> {
let mut base = SourceConfigMap::new(gctx)?;
base.overlays = overlays.into_iter().collect();
Ok(base)
}

Expand All @@ -90,6 +105,7 @@ impl<'gctx> SourceConfigMap<'gctx> {
let mut base = SourceConfigMap {
cfgs: HashMap::new(),
id2name: HashMap::new(),
overlays: HashMap::new(),
gctx,
};
base.add(
Expand Down Expand Up @@ -136,7 +152,7 @@ impl<'gctx> SourceConfigMap<'gctx> {
debug!("loading: {}", id);

let Some(mut name) = self.id2name.get(&id) else {
return id.load(self.gctx, yanked_whitelist);
return self.load_overlaid(id, yanked_whitelist);
};
let mut cfg_loc = "";
let orig_name = name;
Expand All @@ -161,7 +177,7 @@ impl<'gctx> SourceConfigMap<'gctx> {
name = s;
cfg_loc = c;
}
None if id == cfg.id => return id.load(self.gctx, yanked_whitelist),
None if id == cfg.id => return self.load_overlaid(id, yanked_whitelist),
None => {
break cfg.id.with_precise_from(id);
}
Expand All @@ -178,8 +194,8 @@ impl<'gctx> SourceConfigMap<'gctx> {
}
};

let new_src = new_id.load(
self.gctx,
let new_src = self.load_overlaid(
new_id,
&yanked_whitelist
.iter()
.map(|p| p.map_source(id, new_id))
Expand Down Expand Up @@ -215,6 +231,23 @@ restore the source replacement configuration to continue the build
Ok(Box::new(ReplacedSource::new(id, new_id, new_src)))
}

/// Gets the [`Source`] for a given [`SourceId`] without performing any source replacement.
fn load_overlaid(
&self,
id: SourceId,
yanked_whitelist: &HashSet<PackageId>,
) -> CargoResult<Box<dyn Source + 'gctx>> {
let src = id.load(self.gctx, yanked_whitelist)?;
if let Some(overlay_id) = self.overlays.get(&id) {
let overlay = overlay_id.load(self.gctx(), yanked_whitelist)?;
Ok(Box::new(DependencyConfusionThreatOverlaySource::new(
overlay, src,
)))
} else {
Ok(src)
}
}

/// Adds a source config with an associated name.
fn add(&mut self, name: &str, cfg: SourceConfig) -> CargoResult<()> {
if let Some(old_name) = self.id2name.insert(cfg.id, name.to_string()) {
Expand Down
1 change: 1 addition & 0 deletions src/cargo/sources/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ pub use self::replaced::ReplacedSource;
pub mod config;
pub mod directory;
pub mod git;
pub mod overlay;
pub mod path;
pub mod registry;
pub mod replaced;
Expand Down
Loading