Skip to content

Commit

Permalink
Add option for disabling closure computation in GHA cache
Browse files Browse the repository at this point in the history
Allows to only cache the built derivations without their closures.

This avoids polluting the GHA cache with many duplicate entries, which
happens when a downstream derivation changes (and is rebuilt), but
its dependencies don't (but would get pushed to the cache each time
regardless).
  • Loading branch information
yfyf committed Nov 22, 2024
1 parent 296e9dc commit 9870817
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 12 deletions.
28 changes: 16 additions & 12 deletions magic-nix-cache/src/gha.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ pub struct GhaCache {
worker_result: RwLock<Option<tokio::task::JoinHandle<Result<()>>>>,

channel_tx: UnboundedSender<Request>,

compute_closure: bool,
}

#[derive(Debug)]
Expand All @@ -36,6 +38,7 @@ impl GhaCache {
store: Arc<NixStore>,
metrics: Arc<telemetry::TelemetryReport>,
narinfo_negative_cache: Arc<RwLock<HashSet<String>>>,
compute_closure: bool,
) -> Result<GhaCache> {
let mut api = Api::new(credentials)?;

Expand Down Expand Up @@ -64,6 +67,7 @@ impl GhaCache {
api,
worker_result: RwLock::new(Some(worker_result)),
channel_tx,
compute_closure,
})
}

Expand All @@ -85,18 +89,18 @@ impl GhaCache {
store: Arc<NixStore>,
store_paths: Vec<StorePath>,
) -> Result<()> {
// FIXME: make sending the closure optional. We might want to
// only send the paths that have been built by the user, under
// the assumption that everything else is already in a binary
// cache.
// FIXME: compute_fs_closure_multi doesn't return a
// toposort, though it doesn't really matter for the GHA
// cache.
let closure = store
.compute_fs_closure_multi(store_paths, false, false, false)
.await?;

for p in closure {
let final_paths = if self.compute_closure {
// FIXME: compute_fs_closure_multi doesn't return a
// toposort, though it doesn't really matter for the GHA
// cache.
store
.compute_fs_closure_multi(store_paths, false, false, false)
.await?
} else {
store_paths
};

for p in final_paths {
self.channel_tx
.send(Request::Upload(p))
.map_err(|_| Error::Internal("Cannot send upload message".to_owned()))?;
Expand Down
5 changes: 5 additions & 0 deletions magic-nix-cache/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,10 @@ struct Args {
/// Whether or not to diff the store before and after Magic Nix Cache runs
#[arg(long, default_value_t = false)]
diff_store: bool,

/// (GHA only) Don't include the closure of the to-be-cached store paths
#[arg(long, default_value_t = false)]
no_closure: bool,
}

#[derive(Debug, Clone, Copy, PartialEq, clap::ValueEnum)]
Expand Down Expand Up @@ -389,6 +393,7 @@ async fn main_cli() -> Result<()> {
store.clone(),
metrics.clone(),
narinfo_negative_cache.clone(),
!args.no_closure,
)
.with_context(|| "Failed to initialize GitHub Actions Cache API")?;

Expand Down

0 comments on commit 9870817

Please sign in to comment.