diff --git a/docs/development.md b/docs/development.md index b3c14ce..7c99f68 100644 --- a/docs/development.md +++ b/docs/development.md @@ -13,3 +13,17 @@ cargo run -- \ --jwt-issuer-uri http://localhost:8081/jwt/token \ --host http://localhost:8080 ``` + +To test evaluation of a local flake without fetching anything from +GitHub, and writing the tarball and metadata to a local directory +instead of FlakeHub, do: + +```bash +cargo run -- \ + --visibility public \ + --repository foo/bar \ + --tag v0.0.1 \ + --git-root /path/to/repo \ + --directory /path/to/repo/flake \ + --dest-dir /tmp/out +``` diff --git a/src/cli/mod.rs b/src/cli/mod.rs index f6596ca..a91a2ae 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -1,6 +1,13 @@ mod instrumentation; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use std::str::FromStr as _; + +use color_eyre::eyre::{eyre, Context as _, Result}; + +use crate::git_context::GitContext; +use crate::push_context::ExecutionEnvironment; +use crate::{Visibility, DEFAULT_ROLLING_PREFIX}; #[derive(Debug, clap::Parser)] #[clap(version)] @@ -118,6 +125,10 @@ pub(crate) struct FlakeHubPushCli { default_value_t = false )] pub(crate) disable_rename_subgroups: bool, + + /// Write the tarball to a directory instead of pushing it to FlakeHub. + #[clap(long, env = "FLAKEHUB_DEST_DIR", value_parser = PathBufToNoneParser, default_value = "")] + pub(crate) dest_dir: OptionPathBuf, } #[derive(Clone, Debug)] @@ -320,4 +331,96 @@ impl FlakeHubPushCli { } } } + + pub(crate) fn execution_environment(&self) -> ExecutionEnvironment { + if std::env::var("GITHUB_ACTION").ok().is_some() { + ExecutionEnvironment::GitHub + } else if std::env::var("GITLAB_CI").ok().is_some() { + ExecutionEnvironment::GitLab + } else { + ExecutionEnvironment::LocalGitHub + } + } + + pub(crate) fn visibility(&self) -> Result { + match (self.visibility_alt, self.visibility) { + (Some(v), _) => Ok(v), + (None, Some(v)) => Ok(v), + (None, None) => Err(color_eyre::eyre::eyre!( + "Could not determine the flake's desired visibility. Use `--visibility` to set this to one of the following: public, unlisted, private.", + )), + } + } + + pub(crate) fn resolve_local_git_root(&self) -> Result { + let maybe_git_root = match &self.git_root.0 { + Some(gr) => Ok(gr.to_owned()), + None => std::env::current_dir().map(PathBuf::from), + }; + + let local_git_root = maybe_git_root.wrap_err("Could not determine current `git_root`. Pass `--git-root` or set `FLAKEHUB_PUSH_GIT_ROOT`, or run `flakehub-push` with the git root as the current working directory")?; + let local_git_root = local_git_root + .canonicalize() + .wrap_err("Failed to canonicalize `--git-root` argument")?; + + Ok(local_git_root) + } + + pub(crate) fn subdir_from_git_root(&self, local_git_root: &Path) -> Result { + let subdir = + if let Some(directory) = &self.directory.0 { + let absolute_directory = if directory.is_absolute() { + directory.clone() + } else { + local_git_root.join(directory) + }; + let canonical_directory = absolute_directory + .canonicalize() + .wrap_err("Failed to canonicalize `--directory` argument")?; + + PathBuf::from(canonical_directory.strip_prefix(local_git_root).wrap_err( + "Specified `--directory` was not a directory inside the `--git-root`", + )?) + } else { + PathBuf::new() + }; + + Ok(subdir) + } + + pub(crate) fn release_version(&self, git_ctx: &GitContext) -> Result { + let rolling_prefix_or_tag = match (self.rolling_minor.0.as_ref(), &self.tag.0) { + (Some(_), _) if !self.rolling => { + return Err(eyre!( + "You must enable `rolling` to upload a release with a specific `rolling-minor`." + )); + } + (Some(minor), _) => format!("0.{minor}"), + (None, _) if self.rolling => DEFAULT_ROLLING_PREFIX.to_string(), + (None, Some(tag)) => { + let version_only = tag.strip_prefix('v').unwrap_or(tag); + // Ensure the version respects semver + semver::Version::from_str(version_only).wrap_err_with(|| eyre!("Failed to parse version `{tag}` as semver, see https://semver.org/ for specifications"))?; + tag.to_string() + } + (None, None) => { + return Err(eyre!("Could not determine tag or rolling minor version, `--tag`, `GITHUB_REF_NAME`, or `--rolling-minor` must be set")); + } + }; + + let Some(commit_count) = git_ctx.revision_info.commit_count else { + return Err(eyre!("Could not determine commit count, this is normally determined via the `--git-root` argument or via the GitHub API")); + }; + + let rolling_minor_with_postfix_or_tag = if self.rolling_minor.0.is_some() || self.rolling { + format!( + "{rolling_prefix_or_tag}.{}+rev-{}", + commit_count, git_ctx.revision_info.revision + ) + } else { + rolling_prefix_or_tag.to_string() // This will always be the tag since `self.rolling_prefix` was empty. + }; + + Ok(rolling_minor_with_postfix_or_tag) + } } diff --git a/src/main.rs b/src/main.rs index cc5baf9..e731ace 100644 --- a/src/main.rs +++ b/src/main.rs @@ -65,11 +65,45 @@ async fn main() -> Result { } async fn execute() -> Result { - let ctx = { - let mut cli = cli::FlakeHubPushCli::parse(); - cli.instrumentation.setup()?; - PushContext::from_cli_and_env(&mut cli).await? - }; + let mut cli = cli::FlakeHubPushCli::parse(); + cli.instrumentation.setup()?; + + // NOTE(cole-h): If --dest-dir is passed, we're intentionally avoiding doing any actual + // networking (i.e. for FlakeHub and GitHub) + if let Some(dest_dir) = &cli.dest_dir.0 { + let local_git_root = cli.resolve_local_git_root()?; + let local_rev_info = revision_info::RevisionInfo::from_git_root(&local_git_root)?; + let git_ctx = git_context::GitContext { + spdx_expression: cli.spdx_expression.0.clone(), + repo_topics: vec![], + revision_info: local_rev_info, + }; + + let release_version = cli.release_version(&git_ctx)?; + let release_tarball_name = format!("{release_version}.tar.gz"); + let release_json_name = format!("{release_version}.json"); + + let (release_metadata, tarball) = + release_metadata::ReleaseMetadata::new(&cli, &git_ctx, None).await?; + + std::fs::create_dir_all(dest_dir)?; + + { + let dest_file = dest_dir.join(release_tarball_name); + tracing::info!("Writing tarball to {}", dest_file.display()); + std::fs::write(dest_file, tarball.bytes)?; + } + + { + let dest_file = dest_dir.join(release_json_name); + tracing::info!("Writing release metadata to {}", dest_file.display()); + std::fs::write(dest_file, serde_json::to_string(&release_metadata)?)?; + } + + return Ok(ExitCode::SUCCESS); + } + + let ctx = PushContext::from_cli_and_env(&mut cli).await?; let fhclient = FlakeHubClient::new(ctx.flakehub_host, ctx.auth_token)?; diff --git a/src/push_context.rs b/src/push_context.rs index f0fa6fe..18515c1 100644 --- a/src/push_context.rs +++ b/src/push_context.rs @@ -1,28 +1,16 @@ -use std::{ - collections::HashSet, - path::{Path, PathBuf}, - str::FromStr, -}; - use color_eyre::eyre::{eyre, Context, Result}; use crate::{ - build_http_client, - cli::FlakeHubPushCli, - flake_info, flakehub_auth_fake, - flakehub_client::Tarball, - git_context::GitContext, - github::graphql::{GithubGraphqlDataQuery, MAX_LABEL_LENGTH, MAX_NUM_TOTAL_LABELS}, - release_metadata::ReleaseMetadata, - revision_info::RevisionInfo, - DEFAULT_ROLLING_PREFIX, + build_http_client, cli::FlakeHubPushCli, flakehub_auth_fake, flakehub_client::Tarball, + git_context::GitContext, github::graphql::GithubGraphqlDataQuery, + release_metadata::ReleaseMetadata, revision_info::RevisionInfo, }; #[derive(Clone)] pub enum ExecutionEnvironment { GitHub, GitLab, - Local, + LocalGitHub, } pub(crate) struct PushContext { @@ -48,13 +36,7 @@ impl PushContext { let client = build_http_client().build()?; - let exec_env = if std::env::var("GITHUB_ACTION").ok().is_some() { - ExecutionEnvironment::GitHub - } else if std::env::var("GITLAB_CI").ok().is_some() { - ExecutionEnvironment::GitLab - } else { - ExecutionEnvironment::Local - }; + let exec_env = cli.execution_environment(); match exec_env.clone() { ExecutionEnvironment::GitHub => { @@ -66,14 +48,6 @@ impl PushContext { _ => {} }; - let visibility = match (cli.visibility_alt, cli.visibility) { - (Some(v), _) => v, - (None, Some(v)) => v, - (None, None) => return Err(eyre!( - "Could not determine the flake's desired visibility. Use `--visibility` to set this to one of the following: public, unlisted, private.", - )), - }; - // STEP: determine and check 'repository' and 'upload_name' // If the upload name is supplied by the user, ensure that it contains exactly // one slash and no whitespace. Default to the repository name. @@ -89,20 +63,12 @@ impl PushContext { let (upload_name, project_owner, project_name) = determine_names(&cli.name.0, repository, cli.disable_rename_subgroups)?; - let maybe_git_root = match &cli.git_root.0 { - Some(gr) => Ok(gr.to_owned()), - None => std::env::current_dir().map(PathBuf::from), - }; - let local_git_root = maybe_git_root.wrap_err("Could not determine current `git_root`. Pass `--git-root` or set `FLAKEHUB_PUSH_GIT_ROOT`, or run `flakehub-push` with the git root as the current working directory")?; - - let local_git_root = local_git_root - .canonicalize() - .wrap_err("Failed to canonicalize `--git-root` argument")?; + let local_git_root = cli.resolve_local_git_root()?; let local_rev_info = RevisionInfo::from_git_root(&local_git_root)?; // "cli" and "git_ctx" are the user/env supplied info, augmented with data we might have fetched from github/gitlab apis - let (token, git_ctx) = match (exec_env.clone(), &cli.jwt_issuer_uri) { + let (auth_token, git_ctx) = match (&exec_env, &cli.jwt_issuer_uri) { (ExecutionEnvironment::GitHub, None) => { // GITHUB CI let github_token = cli @@ -138,7 +104,7 @@ impl PushContext { (token, git_ctx) } - (ExecutionEnvironment::Local, Some(u)) => { + (ExecutionEnvironment::LocalGitHub, Some(u)) => { // LOCAL, DEV (aka emulating GITHUB) let github_token = cli .github_token @@ -179,184 +145,17 @@ impl PushContext { } }; - // STEP: resolve/canonicalize "subdir" - let subdir = if let Some(directory) = &cli.directory.0 { - let absolute_directory = if directory.is_absolute() { - directory.clone() - } else { - local_git_root.join(directory) - }; - let canonical_directory = absolute_directory - .canonicalize() - .wrap_err("Failed to canonicalize `--directory` argument")?; - - Path::new( - canonical_directory - .strip_prefix(local_git_root.clone()) - .wrap_err( - "Specified `--directory` was not a directory inside the `--git-root`", - )?, - ) - .into() - } else { - PathBuf::new() - }; - - let rolling_prefix_or_tag = match (cli.rolling_minor.0.as_ref(), &cli.tag.0) { - (Some(_), _) if !cli.rolling => { - return Err(eyre!( - "You must enable `rolling` to upload a release with a specific `rolling-minor`." - )); - } - (Some(minor), _) => format!("0.{minor}"), - (None, _) if cli.rolling => DEFAULT_ROLLING_PREFIX.to_string(), - (None, Some(tag)) => { - let version_only = tag.strip_prefix('v').unwrap_or(tag); - // Ensure the version respects semver - semver::Version::from_str(version_only).wrap_err_with(|| eyre!("Failed to parse version `{tag}` as semver, see https://semver.org/ for specifications"))?; - tag.to_string() - } - (None, None) => { - return Err(eyre!("Could not determine tag or rolling minor version, `--tag`, `GITHUB_REF_NAME`, or `--rolling-minor` must be set")); - } - }; - - // TODO(future): (FH-282): change this so commit_count is only set authoritatively, is an explicit error if not set, when rolling, for gitlab - let Some(commit_count) = git_ctx.revision_info.commit_count else { - return Err(eyre!("Could not determine commit count, this is normally determined via the `--git-root` argument or via the GitHub API")); - }; - - let rolling_minor_with_postfix_or_tag = if cli.rolling_minor.0.is_some() || cli.rolling { - format!( - "{rolling_prefix_or_tag}.{}+rev-{}", - commit_count, git_ctx.revision_info.revision - ) - } else { - rolling_prefix_or_tag.to_string() // This will always be the tag since `self.rolling_prefix` was empty. - }; - - // STEP: calculate labels - let merged_labels = { - let mut labels: HashSet<_> = cli - .extra_labels - .clone() - .into_iter() - .filter(|v| !v.is_empty()) - .collect(); - let extra_tags: HashSet<_> = cli - .extra_tags - .clone() - .into_iter() - .filter(|v| !v.is_empty()) - .collect(); - - if !extra_tags.is_empty() { - let message = "`extra-tags` is deprecated and will be removed in the future. Please use `extra-labels` instead."; - tracing::warn!("{message}"); - - if matches!(&exec_env, ExecutionEnvironment::GitHub) { - println!("::warning::{message}"); - } - - if labels.is_empty() { - labels = extra_tags; - } else { - let message = - "Both `extra-tags` and `extra-labels` were set; `extra-tags` will be ignored."; - tracing::warn!("{message}"); - - if matches!(exec_env, ExecutionEnvironment::GitHub) { - println!("::warning::{message}"); - } - } - } - - // Get the "topic" labels from git_ctx, extend local mut labels - let topics = git_ctx.repo_topics; - labels = labels - .into_iter() - .chain(topics.iter().cloned()) - .collect::>(); - - // Here we merge explicitly user-supplied labels and the labels ("topics") - // associated with the repo. Duplicates are excluded and all - // are converted to lower case. - let merged_labels: Vec = labels - .into_iter() - .take(MAX_NUM_TOTAL_LABELS) - .map(|s| s.trim().to_lowercase()) - .filter(|t: &String| { - !t.is_empty() - && t.len() <= MAX_LABEL_LENGTH - && t.chars().all(|c| c.is_alphanumeric() || c == '-') - }) - .collect(); - - merged_labels - }; - - // flake_dir is an absolute path of flake_root(aka git_root)/subdir - let flake_dir = local_git_root.join(&subdir); - - // FIXME: bail out if flake_metadata denotes a dirty tree. - let flake_metadata = - flake_info::FlakeMetadata::from_dir(&flake_dir, cli.my_flake_is_too_big) - .await - .wrap_err("Getting flake metadata")?; - tracing::debug!("Got flake metadata: {:?}", flake_metadata); - - // sanity checks - flake_metadata - .check_evaluates() - .await - .wrap_err("failed to evaluate all system attrs of the flake")?; - flake_metadata - .check_lock_if_exists() - .await - .wrap_err("failed to evaluate all system attrs of the flake")?; - - let flake_outputs = flake_metadata.outputs(cli.include_output_paths).await?; - tracing::debug!("Got flake outputs: {:?}", flake_outputs); - - let description = flake_metadata - .metadata_json - .get("description") - .and_then(serde_json::Value::as_str) - .map(|s| s.to_string()); - - let readme = flake_metadata.get_readme_contents().await?; - - let release_metadata = ReleaseMetadata { - commit_count, - description, - outputs: flake_outputs.0, - raw_flake_metadata: flake_metadata.metadata_json.clone(), - readme, - // TODO(colemickens): remove this confusing, redundant field (FH-267) - repo: upload_name.to_string(), - revision: git_ctx.revision_info.revision, - visibility, - mirrored: cli.mirror, - source_subdirectory: Some( - subdir - .to_str() - .map(|d| d.to_string()) - .ok_or(eyre!("Directory {:?} is not a valid UTF-8 string", subdir))?, - ), - spdx_identifier: git_ctx.spdx_expression, - labels: merged_labels, - }; + let release_version = cli.release_version(&git_ctx)?; - let flake_tarball = flake_metadata - .flake_tarball() - .wrap_err("Making release tarball")?; + let (release_metadata, flake_tarball) = + ReleaseMetadata::new(cli, &git_ctx, Some(&exec_env)).await?; let ctx = Self { flakehub_host: cli.host.clone(), - auth_token: token, + auth_token, upload_name, - release_version: rolling_minor_with_postfix_or_tag, + release_version, error_if_release_conflicts: cli.error_on_conflict, @@ -368,7 +167,7 @@ impl PushContext { } } -fn determine_names( +pub(crate) fn determine_names( explicitly_provided_name: &Option, repository: &str, subgroup_renaming_explicitly_disabled: bool, diff --git a/src/release_metadata.rs b/src/release_metadata.rs index 340b103..af0d757 100644 --- a/src/release_metadata.rs +++ b/src/release_metadata.rs @@ -1,3 +1,13 @@ +use std::collections::HashSet; + +use color_eyre::eyre::{eyre, Context as _, Result}; + +use crate::cli::FlakeHubPushCli; +use crate::flake_info::FlakeMetadata; +use crate::flakehub_client::Tarball; +use crate::git_context::GitContext; +use crate::github::graphql::{MAX_LABEL_LENGTH, MAX_NUM_TOTAL_LABELS}; +use crate::push_context::ExecutionEnvironment; use crate::Visibility; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] @@ -24,6 +34,155 @@ pub(crate) struct ReleaseMetadata { pub(crate) labels: Vec, } +impl ReleaseMetadata { + pub async fn new( + cli: &FlakeHubPushCli, + git_ctx: &GitContext, + exec_env: Option<&ExecutionEnvironment>, + ) -> Result<(Self, Tarball)> { + let local_git_root = cli.resolve_local_git_root()?; + let subdir = cli.subdir_from_git_root(&local_git_root)?; + + // flake_dir is an absolute path of flake_root(aka git_root)/subdir + let flake_dir = local_git_root.join(&subdir); + + let flake_metadata = FlakeMetadata::from_dir(&flake_dir, cli.my_flake_is_too_big) + .await + .wrap_err("Getting flake metadata")?; + tracing::debug!("Got flake metadata: {:?}", flake_metadata); + + // sanity checks + flake_metadata + .check_evaluates() + .await + .wrap_err("failed to evaluate all system attrs of the flake")?; + flake_metadata + .check_lock_if_exists() + .await + .wrap_err("failed to evaluate all system attrs of the flake")?; + + let Some(commit_count) = git_ctx.revision_info.commit_count else { + return Err(eyre!("Could not determine commit count, this is normally determined via the `--git-root` argument or via the GitHub API")); + }; + + let description = flake_metadata + .metadata_json + .get("description") + .and_then(serde_json::Value::as_str) + .map(|s| s.to_string()); + + let flake_outputs = flake_metadata.outputs(cli.include_output_paths).await?; + tracing::debug!("Got flake outputs: {:?}", flake_outputs); + + let readme = flake_metadata.get_readme_contents().await?; + + let Some(ref repository) = cli.repository.0 else { + return Err(eyre!("Could not determine repository name, pass `--repository` formatted like `determinatesystems/flakehub-push`")); + }; + + let (upload_name, _project_owner, _project_name) = crate::push_context::determine_names( + &cli.name.0, + repository, + cli.disable_rename_subgroups, + )?; + + let visibility = cli.visibility()?; + + let labels = if let Some(exec_env) = exec_env { + Self::merged_labels(cli, git_ctx, exec_env) + } else { + Vec::new() + }; + + let release_metadata = ReleaseMetadata { + commit_count, + description, + outputs: flake_outputs.0, + raw_flake_metadata: flake_metadata.metadata_json.clone(), + readme, + // TODO(colemickens): remove this confusing, redundant field (FH-267) + repo: upload_name.to_string(), + revision: git_ctx.revision_info.revision.clone(), + visibility, + mirrored: cli.mirror, + source_subdirectory: Some(subdir.to_str().map(|d| d.to_string()).ok_or( + color_eyre::eyre::eyre!("Directory {:?} is not a valid UTF-8 string", subdir), + )?), + spdx_identifier: git_ctx.spdx_expression.clone(), + labels, + }; + + let flake_tarball = flake_metadata + .flake_tarball() + .wrap_err("Making release tarball")?; + + Ok((release_metadata, flake_tarball)) + } + + fn merged_labels( + cli: &FlakeHubPushCli, + git_ctx: &GitContext, + exec_env: &ExecutionEnvironment, + ) -> Vec { + let mut labels: HashSet<_> = cli + .extra_labels + .clone() + .into_iter() + .filter(|v| !v.is_empty()) + .collect(); + let extra_tags: HashSet<_> = cli + .extra_tags + .clone() + .into_iter() + .filter(|v| !v.is_empty()) + .collect(); + + if !extra_tags.is_empty() { + let message = "`extra-tags` is deprecated and will be removed in the future. Please use `extra-labels` instead."; + tracing::warn!("{message}"); + + if matches!(&exec_env, ExecutionEnvironment::GitHub) { + println!("::warning::{message}"); + } + + if labels.is_empty() { + labels = extra_tags; + } else { + let message = + "Both `extra-tags` and `extra-labels` were set; `extra-tags` will be ignored."; + tracing::warn!("{message}"); + + if matches!(exec_env, ExecutionEnvironment::GitHub) { + println!("::warning::{message}"); + } + } + } + + // Get the "topic" labels from git_ctx, extend local mut labels + let topics = &git_ctx.repo_topics; + labels = labels + .into_iter() + .chain(topics.iter().cloned()) + .collect::>(); + + // Here we merge explicitly user-supplied labels and the labels ("topics") + // associated with the repo. Duplicates are excluded and all + // are converted to lower case. + let merged_labels: Vec = labels + .into_iter() + .take(MAX_NUM_TOTAL_LABELS) + .map(|s| s.trim().to_lowercase()) + .filter(|t: &String| { + !t.is_empty() + && t.len() <= MAX_LABEL_LENGTH + && t.chars().all(|c| c.is_alphanumeric() || c == '-') + }) + .collect(); + + merged_labels + } +} + // TODO(review,colemickens): I don't really undersatnd why these are nededed??? we don't need the OptionString-y stuff since this isn't GHA adjacent? fn option_string_to_spdx<'de, D>(deserializer: D) -> Result, D::Error>