From b4dc4a02ef6b3cfb82475e37c4042e1e4cc2590f Mon Sep 17 00:00:00 2001 From: Aiden Fujiwara Date: Sat, 7 Oct 2023 16:06:56 -0700 Subject: [PATCH] added support for artifactory download --- Cargo.lock | 2 + Cargo.toml | 2 + artiaa_auth/src/lib.rs | 8 +- src/auth_store.rs | 1 - src/config.rs | 68 +++++++++------- src/error.rs | 16 ++-- src/paths.rs | 55 ++++++++++++- src/tool_cache.rs | 2 +- src/tool_provider/artifactory.rs | 132 ++++++++++++++++++++++++++++--- src/tool_provider/github.rs | 6 +- src/tool_provider/gitlab.rs | 6 +- src/tool_provider/mod.rs | 6 +- 12 files changed, 241 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b73a586..2700b49 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -392,6 +392,7 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" name = "foreman" version = "1.2.0" dependencies = [ + "artiaa_auth", "assert_cmd", "command-group", "dirs", @@ -409,6 +410,7 @@ dependencies = [ "tokio", "toml", "toml_edit", + "url", "urlencoding", "zip", ] diff --git a/Cargo.toml b/Cargo.toml index 9dfcc05..19b0fd5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,8 @@ toml = "0.5.9" toml_edit = "0.14.4" urlencoding = "2.1.0" zip = "0.5" +url = "2.4.1" +artiaa_auth = { path = "./artiaa_auth" } [target.'cfg(windows)'.dependencies] command-group = "1.0.8" diff --git a/artiaa_auth/src/lib.rs b/artiaa_auth/src/lib.rs index 149cf76..c428456 100644 --- a/artiaa_auth/src/lib.rs +++ b/artiaa_auth/src/lib.rs @@ -1,4 +1,4 @@ -mod error; +pub mod error; mod fs; use std::{collections::HashMap, path::Path}; @@ -10,8 +10,8 @@ use crate::error::{ArtifactoryAuthError, ArtifactoryAuthResult}; #[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq)] pub struct Credentials { - username: String, - token: String, + pub username: String, + pub token: String, } /// Contains stored user tokens that are used to download artifacts from Artifactory. @@ -21,7 +21,6 @@ pub struct Tokens { } impl Tokens { - #[allow(dead_code)] pub fn load(path: &Path) -> ArtifactoryAuthResult { if let Some(contents) = fs::try_read(path)? { let tokens: Tokens = serde_json::from_slice(&contents) @@ -34,7 +33,6 @@ impl Tokens { } } - #[allow(dead_code)] pub fn get_credentials(&self, url: &Url) -> Option<&Credentials> { if let Some(domain) = url.domain() { self.tokens.get(domain) diff --git a/src/auth_store.rs b/src/auth_store.rs index 0fcec6d..1a210fc 100644 --- a/src/auth_store.rs +++ b/src/auth_store.rs @@ -7,7 +7,6 @@ use crate::{ error::{ForemanError, ForemanResult}, fs, }; - pub static DEFAULT_AUTH_CONFIG: &str = include_str!("../resources/default-auth.toml"); /// Contains stored user tokens that Foreman can use to download tools. diff --git a/src/config.rs b/src/config.rs index 177de7f..b5b41c6 100644 --- a/src/config.rs +++ b/src/config.rs @@ -11,13 +11,14 @@ use std::{ env, fmt, }; use toml::Value; +use url::Url; const GITHUB: &'static str = "https://github.com"; const GITLAB: &'static str = "https://gitlab.com"; #[derive(Debug, Clone, PartialEq)] pub struct ToolSpec { - host: String, + host: Url, path: String, version: VersionReq, protocol: Protocol, @@ -70,7 +71,7 @@ impl ToolSpec { }); } - let host = host_source.source.to_string(); + let host = host_source.source.to_owned(); let path = path_val .as_str() .ok_or_else(|| ConfigFileParseError::Tool { @@ -126,6 +127,10 @@ impl ToolSpec { Protocol::Artifactory => Provider::Artifactory, } } + + pub fn host(&self) -> &Url { + &self.host + } } impl fmt::Display for ToolSpec { @@ -142,21 +147,18 @@ pub struct ConfigFile { #[derive(Debug, PartialEq)] pub struct Host { - source: String, + source: Url, protocol: Protocol, } impl Host { - pub fn new>(source: S, protocol: Protocol) -> Self { - Self { - source: source.into(), - protocol, - } + pub fn new(source: Url, protocol: Protocol) -> Self { + Self { source, protocol } } pub fn from_value(value: &Value) -> ConfigFileParseResult { if let Value::Table(mut map) = value.clone() { - let source = map + let source_string = map .remove("source") .ok_or_else(|| ConfigFileParseError::Host { host: value.to_string(), @@ -167,6 +169,9 @@ impl Host { })? .to_string(); + let source = Url::parse(&source_string).map_err(|_| ConfigFileParseError::Host { + host: value.to_string(), + })?; let protocol_value = map.remove("protocol") .ok_or_else(|| ConfigFileParseError::Host { @@ -211,9 +216,18 @@ impl ConfigFile { Self { tools: BTreeMap::new(), hosts: HashMap::from([ - ("source".to_string(), Host::new(GITHUB, Protocol::Github)), - ("github".to_string(), Host::new(GITHUB, Protocol::Github)), - ("gitlab".to_string(), Host::new(GITLAB, Protocol::Gitlab)), + ( + "source".to_string(), + Host::new(Url::parse(GITHUB).unwrap(), Protocol::Github), + ), + ( + "github".to_string(), + Host::new(Url::parse(GITHUB).unwrap(), Protocol::Github), + ), + ( + "gitlab".to_string(), + Host::new(Url::parse(GITLAB).unwrap(), Protocol::Gitlab), + ), ]), } } @@ -332,7 +346,7 @@ mod test { fn new_github>(github: S, version: VersionReq) -> ToolSpec { ToolSpec { - host: GITHUB.to_string(), + host: Url::parse(GITHUB).unwrap(), path: github.into(), version: version, protocol: Protocol::Github, @@ -341,7 +355,7 @@ mod test { fn new_gitlab>(gitlab: S, version: VersionReq) -> ToolSpec { ToolSpec { - host: GITLAB.to_string(), + host: Url::parse(GITLAB).unwrap(), path: gitlab.into(), version: version, protocol: Protocol::Gitlab, @@ -350,7 +364,7 @@ mod test { fn new_artifactory>(host: S, path: S, version: VersionReq) -> ToolSpec { ToolSpec { - host: host.into(), + host: Url::parse(host.into().as_str()).unwrap(), path: path.into(), version: version, protocol: Protocol::Artifactory, @@ -367,26 +381,23 @@ mod test { VersionReq::parse(string).unwrap() } - fn new_host>(source: S, protocol: Protocol) -> Host { - Host { - source: source.into(), - protocol, - } + fn new_host(source: Url, protocol: Protocol) -> Host { + Host { source, protocol } } fn default_hosts() -> HashMap { HashMap::from([ ( "source".to_string(), - Host::new(GITHUB.to_string(), Protocol::Github), + Host::new(Url::parse(GITHUB).unwrap(), Protocol::Github), ), ( "github".to_string(), - Host::new(GITHUB.to_string(), Protocol::Github), + Host::new(Url::parse(GITHUB).unwrap(), Protocol::Github), ), ( "gitlab".to_string(), - Host::new(GITLAB.to_string(), Protocol::Gitlab), + Host::new(Url::parse(GITLAB).unwrap(), Protocol::Gitlab), ), ]) } @@ -395,7 +406,7 @@ mod test { let mut hosts = default_hosts(); hosts.insert( "artifactory".to_string(), - Host::new(ARTIFACTORY.to_string(), Protocol::Artifactory), + Host::new(Url::parse(ARTIFACTORY).unwrap(), Protocol::Artifactory), ); hosts } @@ -469,7 +480,10 @@ mod test { let host = Host::from_value(&value).unwrap(); assert_eq!( host, - new_host("https://artifactory.com", Protocol::Artifactory) + new_host( + Url::parse("https://artifactory.com").unwrap(), + Protocol::Artifactory + ) ) } @@ -546,7 +560,7 @@ mod test { BTreeMap::from([( "tool".to_string(), ToolSpec { - host: "https://artifactory.com".to_string(), + host: Url::parse("https://artifactory.com").unwrap(), path: "path/to/tool".to_string(), version: VersionReq::parse("1.0.0").unwrap(), protocol: Protocol::Artifactory @@ -555,7 +569,7 @@ mod test { HashMap::from([( "artifactory".to_string(), Host { - source: "https://artifactory.com".to_string(), + source: Url::parse("https://artifactory.com").unwrap(), protocol: Protocol::Artifactory } )]) diff --git a/src/error.rs b/src/error.rs index 8ece41f..581a505 100644 --- a/src/error.rs +++ b/src/error.rs @@ -3,7 +3,7 @@ use std::{fmt, io, path::PathBuf}; use semver::Version; use crate::config::{ConfigFile, ToolSpec}; - +use artiaa_auth::error::ArtifactoryAuthError; pub type ForemanResult = Result; pub type ConfigFileParseResult = Result; #[derive(Debug)] @@ -71,8 +71,11 @@ pub enum ForemanError { ToolsNotDownloaded { tools: Vec, }, - Other { - message: String, + EnvVarNotFound { + env_var: String, + }, + ArtiAAError { + error: ArtifactoryAuthError, }, } @@ -319,8 +322,11 @@ impl fmt::Display for ForemanError { Self::ToolsNotDownloaded { tools } => { write!(f, "The following tools were not installed:\n{:#?}", tools) } - Self::Other { message } => { - write!(f, "{}", message) + Self::EnvVarNotFound { env_var } => { + write!(f, "Environment Variable not found: {}", env_var) + } + Self::ArtiAAError { error } => { + write!(f, "{}", error) } } } diff --git a/src/paths.rs b/src/paths.rs index ea87e47..8c58976 100644 --- a/src/paths.rs +++ b/src/paths.rs @@ -1,8 +1,15 @@ //! Contains all of the paths that Foreman needs to deal with. -use std::path::{Path, PathBuf}; +use std::{ + env, + path::{Path, PathBuf}, +}; -use crate::{auth_store::DEFAULT_AUTH_CONFIG, error::ForemanError, fs}; +use crate::{ + auth_store::DEFAULT_AUTH_CONFIG, + error::{ForemanError, ForemanResult}, + fs, +}; static DEFAULT_USER_CONFIG: &str = include_str!("../resources/default-foreman.toml"); @@ -20,7 +27,7 @@ impl ForemanPaths { .ok() .and_then(|path| { if path.is_dir() { - Some(Self { root_dir:path }) + Some(Self { root_dir: path }) } else { if path.exists() { log::warn!( @@ -87,6 +94,48 @@ impl ForemanPaths { Ok(()) } + + pub fn artiaa_path(&self) -> ForemanResult { + get_artiaa_path_based_on_os() + } +} + +#[cfg(target_os = "windows")] +fn get_artiaa_path_based_on_os() -> ForemanResult { + let localappdata = env::var("LOCALAPPDATA").map_err(|_| ForemanError::EnvVarNotFound { + env_var: "%$LOCALAPPDATA%".to_string(), + })?; + Ok(PathBuf::from(format!( + "{}/ArtiAA/artiaa-tokens.json", + localappdata + ))) +} + +#[cfg(target_os = "macos")] +fn get_artiaa_path_based_on_os() -> ForemanResult { + let home = env::var("HOME").map_err(|_| ForemanError::EnvVarNotFound { + env_var: "$HOME".to_string(), + })?; + Ok(PathBuf::from(format!( + "{}/Library/Application Support/ArtiAA/artiaa-tokens.json", + home + ))) +} + +#[cfg(all(not(target_os = "macos"), target_family = "unix"))] +fn get_artiaa_path_based_on_os() -> ForemanResult { + let xdg_data_home = env::var("XDG_DATA_HOME").map_err(|_| ForemanError::EnvVarNotFound { + env_var: "$XDG_DATA_HOME".to_string(), + })?; + Ok(PathBuf::from(format!( + "{}/ArtiAA/artiaa-tokens.json", + xdg_data_home + ))) +} + +#[cfg(other)] +fn get_artiaa_path_based_on_os() -> PathBuf { + unimplemented!("artiaa_path is only defined for windows or unix operating systems") } impl Default for ForemanPaths { diff --git a/src/tool_cache.rs b/src/tool_cache.rs index ceb1726..289693c 100644 --- a/src/tool_cache.rs +++ b/src/tool_cache.rs @@ -103,7 +103,7 @@ impl ToolCache { log::info!("Downloading {}", tool); let provider = providers.get(&tool.provider()); - let releases = provider.get_releases(tool.path())?; + let releases = provider.get_releases(tool.path(), tool.host())?; // Filter down our set of releases to those that are valid versions and // have release assets for our current platform. diff --git a/src/tool_provider/artifactory.rs b/src/tool_provider/artifactory.rs index 1b76a46..d24a856 100644 --- a/src/tool_provider/artifactory.rs +++ b/src/tool_provider/artifactory.rs @@ -1,14 +1,22 @@ //! Slice of GitHub's API that Foreman consumes. +use super::{Release, ReleaseAsset, ToolProviderImpl}; use crate::{ error::{ForemanError, ForemanResult}, paths::ForemanPaths, }; +use artiaa_auth; +use reqwest::{ + blocking::Client, + header::{AUTHORIZATION, USER_AGENT}, +}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use url::Url; -use super::{Release, ToolProviderImpl}; +const ARTIFACTORY_API_KEY_HEADER: &str = "X-JFrog-Art-Api"; #[derive(Debug)] -#[allow(unused)] pub struct ArtifactoryProvider { paths: ForemanPaths, } @@ -18,19 +26,119 @@ impl ArtifactoryProvider { Self { paths } } } -#[allow(unused)] + impl ToolProviderImpl for ArtifactoryProvider { - fn get_releases(&self, repo: &str) -> ForemanResult> { - Err(ForemanError::Other { - message: "Artifactory is not yet supported. Please use Github or Gitlab as your source" - .to_owned(), - }) + fn get_releases(&self, repo: &str, host: &Url) -> ForemanResult> { + let client = Client::new(); + + let url = format!("{}artifactory/api/storage/{}", host, repo); + let params = vec![("list", ""), ("deep", "1")]; + let mut builder = client + .get(&url) + .header(USER_AGENT, "Roblox/foreman") + .query(¶ms); + + let tokens = artiaa_auth::Tokens::load(&self.paths.artiaa_path()?) + .map_err(|error| ForemanError::ArtiAAError { error })?; + + if let Some(credentials) = tokens.get_credentials(host) { + builder = builder.header(ARTIFACTORY_API_KEY_HEADER, credentials.token.to_string()); + } + log::debug!("Downloading artifactory releases for {}", repo); + let response_body = builder + .send() + .map_err(ForemanError::request_failed)? + .text() + .map_err(ForemanError::request_failed)?; + + let response: ArtifactoryResponse = + serde_json::from_str(&response_body).map_err(|err| { + ForemanError::unexpected_response_body(err.to_string(), response_body, url) + })?; + + let mut releases: Vec = Vec::new(); + let mut release_map: HashMap<&str, Vec> = HashMap::new(); + for file in &response.files { + let uri = file.uri.split("/").collect::>(); + // file.uri will look something like //, so uri will be ["", , ForemanResult> { - Err(ForemanError::Other { - message: "Artifactory is not yet supported. Please use Github or Gitlab as your source" - .to_owned(), - }) + let client = Client::new(); + let artifactory_url = Url::parse(url).unwrap(); + + let mut builder = client.get(url).header(USER_AGENT, "Roblox/foreman"); + + let tokens = artiaa_auth::Tokens::load(&self.paths.artiaa_path()?).unwrap(); + if let Some(credentials) = tokens.get_credentials(&artifactory_url) { + builder = builder.header(AUTHORIZATION, format!("bearer {}", credentials.token)); + } + + log::debug!("Downloading release asset {}", url); + let mut response = builder.send().map_err(ForemanError::request_failed)?; + + let mut output = Vec::new(); + response + .copy_to(&mut output) + .map_err(ForemanError::request_failed)?; + Ok(output) + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct ArtifactoryResponse { + files: Vec, +} +#[derive(Debug, Serialize, Deserialize)] +struct ArtifactoryResponseFiles { + uri: String, +} +#[derive(Debug)] +struct ArtifactoryRelease { + tag_name: String, + assets: Vec, +} +#[derive(Debug)] +struct ArtifactoryAsset { + url: String, + name: String, +} + +impl From for Release { + fn from(release: ArtifactoryRelease) -> Self { + Release { + tag_name: release.tag_name, + prerelease: false, + assets: release.assets.into_iter().map(Into::into).collect(), + } + } +} + +impl From for ReleaseAsset { + fn from(asset: ArtifactoryAsset) -> Self { + ReleaseAsset { + url: asset.url, + name: asset.name, + } } } diff --git a/src/tool_provider/github.rs b/src/tool_provider/github.rs index ca221d7..0b40c1f 100644 --- a/src/tool_provider/github.rs +++ b/src/tool_provider/github.rs @@ -6,13 +6,13 @@ use reqwest::{ }; use serde::{Deserialize, Serialize}; +use super::{Release, ReleaseAsset, ToolProviderImpl}; use crate::{ auth_store::AuthStore, error::{ForemanError, ForemanResult}, paths::ForemanPaths, }; - -use super::{Release, ReleaseAsset, ToolProviderImpl}; +use url::Url; #[derive(Debug)] pub struct GithubProvider { @@ -26,7 +26,7 @@ impl GithubProvider { } impl ToolProviderImpl for GithubProvider { - fn get_releases(&self, repo: &str) -> ForemanResult> { + fn get_releases(&self, repo: &str, _host: &Url) -> ForemanResult> { let client = Client::new(); let url = format!("https://api.github.com/repos/{}/releases", repo); diff --git a/src/tool_provider/gitlab.rs b/src/tool_provider/gitlab.rs index 01c4b6b..b1edabd 100644 --- a/src/tool_provider/gitlab.rs +++ b/src/tool_provider/gitlab.rs @@ -6,13 +6,13 @@ use reqwest::{ }; use serde::{Deserialize, Serialize}; +use super::{Release, ReleaseAsset, ToolProviderImpl}; use crate::{ auth_store::AuthStore, error::{ForemanError, ForemanResult}, paths::ForemanPaths, }; - -use super::{Release, ReleaseAsset, ToolProviderImpl}; +use url::Url; #[derive(Debug, Default)] pub struct GitlabProvider { @@ -26,7 +26,7 @@ impl GitlabProvider { } impl ToolProviderImpl for GitlabProvider { - fn get_releases(&self, repo: &str) -> ForemanResult> { + fn get_releases(&self, repo: &str, _host: &Url) -> ForemanResult> { let client = Client::new(); let url = format!( diff --git a/src/tool_provider/mod.rs b/src/tool_provider/mod.rs index ae701c0..aec8137 100644 --- a/src/tool_provider/mod.rs +++ b/src/tool_provider/mod.rs @@ -2,15 +2,15 @@ mod artifactory; mod github; mod gitlab; -use std::{collections::HashMap, fmt}; - use crate::{error::ForemanResult, paths::ForemanPaths}; use artifactory::ArtifactoryProvider; use github::GithubProvider; use gitlab::GitlabProvider; +use std::{collections::HashMap, fmt}; +use url::Url; pub trait ToolProviderImpl: fmt::Debug { - fn get_releases(&self, repo: &str) -> ForemanResult>; + fn get_releases(&self, repo: &str, host: &Url) -> ForemanResult>; fn download_asset(&self, url: &str) -> ForemanResult>; }