From c4d6f20069f5d1bcad237400a516f79a1d5ef579 Mon Sep 17 00:00:00 2001 From: Mark Elliot <123787712+mark-thm@users.noreply.github.com> Date: Mon, 27 May 2024 20:21:18 -0400 Subject: [PATCH 1/3] Move lockfile to its own module --- Cargo.lock | 1 + Cargo.toml | 1 + src/lockfile.rs | 148 +++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 155 +++++++++++++++--------------------------------- 4 files changed, 198 insertions(+), 107 deletions(-) create mode 100644 src/lockfile.rs diff --git a/Cargo.lock b/Cargo.lock index da0a33b..9b110d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -528,6 +528,7 @@ name = "multitool" version = "0.2.1" dependencies = [ "clap", + "once_cell", "regex", "reqwest", "serde", diff --git a/Cargo.toml b/Cargo.toml index d56ca4d..cf6fa27 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ regex = "1.10.4" serde = { version = "1.0.200", features = ["derive"] } serde_json = "1.0.116" sha256 = "1.5.0" +once_cell = "1.19.0" # The profile that 'cargo dist' will build with [profile.dist] diff --git a/src/lockfile.rs b/src/lockfile.rs new file mode 100644 index 0000000..9415826 --- /dev/null +++ b/src/lockfile.rs @@ -0,0 +1,148 @@ +use serde::{Deserialize, Serialize}; +use std::{ + collections::{BTreeMap, HashMap}, + fmt::Display, +}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum SupportedOs { + Linux, + MacOS, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum SupportedCpu { + Arm64, + X86_64, +} + +#[derive(Clone, Serialize, Deserialize)] +pub struct FileBinary { + pub url: String, + pub sha256: String, + pub os: SupportedOs, + pub cpu: SupportedCpu, + #[serde(skip_serializing_if = "Option::is_none")] + pub headers: Option>, +} + +#[derive(Clone, Serialize, Deserialize)] +pub struct ArchiveBinary { + pub url: String, + pub file: String, + pub sha256: String, + pub os: SupportedOs, + pub cpu: SupportedCpu, + #[serde(skip_serializing_if = "Option::is_none")] + pub headers: Option>, + #[serde(rename = "type", skip_serializing_if = "Option::is_none")] + pub type_: Option, // TODO(mark): we should probably make this an enum +} + +#[derive(Clone, Serialize, Deserialize)] +pub struct PkgBinary { + pub url: String, + pub file: String, + pub sha256: String, + pub os: SupportedOs, + pub cpu: SupportedCpu, + #[serde(skip_serializing_if = "Option::is_none")] + pub headers: Option>, +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "lowercase")] +pub enum Binary { + File(FileBinary), + Archive(ArchiveBinary), + Pkg(PkgBinary), +} + +#[derive(Serialize, Deserialize)] +pub struct ToolDefinition { + pub binaries: Vec, +} + +impl Display for SupportedCpu { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self { + SupportedCpu::Arm64 => write!(f, "arm64"), + SupportedCpu::X86_64 => write!(f, "x86_64"), + } + } +} + +impl Display for SupportedOs { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self { + SupportedOs::Linux => write!(f, "linux"), + SupportedOs::MacOS => write!(f, "macos"), + } + } +} + +fn schema() -> String { + "https://raw.githubusercontent.com/theoremlp/rules_multitool/main/lockfile.schema.json" + .to_owned() +} + +#[derive(Serialize, Deserialize)] +pub struct Lockfile { + #[serde(rename = "$schema", default = "schema")] + pub schema: String, + + #[serde(flatten)] + pub tools: BTreeMap, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn deserialize_empty_lockfile() { + let lockfile: Lockfile = serde_json::from_str("{}").unwrap(); + assert_eq!(lockfile.schema, schema()); + assert_eq!(lockfile.tools.len(), 0); + } + + #[test] + fn deserialize_lockfile_with_schema_and_no_tools() { + let lockfile: Lockfile = serde_json::from_str(r#"{ + "$schema": "https://raw.githubusercontent.com/theoremlp/rules_multitool/main/lockfile.schema.json" + }"#).unwrap(); + assert_eq!( + lockfile.schema, + "https://raw.githubusercontent.com/theoremlp/rules_multitool/main/lockfile.schema.json" + .to_owned() + ); + assert_eq!(lockfile.tools.len(), 0); + } + + #[test] + fn deserialize_lockfile_with_schema_and_tools() { + let lockfile: Lockfile = serde_json::from_str(r#"{ + "$schema": "https://raw.githubusercontent.com/theoremlp/rules_multitool/main/lockfile.schema.json", + "tool-name": { + "binaries": [ + { + "kind": "file", + "url": "https://github.com/theoremlp/multitool/releases/download/v0.2.1/multitool-x86_64-unknown-linux-gnu.tar.xz", + "sha256": "9523faf97e4e3fea5f98ba9d051e67c90799182580d8ae56cba2e45c7de0b4ce", + "os": "linux", + "cpu": "x86_64" + } + ] + } + }"#).unwrap(); + assert_eq!( + lockfile.schema, + Some("https://raw.githubusercontent.com/theoremlp/rules_multitool/main/lockfile.schema.json".to_owned()) + ); + assert_eq!(lockfile.tools.len(), 1); + assert_eq!(lockfile.tools["tool-name"].binaries.len(), 1); + // TOOD(mark): richer tests + } +} diff --git a/src/main.rs b/src/main.rs index 47026cd..595c80a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,14 +1,29 @@ use clap::{Parser, Subcommand}; +use lockfile::{ArchiveBinary, Binary, FileBinary, Lockfile, PkgBinary, ToolDefinition}; +use once_cell::sync::Lazy as LazyLock; use regex::Regex; -use serde::{Deserialize, Serialize}; use serde_json::Value; use std::{ collections::{BTreeMap, HashMap}, error::Error, - fmt::Display, fs, }; +mod lockfile; + +static GITHUB_RELEASE_PATTERN: LazyLock = LazyLock::new(|| { + Regex::new( + r"(?x) + https://github\.com/ + (?P[A-Za-z0-9_-]+)/ + (?P[A-Za-z0-9_-]+)/ + releases/download/ + (?Pv?[^/]+)/ + (?P.+)", + ) + .unwrap() +}); + #[derive(Parser)] struct Cli { #[clap(long)] @@ -25,122 +40,44 @@ enum Commands { Update, } -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -enum SupportedOs { - Linux, - MacOS, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -enum SupportedCpu { - Arm64, - X86_64, -} - -#[derive(Clone, Serialize, Deserialize)] -struct FileBinary { - url: String, - sha256: String, - os: SupportedOs, - cpu: SupportedCpu, - #[serde(skip_serializing_if = "Option::is_none")] - headers: Option>, -} - -#[derive(Clone, Serialize, Deserialize)] -struct ArchiveBinary { - url: String, - file: String, - sha256: String, - os: SupportedOs, - cpu: SupportedCpu, - #[serde(skip_serializing_if = "Option::is_none")] - headers: Option>, - #[serde(rename = "type", skip_serializing_if = "Option::is_none")] - type_: Option, // TODO(mark): we should probably make this an enum -} - -#[derive(Clone, Serialize, Deserialize)] -struct PkgBinary { - url: String, - file: String, - sha256: String, - os: SupportedOs, - cpu: SupportedCpu, - #[serde(skip_serializing_if = "Option::is_none")] - headers: Option>, -} - -#[derive(Clone, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "lowercase")] -enum BinaryUnion { - File(FileBinary), - Archive(ArchiveBinary), - Pkg(PkgBinary), -} - -#[derive(Serialize, Deserialize)] -struct Binary { - binaries: Vec, -} - -impl Display for SupportedCpu { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match &self { - SupportedCpu::Arm64 => write!(f, "arm64"), - SupportedCpu::X86_64 => write!(f, "x86_64"), - } - } -} - -impl Display for SupportedOs { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match &self { - SupportedOs::Linux => write!(f, "linux"), - SupportedOs::MacOS => write!(f, "macos"), - } - } -} - trait Common { fn url(&self) -> &str; fn sort_key(&self) -> String; } -impl Common for BinaryUnion { +impl Common for Binary { fn url(&self) -> &str { match &self { - BinaryUnion::File(file) => &file.url, - BinaryUnion::Archive(archive) => &archive.url, - BinaryUnion::Pkg(pkg) => &pkg.url, + Binary::File(file) => &file.url, + Binary::Archive(archive) => &archive.url, + Binary::Pkg(pkg) => &pkg.url, } } fn sort_key(&self) -> String { match &self { - BinaryUnion::File(bin) => format!("{}_{}", bin.os, bin.cpu), - BinaryUnion::Archive(bin) => format!("{}_{}", bin.os, bin.cpu), - BinaryUnion::Pkg(bin) => format!("{}_{}", bin.os, bin.cpu), + Binary::File(bin) => format!("{}_{}", bin.os, bin.cpu), + Binary::Archive(bin) => format!("{}_{}", bin.os, bin.cpu), + Binary::Pkg(bin) => format!("{}_{}", bin.os, bin.cpu), } } } fn compute_sha256(client: &reqwest::blocking::Client, url: &str) -> Result> { - let bytes = client.get(url).send()?.bytes()?; + let response = client.get(url).send()?.error_for_status()?; + let bytes = response.bytes()?; Ok(sha256::digest(bytes.to_vec())) } fn update_github_release( client: &reqwest::blocking::Client, gh_latest_releases: &mut HashMap, - binary: &BinaryUnion, + binary: &Binary, org: &str, repo: &str, version: &str, path: &str, -) -> Result> { +) -> Result> { let key = format!("https://api.github.com/repos/{org}/{repo}/releases/latest"); let raw = gh_latest_releases.entry(key.clone()).or_insert_with(|| { client @@ -171,15 +108,17 @@ fn update_github_release( let sha256 = compute_sha256(client, &url)?; + println!("Updating {org}/{repo} from {version} to {latest}"); + Ok(match binary { - BinaryUnion::File(bin) => BinaryUnion::File(FileBinary { + Binary::File(bin) => Binary::File(FileBinary { url, cpu: bin.cpu.clone(), os: bin.os.clone(), sha256, headers: bin.headers.clone(), }), - BinaryUnion::Archive(bin) => BinaryUnion::Archive(ArchiveBinary { + Binary::Archive(bin) => Binary::Archive(ArchiveBinary { url, file: bin.file.replace(version, latest), cpu: bin.cpu.clone(), @@ -188,7 +127,7 @@ fn update_github_release( headers: bin.headers.clone(), type_: bin.type_.clone(), }), - BinaryUnion::Pkg(bin) => BinaryUnion::Pkg(PkgBinary { + Binary::Pkg(bin) => Binary::Pkg(PkgBinary { url, file: bin.file.replace(version, latest), cpu: bin.cpu.clone(), @@ -199,16 +138,12 @@ fn update_github_release( }) } -fn update_lockfile(lockfile: &std::path::Path) { - let contents = fs::read_to_string(lockfile).expect("Unable to load lockfile"); +fn update_lockfile(path: &std::path::Path) { + let contents = fs::read_to_string(path).expect("Unable to load lockfile"); - let tools: HashMap = + let lockfile: Lockfile = serde_json::from_str(&contents).expect("Unable to deserialize lockfile"); - let github_release_pattern = Regex::new( - r"https://github\.com/(?P[A-Za-z0-9_-]+)/(?P[A-Za-z0-9_-]+)/releases/download/(?Pv?[^/]+)/(?P.+)" - ).unwrap(); - let client = reqwest::blocking::Client::builder() .user_agent("multitool") .build() @@ -217,14 +152,15 @@ fn update_lockfile(lockfile: &std::path::Path) { // basic cache of latest release lookups let mut gh_latest_releases: HashMap = HashMap::new(); - let tools: BTreeMap = tools + let tools: BTreeMap = lockfile + .tools .into_iter() .map(|(tool, binary)| { - let mut binaries: Vec = binary + let mut binaries: Vec = binary .binaries .into_iter() .map( - |binary| match github_release_pattern.captures(binary.url()) { + |binary| match GITHUB_RELEASE_PATTERN.captures(binary.url()) { Some(cap) => { let (_, [org, repo, version, path]) = cap.extract(); update_github_release( @@ -248,12 +184,17 @@ fn update_lockfile(lockfile: &std::path::Path) { binaries.sort_by_key(|v| v.sort_key()); - (tool, Binary { binaries }) + (tool, ToolDefinition { binaries }) }) .collect(); - let contents = serde_json::to_string_pretty(&tools).unwrap(); - fs::write(lockfile, contents + "\n").expect("Error updating lockfile") + let lockfile = Lockfile { + schema: lockfile.schema, + tools, + }; + + let contents = serde_json::to_string_pretty(&lockfile).unwrap(); + fs::write(path, contents + "\n").expect("Error updating lockfile") } fn main() { From b7e3f79c4c4bbf9049455e70b2c2042f4a98658e Mon Sep 17 00:00:00 2001 From: Mark Elliot <123787712+mark-thm@users.noreply.github.com> Date: Mon, 27 May 2024 20:37:09 -0400 Subject: [PATCH 2/3] more --- src/lockfile.rs | 6 +- src/main.rs | 145 ++++++++++++++++++++++++++++++------------------ 2 files changed, 94 insertions(+), 57 deletions(-) diff --git a/src/lockfile.rs b/src/lockfile.rs index 9415826..d4c1feb 100644 --- a/src/lockfile.rs +++ b/src/lockfile.rs @@ -4,6 +4,9 @@ use std::{ fmt::Display, }; +pub const SCHEMA: &str = + "https://raw.githubusercontent.com/theoremlp/rules_multitool/main/lockfile.schema.json"; + #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum SupportedOs { @@ -84,8 +87,7 @@ impl Display for SupportedOs { } fn schema() -> String { - "https://raw.githubusercontent.com/theoremlp/rules_multitool/main/lockfile.schema.json" - .to_owned() + SCHEMA.to_owned() } #[derive(Serialize, Deserialize)] diff --git a/src/main.rs b/src/main.rs index 595c80a..9ab7395 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,5 @@ use clap::{Parser, Subcommand}; -use lockfile::{ArchiveBinary, Binary, FileBinary, Lockfile, PkgBinary, ToolDefinition}; +use lockfile::{ArchiveBinary, Binary, FileBinary, Lockfile, PkgBinary, ToolDefinition, SCHEMA}; use once_cell::sync::Lazy as LazyLock; use regex::Regex; use serde_json::Value; @@ -63,6 +63,27 @@ impl Common for Binary { } } +struct GitHubRelease<'a> { + org: &'a str, + repo: &'a str, + version: &'a str, + path: &'a str, +} + +impl GitHubRelease<'_> { + fn from(url: &str) -> Option { + GITHUB_RELEASE_PATTERN.captures(url).map(|capture| { + let (_, [org, repo, version, path]) = capture.extract(); + GitHubRelease { + org, + repo, + version, + path, + } + }) + } +} + fn compute_sha256(client: &reqwest::blocking::Client, url: &str) -> Result> { let response = client.get(url).send()?.error_for_status()?; let bytes = response.bytes()?; @@ -72,12 +93,13 @@ fn compute_sha256(client: &reqwest::blocking::Client, url: &str) -> Result, + tool: &str, binary: &Binary, - org: &str, - repo: &str, - version: &str, - path: &str, + release: &GitHubRelease, ) -> Result> { + let org = release.org; + let repo = release.repo; + let key = format!("https://api.github.com/repos/{org}/{repo}/releases/latest"); let raw = gh_latest_releases.entry(key.clone()).or_insert_with(|| { client @@ -93,48 +115,64 @@ fn update_github_release( .as_str() .unwrap_or_else(|| panic!("Failed to find tag_name in response:\n===\n{raw}\n===\n")); - if version == latest_tag { + if release.version == latest_tag { return Ok(binary.clone()); } - let version = version.strip_prefix('v').unwrap_or(version); + let version = release.version.strip_prefix('v').unwrap_or(release.version); let latest = latest_tag.strip_prefix('v').unwrap_or(latest_tag); let url = format!( "https://github.com/{org}/{repo}/releases/download/{latest_tag}/{0}", - path.replace(version, latest) + release.path.replace(version, latest) ); // TODO(mark): check that the new url is in .assets[].browser_download_url let sha256 = compute_sha256(client, &url)?; - println!("Updating {org}/{repo} from {version} to {latest}"); - Ok(match binary { - Binary::File(bin) => Binary::File(FileBinary { - url, - cpu: bin.cpu.clone(), - os: bin.os.clone(), - sha256, - headers: bin.headers.clone(), - }), - Binary::Archive(bin) => Binary::Archive(ArchiveBinary { - url, - file: bin.file.replace(version, latest), - cpu: bin.cpu.clone(), - os: bin.os.clone(), - sha256, - headers: bin.headers.clone(), - type_: bin.type_.clone(), - }), - Binary::Pkg(bin) => Binary::Pkg(PkgBinary { - url, - file: bin.file.replace(version, latest), - cpu: bin.cpu.clone(), - os: bin.os.clone(), - sha256, - headers: bin.headers.clone(), - }), + Binary::File(bin) => { + println!( + "Updating {tool} ({}/{}) from {version} to {latest}", + bin.os, bin.cpu + ); + Binary::File(FileBinary { + url, + cpu: bin.cpu.clone(), + os: bin.os.clone(), + sha256, + headers: bin.headers.clone(), + }) + } + Binary::Archive(bin) => { + println!( + "Updating {tool} ({}/{}) from {version} to {latest}", + bin.os, bin.cpu + ); + Binary::Archive(ArchiveBinary { + url, + file: bin.file.replace(version, latest), + cpu: bin.cpu.clone(), + os: bin.os.clone(), + sha256, + headers: bin.headers.clone(), + type_: bin.type_.clone(), + }) + } + Binary::Pkg(bin) => { + println!( + "Updating {tool} ({}/{}) from {version} to {latest}", + bin.os, bin.cpu + ); + Binary::Pkg(PkgBinary { + url, + file: bin.file.replace(version, latest), + cpu: bin.cpu.clone(), + os: bin.os.clone(), + sha256, + headers: bin.headers.clone(), + }) + } }) } @@ -144,6 +182,10 @@ fn update_lockfile(path: &std::path::Path) { let lockfile: Lockfile = serde_json::from_str(&contents).expect("Unable to deserialize lockfile"); + if lockfile.schema != SCHEMA { + panic!("Unsupported lockfile schema {}", lockfile.schema) + } + let client = reqwest::blocking::Client::builder() .user_agent("multitool") .build() @@ -159,27 +201,20 @@ fn update_lockfile(path: &std::path::Path) { let mut binaries: Vec = binary .binaries .into_iter() - .map( - |binary| match GITHUB_RELEASE_PATTERN.captures(binary.url()) { - Some(cap) => { - let (_, [org, repo, version, path]) = cap.extract(); - update_github_release( - &client, - &mut gh_latest_releases, - &binary, - org, - repo, - version, - path, - ) - .map_err(|e| { - println!("Encountered error while attempting to update {tool}: {e}") - }) - .unwrap_or(binary) - } - None => binary, - }, - ) + .map(|binary| match GitHubRelease::from(binary.url()) { + Some(release) => update_github_release( + &client, + &mut gh_latest_releases, + &tool, + &binary, + &release, + ) + .map_err(|e| { + println!("Encountered error while attempting to update {tool}: {e}") + }) + .unwrap_or(binary), + None => binary, + }) .collect(); binaries.sort_by_key(|v| v.sort_key()); From 4de5c8743e31d44b19fb458ad064568a71abf1b6 Mon Sep 17 00:00:00 2001 From: Mark Elliot <123787712+mark-thm@users.noreply.github.com> Date: Mon, 27 May 2024 20:40:10 -0400 Subject: [PATCH 3/3] more --- src/lockfile.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lockfile.rs b/src/lockfile.rs index d4c1feb..24b23fc 100644 --- a/src/lockfile.rs +++ b/src/lockfile.rs @@ -141,7 +141,8 @@ mod tests { }"#).unwrap(); assert_eq!( lockfile.schema, - Some("https://raw.githubusercontent.com/theoremlp/rules_multitool/main/lockfile.schema.json".to_owned()) + "https://raw.githubusercontent.com/theoremlp/rules_multitool/main/lockfile.schema.json" + .to_owned() ); assert_eq!(lockfile.tools.len(), 1); assert_eq!(lockfile.tools["tool-name"].binaries.len(), 1);