From 09e858d93971ccc768080fd5b658ecb7c8570908 Mon Sep 17 00:00:00 2001 From: Pankaj Garg Date: Fri, 13 Oct 2023 07:54:37 -0700 Subject: [PATCH] Handle cargo registry index lookup requests (#33681) --- Cargo.lock | 82 +-------------------- Cargo.toml | 2 - cargo-registry/Cargo.toml | 3 +- cargo-registry/src/main.rs | 20 +++-- cargo-registry/src/publisher.rs | 81 ++++++++++++--------- cargo-registry/src/sparse_index.rs | 113 ++++++++++++++++++++++++++--- 6 files changed, 165 insertions(+), 136 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c0261886a6e459..ffec7fbc909432 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2209,21 +2209,6 @@ version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" -[[package]] -name = "git2" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbf97ba92db08df386e10c8ede66a2a0369bd277090afd8710e19e38de9ec0cd" -dependencies = [ - "bitflags 2.3.3", - "libc", - "libgit2-sys", - "log", - "openssl-probe", - "openssl-sys", - "url 2.4.1", -] - [[package]] name = "glob" version = "0.3.0" @@ -2462,12 +2447,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "http-range" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21dec9db110f5f872ed9699c3ecf50cf16f423502706ba5c72462e28d3157573" - [[package]] name = "httparse" version = "1.8.0" @@ -2542,25 +2521,6 @@ dependencies = [ "tokio-rustls", ] -[[package]] -name = "hyper-staticfile" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "318ca89e4827e7fe4ddd2824f52337239796ae8ecc761a663324407dc3d8d7e7" -dependencies = [ - "futures-util", - "http", - "http-range", - "httpdate", - "hyper", - "mime_guess", - "percent-encoding 2.3.0", - "rand 0.8.5", - "tokio", - "url 2.4.1", - "winapi 0.3.9", -] - [[package]] name = "hyper-timeout" version = "0.4.1" @@ -2912,20 +2872,6 @@ version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" -[[package]] -name = "libgit2-sys" -version = "0.16.1+1.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2a2bb3680b094add03bb3732ec520ece34da31a8cd2d633d1389d0f0fb60d0c" -dependencies = [ - "cc", - "libc", - "libssh2-sys", - "libz-sys", - "openssl-sys", - "pkg-config", -] - [[package]] name = "libloading" version = "0.7.4" @@ -3005,20 +2951,6 @@ dependencies = [ "libsecp256k1-core", ] -[[package]] -name = "libssh2-sys" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dc8a030b787e2119a731f1951d6a773e2280c660f8ec4b0f5e1505a386e71ee" -dependencies = [ - "cc", - "libc", - "libz-sys", - "openssl-sys", - "pkg-config", - "vcpkg", -] - [[package]] name = "libz-sys" version = "1.1.3" @@ -3026,7 +2958,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66" dependencies = [ "cc", - "libc", "pkg-config", "vcpkg", ] @@ -3182,16 +3113,6 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" -[[package]] -name = "mime_guess" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" -dependencies = [ - "mime", - "unicase", -] - [[package]] name = "min-max-heap" version = "1.3.0" @@ -5565,13 +5486,12 @@ version = "1.18.0" dependencies = [ "clap 2.33.3", "flate2", - "git2", "hyper", - "hyper-staticfile", "log", "rustc_version 0.4.0", "serde", "serde_json", + "sha2 0.10.8", "solana-clap-utils", "solana-cli", "solana-cli-config", diff --git a/Cargo.toml b/Cargo.toml index c63b62a47c9782..f26209e5104929 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -206,7 +206,6 @@ gag = "1.0.0" generic-array = { version = "0.14.7", default-features = false } gethostname = "0.2.3" getrandom = "0.2.10" -git2 = "0.18.1" goauth = "0.13.1" hex = "0.4.3" hidapi = { version = "2.4.1", default-features = false } @@ -216,7 +215,6 @@ http = "0.2.9" humantime = "2.0.1" hyper = "0.14.27" hyper-proxy = "0.9.1" -hyper-staticfile = "0.9.5" im = "15.1.0" index_list = "0.2.7" indexmap = "2.0.2" diff --git a/cargo-registry/Cargo.toml b/cargo-registry/Cargo.toml index 43aed1f4fa2097..afc5bf363b0bab 100644 --- a/cargo-registry/Cargo.toml +++ b/cargo-registry/Cargo.toml @@ -12,12 +12,11 @@ edition = { workspace = true } [dependencies] clap = { workspace = true } flate2 = { workspace = true } -git2 = { workspace = true } hyper = { workspace = true, features = ["full"] } -hyper-staticfile = { workspace = true } log = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +sha2 = { workspace = true } solana-clap-utils = { workspace = true } solana-cli = { workspace = true } solana-cli-config = { workspace = true } diff --git a/cargo-registry/src/main.rs b/cargo-registry/src/main.rs index 60227fa32a9962..4ba61c917969b7 100644 --- a/cargo-registry/src/main.rs +++ b/cargo-registry/src/main.rs @@ -3,6 +3,7 @@ use { crate::{ client::Client, publisher::{Error, Publisher}, + sparse_index::RegistryIndex, }, hyper::{ body, @@ -30,15 +31,17 @@ impl CargoRegistryService { async fn handle_publish_request( request: hyper::Request, client: Arc, + index: Arc, ) -> hyper::Response { info!("Handling request to publish the crate"); let bytes = body::to_bytes(request.into_body()).await; match bytes { Ok(data) => { - let Ok(result) = - tokio::task::spawn_blocking(move || Publisher::publish_crate(data, client)) - .await + let Ok(result) = tokio::task::spawn_blocking(move || { + Publisher::publish_crate(data, client, index) + }) + .await else { return response_builder::error_response( hyper::StatusCode::INTERNAL_SERVER_ERROR, @@ -218,7 +221,7 @@ impl CargoRegistryService { } async fn handler( - index: sparse_index::RegistryIndex, + index: Arc, request: hyper::Request, client: Arc, ) -> Result, Error> { @@ -257,7 +260,7 @@ impl CargoRegistryService { "Invalid length of the request.", ) } else { - Self::handle_publish_request(request, client.clone()).await + Self::handle_publish_request(request, client.clone(), index.clone()).await } } "unyank" => Self::handle_unyank_request(path, &request), @@ -297,7 +300,10 @@ async fn main() { let client = Arc::new(Client::new().expect("Failed to get RPC Client instance")); let bind_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), client.port); - let index = sparse_index::RegistryIndex::new("/index", &client.server_url); + let index = Arc::new(sparse_index::RegistryIndex::new( + "/index", + &client.server_url, + )); let registry_service = make_service_fn(move |_| { let client_inner = client.clone(); @@ -310,7 +316,7 @@ async fn main() { }); let server = Server::bind(&bind_addr).serve(registry_service); - info!("Server running on on http://{}", bind_addr); + info!("Server running on http://{}", bind_addr); let _ = server.await; } diff --git a/cargo-registry/src/publisher.rs b/cargo-registry/src/publisher.rs index a712da35895e1c..5940191b46dcaf 100644 --- a/cargo-registry/src/publisher.rs +++ b/cargo-registry/src/publisher.rs @@ -1,10 +1,14 @@ use { - crate::client::{Client, ClientConfig}, + crate::{ + client::{Client, ClientConfig}, + sparse_index::{IndexEntry, RegistryIndex}, + }, flate2::read::GzDecoder, hyper::body::Bytes, log::*, serde::{Deserialize, Serialize}, serde_json::from_slice, + sha2::{Digest, Sha256}, solana_cli::program_v4::{process_deploy_program, read_and_verify_elf}, solana_sdk::{ signature::{Keypair, Signer}, @@ -22,11 +26,11 @@ use { tempfile::{tempdir, TempDir}, }; -pub type Error = Box; +pub(crate) type Error = Box; #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "lowercase")] -enum DependencyType { +pub(crate) enum DependencyType { Dev, Build, Normal, @@ -34,39 +38,39 @@ enum DependencyType { #[allow(dead_code)] #[derive(Debug, Deserialize)] -struct Dependency { - name: String, - version_req: String, - features: Vec, - optional: bool, - default_features: bool, - target: Option, - kind: DependencyType, - registry: Option, - explicit_name_in_toml: Option, +pub(crate) struct Dependency { + pub name: String, + pub version_req: String, + pub features: Vec, + pub optional: bool, + pub default_features: bool, + pub target: Option, + pub kind: DependencyType, + pub registry: Option, + pub explicit_name_in_toml: Option, } #[derive(Debug, Deserialize)] #[allow(unused)] -struct PackageMetaData { - name: String, - vers: String, - deps: Vec, - features: BTreeMap>, - authors: Vec, - description: Option, - documentation: Option, - homepage: Option, - readme: Option, - readme_file: Option, - keywords: Vec, - categories: Vec, - license: Option, - license_file: Option, - repository: Option, - badges: BTreeMap>, - links: Option, - rust_version: Option, +pub(crate) struct PackageMetaData { + pub name: String, + pub vers: String, + pub deps: Vec, + pub features: BTreeMap>, + pub authors: Vec, + pub description: Option, + pub documentation: Option, + pub homepage: Option, + pub readme: Option, + pub readme_file: Option, + pub keywords: Vec, + pub categories: Vec, + pub license: Option, + pub license_file: Option, + pub repository: Option, + pub badges: BTreeMap>, + pub links: Option, + pub rust_version: Option, } impl PackageMetaData { @@ -86,7 +90,7 @@ impl PackageMetaData { } } -pub struct Publisher {} +pub(crate) struct Publisher {} impl Publisher { fn make_path>(tempdir: &TempDir, meta: &PackageMetaData, append: P) -> PathBuf { @@ -107,12 +111,17 @@ impl Publisher { Ok(library_name.to_string()) } - pub(crate) fn publish_crate(bytes: Bytes, client: Arc) -> Result<(), Error> { + pub(crate) fn publish_crate( + bytes: Bytes, + client: Arc, + index: Arc, + ) -> Result<(), Error> { let (meta_data, offset) = PackageMetaData::new(&bytes)?; let (_crate_file_length, length_size) = PackageMetaData::read_u32_length(&bytes.slice(offset..))?; let crate_bytes = bytes.slice(offset.saturating_add(length_size)..); + let crate_cksum = format!("{:x}", Sha256::digest(&crate_bytes)); let decoder = GzDecoder::new(crate_bytes.as_ref()); let mut archive = Archive::new(decoder); @@ -154,6 +163,10 @@ impl Publisher { format!("Failed to deploy the program: {}", e) })?; + let mut entry: IndexEntry = meta_data.into(); + entry.cksum = crate_cksum; + index.insert_entry(entry)?; + info!("Successfully deployed the program"); Ok(()) } diff --git a/cargo-registry/src/sparse_index.rs b/cargo-registry/src/sparse_index.rs index 59b9b88985c445..e29a581c1c7819 100644 --- a/cargo-registry/src/sparse_index.rs +++ b/cargo-registry/src/sparse_index.rs @@ -1,7 +1,11 @@ use { - crate::response_builder, + crate::{ + publisher::{Dependency, Error, PackageMetaData}, + response_builder, + }, log::info, serde::{Deserialize, Serialize}, + std::{collections::BTreeMap, sync::RwLock}, }; #[derive(Debug, Default, Deserialize, Serialize)] @@ -10,14 +14,68 @@ struct RegistryConfig { api: Option, } -#[derive(Clone)] -pub struct RegistryIndex { +pub(crate) struct RegistryIndex { pub(crate) index_root: String, config: String, + index: RwLock>, +} + +#[derive(Serialize)] +pub(crate) struct IndexEntryDep { + pub name: String, + pub req: String, + pub features: Vec, + pub optional: bool, + pub default_features: bool, + pub target: Option, + pub kind: String, + pub registry: Option, + pub package: Option, +} + +impl From for IndexEntryDep { + fn from(v: Dependency) -> Self { + IndexEntryDep { + name: v.name, + req: v.version_req, + features: v.features, + optional: v.optional, + default_features: v.default_features, + target: v.target, + kind: serde_json::to_string(&v.kind).expect("Failed to stringify dep kind"), + registry: v.registry, + package: None, + } + } +} + +#[derive(Serialize)] +pub(crate) struct IndexEntry { + pub name: String, + pub vers: String, + pub deps: Vec, + pub cksum: String, + pub features: BTreeMap>, + pub yanked: bool, + pub links: Option, +} + +impl From for IndexEntry { + fn from(v: PackageMetaData) -> Self { + IndexEntry { + name: v.name, + vers: v.vers, + deps: v.deps.into_iter().map(|v| v.into()).collect(), + cksum: String::new(), + features: v.features, + yanked: false, + links: v.links, + } + } } impl RegistryIndex { - pub fn new(root: &str, server_url: &str) -> Self { + pub(crate) fn new(root: &str, server_url: &str) -> Self { let registry_config = RegistryConfig { dl: format!("{}/api/v1/crates", server_url), api: Some(server_url.to_string()), @@ -29,10 +87,14 @@ impl RegistryIndex { Self { index_root: root.to_string(), config, + index: RwLock::new(BTreeMap::new()), } } - pub fn handler(&self, request: hyper::Request) -> hyper::Response { + pub(crate) fn handler( + &self, + request: hyper::Request, + ) -> hyper::Response { let path = request.uri().path(); let expected_root = self.index_root.as_str(); if !path.starts_with(expected_root) { @@ -53,7 +115,17 @@ impl RegistryIndex { return response_builder::success_response_str(&self.config); } - Self::handle_crate_lookup_request(path) + self.handle_crate_lookup_request(path) + } + + pub(crate) fn insert_entry(&self, entry: IndexEntry) -> Result<(), Error> { + let mut write_index = self + .index + .write() + .map_err(|e| format!("Failed to lock the index for writing: {}", e))?; + info!("Inserting {}-{} in registry index", entry.name, entry.vers); + write_index.insert(entry.name.clone(), entry); + Ok(()) } fn get_crate_name_from_path(path: &str) -> Option<&str> { @@ -78,7 +150,7 @@ impl RegistryIndex { .then_some(crate_name) } - fn handle_crate_lookup_request(path: &str) -> hyper::Response { + fn handle_crate_lookup_request(&self, path: &str) -> hyper::Response { let Some(crate_name) = Self::get_crate_name_from_path(path) else { return response_builder::error_response( hyper::StatusCode::BAD_REQUEST, @@ -86,10 +158,31 @@ impl RegistryIndex { ); }; - // Fetch the index information for the crate - info!("Received a request to fetch {:?}", crate_name); + info!("Looking up index for {:?}", crate_name); + + let Ok(read_index) = self.index.read() else { + return response_builder::error_response( + hyper::StatusCode::INTERNAL_SERVER_ERROR, + "Internal error. Failed to lock the index for reading", + ); + }; + + let Some(entry) = read_index.get(crate_name) else { + // The index currently doesn't contain the program entry. + // Fetch the program information from the network using RPC client. + // In the meanwhile, return empty success response, so that the registry + // client continues to poll us for the index information. + return response_builder::success_response(); + }; + + let Ok(response) = serde_json::to_string(entry) else { + return response_builder::error_response( + hyper::StatusCode::INTERNAL_SERVER_ERROR, + "Internal error. index entry is corrupted", + ); + }; - response_builder::success_response() + response_builder::success_response_str(response.as_str()) } }