From 25bef4aa20ac6bf6c8e2af55d5bb7b4055e87e10 Mon Sep 17 00:00:00 2001 From: Nathan Bruer Date: Sat, 2 Jan 2021 03:04:13 -0800 Subject: [PATCH] Add DigestInfo utility --- Cargo.lock | 122 ++------ Cargo.toml | 2 +- cas/grpc_service/BUILD | 8 +- cas/grpc_service/ac_server.rs | 28 +- cas/grpc_service/bytestream_server.rs | 5 +- cas/grpc_service/cas_server.rs | 38 +-- cas/grpc_service/tests/ac_server_test.rs | 14 +- .../tests/bytestream_server_test.rs | 10 +- cas/grpc_service/tests/cas_server_test.rs | 46 ++- cas/store/BUILD | 5 +- cas/store/memory_store.rs | 32 +- cas/store/store_trait.rs | 9 +- cas/store/tests/memory_store_test.rs | 46 ++- third_party/BUILD.bazel | 18 +- third_party/crates.bzl | 70 +---- .../remote/BUILD.async-stream-0.3.0.bazel | 65 ---- third_party/remote/BUILD.bytes-1.0.0.bazel | 83 ----- ....3.0.bazel => BUILD.lazy-init-0.4.0.bazel} | 13 +- third_party/remote/BUILD.tokio-1.0.1.bazel | 292 ------------------ .../remote/BUILD.tokio-stream-0.1.0.bazel | 83 ----- .../remote/BUILD.tokio-test-0.4.0.bazel | 64 ---- util/BUILD | 12 + util/common.rs | 77 +++++ 23 files changed, 286 insertions(+), 856 deletions(-) delete mode 100644 third_party/remote/BUILD.async-stream-0.3.0.bazel delete mode 100644 third_party/remote/BUILD.bytes-1.0.0.bazel rename third_party/remote/{BUILD.async-stream-impl-0.3.0.bazel => BUILD.lazy-init-0.4.0.bazel} (76%) delete mode 100644 third_party/remote/BUILD.tokio-1.0.1.bazel delete mode 100644 third_party/remote/BUILD.tokio-stream-0.1.0.bazel delete mode 100644 third_party/remote/BUILD.tokio-test-0.4.0.bazel create mode 100644 util/common.rs diff --git a/Cargo.lock b/Cargo.lock index 7a0e5244b..ab69630e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -66,17 +66,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22068c0c19514942eefcfd4daf8976ef1aad84e61539f95cd200c35202f80af5" dependencies = [ - "async-stream-impl 0.2.1", - "futures-core", -] - -[[package]] -name = "async-stream" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3670df70cbc01729f901f94c887814b3c68db038aad1329a418bae178bc5295c" -dependencies = [ - "async-stream-impl 0.3.0", + "async-stream-impl", "futures-core", ] @@ -91,17 +81,6 @@ dependencies = [ "syn", ] -[[package]] -name = "async-stream-impl" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "async-trait" version = "0.1.42" @@ -213,12 +192,6 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" -[[package]] -name = "bytes" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1f8e949d755f9d79112b5bb46938e0ef9d3804a0b16dfab13aafcaa5f0fa72" - [[package]] name = "cargo_metadata" version = "0.8.2" @@ -415,19 +388,19 @@ version = "0.0.0" dependencies = [ "async-mutex", "async-trait", - "bytes 0.5.6", + "bytes", "clap", "fixed-buffer", "futures", "hex", + "lazy-init", "pretty_assertions", "prost", "prost-build", "prost-types", "rustfmt-nightly", "stdext", - "tokio 0.2.24", - "tokio-test", + "tokio", "tonic", "tonic-build", ] @@ -649,7 +622,7 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" dependencies = [ - "bytes 0.5.6", + "bytes", "fnv", "futures-core", "futures-sink", @@ -657,7 +630,7 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 0.2.24", + "tokio", "tokio-util", "tracing", "tracing-futures", @@ -699,7 +672,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84129d298a6d57d246960ff8eb831ca4af3f96d29e2e28848dae275408658e26" dependencies = [ - "bytes 0.5.6", + "bytes", "fnv", "itoa", ] @@ -710,7 +683,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" dependencies = [ - "bytes 0.5.6", + "bytes", "http", ] @@ -741,7 +714,7 @@ version = "0.13.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf" dependencies = [ - "bytes 0.5.6", + "bytes", "futures-channel", "futures-core", "futures-util", @@ -753,7 +726,7 @@ dependencies = [ "itoa", "pin-project 1.0.2", "socket2", - "tokio 0.2.24", + "tokio", "tower-service", "tracing", "want", @@ -830,6 +803,12 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "lazy-init" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671025f3cc9b5d5188eca51493d8097e6cd1dc3828e0a59f7befbaaf56e930ab" + [[package]] name = "lazy_static" version = "1.4.0" @@ -1186,7 +1165,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce49aefe0a6144a45de32927c77bd2859a5f7677b55f220ae5b744e87389c212" dependencies = [ - "bytes 0.5.6", + "bytes", "prost-derive", ] @@ -1196,7 +1175,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02b10678c913ecbd69350e8535c3aef91a8676c0773fc1d7b95cdd196d7f2f26" dependencies = [ - "bytes 0.5.6", + "bytes", "heck", "itertools", "log", @@ -1227,7 +1206,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1834f67c0697c001304b75be76f67add9c89742eda3a085ad8ee0bb38c3417aa" dependencies = [ - "bytes 0.5.6", + "bytes", "prost", ] @@ -2048,7 +2027,7 @@ version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48" dependencies = [ - "bytes 0.5.6", + "bytes", "fnv", "futures-core", "iovec", @@ -2060,16 +2039,6 @@ dependencies = [ "tokio-macros", ] -[[package]] -name = "tokio" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d258221f566b6c803c7b4714abadc080172b272090cdc5e244a6d4dd13c3a6bd" -dependencies = [ - "autocfg", - "pin-project-lite 0.2.0", -] - [[package]] name = "tokio-macros" version = "0.2.6" @@ -2081,43 +2050,18 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-stream" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3be913b74b13210c8fe04b17ab833f5a124f45b93d0f99f59fff621f64392a" -dependencies = [ - "async-stream 0.3.0", - "futures-core", - "pin-project-lite 0.2.0", - "tokio 1.0.1", -] - -[[package]] -name = "tokio-test" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c7d205f6f59b03f9e824ac86eaba635a98395f287756ecc8a06464779c399bf" -dependencies = [ - "async-stream 0.3.0", - "bytes 1.0.0", - "futures-core", - "tokio 1.0.1", - "tokio-stream", -] - [[package]] name = "tokio-util" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" dependencies = [ - "bytes 0.5.6", + "bytes", "futures-core", "futures-sink", "log", "pin-project-lite 0.1.11", - "tokio 0.2.24", + "tokio", ] [[package]] @@ -2135,10 +2079,10 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74a5d6e7439ecf910463667080de772a9c7ddf26bc9fb4f3252ac3862e43337d" dependencies = [ - "async-stream 0.2.1", + "async-stream", "async-trait", "base64 0.12.3", - "bytes 0.5.6", + "bytes", "futures-core", "futures-util", "http", @@ -2148,7 +2092,7 @@ dependencies = [ "pin-project 0.4.27", "prost", "prost-derive", - "tokio 0.2.24", + "tokio", "tokio-util", "tower", "tower-balance", @@ -2201,7 +2145,7 @@ dependencies = [ "pin-project 0.4.27", "rand", "slab", - "tokio 0.2.24", + "tokio", "tower-discover", "tower-layer", "tower-load", @@ -2219,7 +2163,7 @@ checksum = "c4887dc2a65d464c8b9b66e0e4d51c2fd6cf5b3373afc72805b0a60bce00446a" dependencies = [ "futures-core", "pin-project 0.4.27", - "tokio 0.2.24", + "tokio", "tower-layer", "tower-service", "tracing", @@ -2250,7 +2194,7 @@ checksum = "92c3040c5dbed68abffaa0d4517ac1a454cd741044f33ab0eefab6b8d1361404" dependencies = [ "futures-core", "pin-project 0.4.27", - "tokio 0.2.24", + "tokio", "tower-layer", "tower-load", "tower-service", @@ -2265,7 +2209,7 @@ dependencies = [ "futures-core", "log", "pin-project 0.4.27", - "tokio 0.2.24", + "tokio", "tower-discover", "tower-service", ] @@ -2288,7 +2232,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce50370d644a0364bf4877ffd4f76404156a248d104e2cc234cd391ea5cdc965" dependencies = [ - "tokio 0.2.24", + "tokio", "tower-service", ] @@ -2302,7 +2246,7 @@ dependencies = [ "futures-util", "indexmap", "log", - "tokio 0.2.24", + "tokio", "tower-service", ] @@ -2314,7 +2258,7 @@ checksum = "e6727956aaa2f8957d4d9232b308fe8e4e65d99db30f42b225646e86c9b6a952" dependencies = [ "futures-core", "pin-project 0.4.27", - "tokio 0.2.24", + "tokio", "tower-layer", "tower-service", ] @@ -2332,7 +2276,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "127b8924b357be938823eaaec0608c482d40add25609481027b96198b2e4b31e" dependencies = [ "pin-project 0.4.27", - "tokio 0.2.24", + "tokio", "tower-layer", "tower-service", ] diff --git a/Cargo.toml b/Cargo.toml index fd3e0178a..df8839a39 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ futures = "0.3.8" # We must use tokio 0.2.x because tonic runtime uses it. tokio = { version = "0.2", features = ["macros"] } tonic = "0.3.1" -tokio-test = "0.4.0" +lazy-init = "0.4.0" [dev-dependencies] clap = "2.33.3" diff --git a/cas/grpc_service/BUILD b/cas/grpc_service/BUILD index a6be91ad5..dc45d3c11 100644 --- a/cas/grpc_service/BUILD +++ b/cas/grpc_service/BUILD @@ -10,6 +10,7 @@ rust_library( "//third_party:futures", "//third_party:stdext", "//cas/store", + "//util:common", "//util:error", ], visibility = ["//cas:__pkg__"] @@ -22,6 +23,7 @@ rust_library( "//proto", "//cas/store", "//util:error", + "//util:common", "//third_party:stdext", "//third_party:prost", "//third_party:tonic", @@ -47,10 +49,11 @@ rust_library( "//proto", "//third_party:tonic", "//util:async_fixed_buffer", + "//util:common", + "//util:error", "//cas/store", "//third_party:tokio", "//third_party:futures", - "//util:error", ], visibility = ["//cas:__pkg__"] ) @@ -74,6 +77,7 @@ rust_test( ":cas_server", "//cas/store", "//util:error", + "//util:common", "//proto", "//third_party:tonic", "//third_party:tokio", @@ -88,6 +92,7 @@ rust_test( ":ac_server", "//cas/store", "//proto", + "//util:common", "//third_party:tonic", "//third_party:tokio", "//third_party:prost", @@ -103,6 +108,7 @@ rust_test( "//cas/store", "//proto", "//util:error", + "//util:common", "//third_party:tonic", "//third_party:bytes", "//third_party:tokio", diff --git a/cas/grpc_service/ac_server.rs b/cas/grpc_service/ac_server.rs index fbabff8b3..1a4f67cb4 100644 --- a/cas/grpc_service/ac_server.rs +++ b/cas/grpc_service/ac_server.rs @@ -1,6 +1,7 @@ // Copyright 2020 Nathan (Blaise) Bruer. All rights reserved. use std::convert::TryFrom; +use std::convert::TryInto; use std::io::Cursor; use std::sync::Arc; @@ -12,6 +13,7 @@ use proto::build::bazel::remote::execution::v2::{ ActionResult, GetActionResultRequest, UpdateActionResultRequest, }; +use common::DigestInfo; use error::{error_if, make_err, Code, Error, ResultExt}; use store::Store; @@ -40,18 +42,15 @@ impl AcServer { // TODO(blaise.bruer) This needs to be fixed. It is using wrong macro. // We also should write a test for these errors. - let digest = get_action_request + let digest: DigestInfo = get_action_request .action_digest - .err_tip(|| "Action digest was not set in message")?; - let size_bytes = usize::try_from(digest.size_bytes) - .err_tip(|| "Digest size_bytes was not convertable to usize")?; + .err_tip(|| "Action digest was not set in message")? + .try_into()?; // TODO(allada) There is a security risk here of someone taking all the memory on the instance. - let mut store_data = Vec::with_capacity(size_bytes); + let mut store_data = Vec::with_capacity(digest.size_bytes as usize); let mut cursor = Cursor::new(&mut store_data); - self.ac_store - .get(&digest.hash, size_bytes, &mut cursor) - .await?; + self.ac_store.get(&digest, &mut cursor).await?; let action_result = ActionResult::decode(Cursor::new(&store_data)).err_tip_with_code(|e| { @@ -61,7 +60,7 @@ impl AcServer { ) })?; - if store_data.len() != size_bytes { + if store_data.len() != digest.size_bytes as usize { return Err(make_err!( Code::NotFound, "Found item, but size does not match" @@ -78,9 +77,10 @@ impl AcServer { // TODO(blaise.bruer) This needs to be fixed. It is using wrong macro. // We also should write a test for these errors. - let digest = update_action_request + let digest: DigestInfo = update_action_request .action_digest - .err_tip(|| "Action digest was not set in message")?; + .err_tip(|| "Action digest was not set in message")? + .try_into()?; let size_bytes = usize::try_from(digest.size_bytes) .err_tip(|| "Digest size_bytes was not convertable to usize")?; @@ -101,11 +101,7 @@ impl AcServer { size_bytes ); self.ac_store - .update( - &digest.hash, - store_data.len(), - Box::new(Cursor::new(store_data)), - ) + .update(&digest, Box::new(Cursor::new(store_data))) .await?; Ok(Response::new(action_result)) } diff --git a/cas/grpc_service/bytestream_server.rs b/cas/grpc_service/bytestream_server.rs index c92eb5624..f641e5abb 100644 --- a/cas/grpc_service/bytestream_server.rs +++ b/cas/grpc_service/bytestream_server.rs @@ -14,6 +14,7 @@ use proto::google::bytestream::{ WriteResponse, }; +use common::DigestInfo; use error::{error_if, Error, ResultExt}; use store::Store; @@ -53,7 +54,9 @@ impl ByteStreamServer { let expected_size = stream.expected_size; tokio::spawn(async move { let rx = Box::new(rx.take(expected_size as u64)); - store.update(&hash, expected_size, rx).await + store + .update(&DigestInfo::try_new(&hash, expected_size)?, rx) + .await }) }; diff --git a/cas/grpc_service/cas_server.rs b/cas/grpc_service/cas_server.rs index aff41d2e0..dfb371ee2 100644 --- a/cas/grpc_service/cas_server.rs +++ b/cas/grpc_service/cas_server.rs @@ -1,6 +1,7 @@ // Copyright 2020 Nathan (Blaise) Bruer. All rights reserved. use std::convert::TryFrom; +use std::convert::TryInto; use std::io::Cursor; use std::pin::Pin; use std::sync::Arc; @@ -8,8 +9,6 @@ use std::sync::Arc; use futures::{stream::Stream, FutureExt, StreamExt}; use tonic::{Request, Response, Status}; -use error::{error_if, Error, ResultExt}; - use proto::build::bazel::remote::execution::v2::{ batch_read_blobs_response, batch_update_blobs_response, content_addressable_storage_server::ContentAddressableStorage, @@ -19,6 +18,9 @@ use proto::build::bazel::remote::execution::v2::{ GetTreeResponse, }; use proto::google::rpc::Status as GrpcStatus; + +use common::DigestInfo; +use error::{error_if, Error, ResultExt}; use store::Store; pub struct CasServer { @@ -41,21 +43,19 @@ impl CasServer { let mut futures = futures::stream::FuturesOrdered::new(); for digest in request.into_inner().blob_digests.into_iter() { let store = self.store.clone(); + let digest: DigestInfo = digest.try_into()?; futures.push(tokio::spawn(async move { - store - .has(&digest.hash, digest.hash.len()) - .await - .map_or_else( - |_| None, - |success| if success { None } else { Some(digest) }, - ) + store.has(&digest).await.map_or_else( + |_| None, + |success| if success { None } else { Some(digest) }, + ) })); } let mut responses = Vec::with_capacity(futures.len()); while let Some(result) = futures.next().await { let val = result.err_tip(|| "Internal error joining future")?; if let Some(digest) = val { - responses.push(digest); + responses.push(digest.into()); } } Ok(Response::new(FindMissingBlobsResponse { @@ -69,7 +69,10 @@ impl CasServer { ) -> Result, Error> { let mut futures = futures::stream::FuturesOrdered::new(); for request in grpc_request.into_inner().requests { - let digest = request.digest.err_tip(|| "Digest not found in request")?; + let digest: DigestInfo = request + .digest + .err_tip(|| "Digest not found in request")? + .try_into()?; let digest_copy = digest.clone(); let store = self.store.clone(); let request_data = request.data; @@ -85,12 +88,12 @@ impl CasServer { ); let cursor = Box::new(Cursor::new(request_data)); store - .update(&digest_copy.hash, size_bytes, cursor) + .update(&digest_copy, cursor) .await .err_tip(|| "Error writing to store") } .map(|result| batch_update_blobs_response::Response { - digest: Some(digest), + digest: Some(digest.into()), status: Some(result.map_or_else(|e| e.into(), |_| GrpcStatus::default())), }), )); @@ -111,6 +114,7 @@ impl CasServer { let mut futures = futures::stream::FuturesOrdered::new(); for digest in grpc_request.into_inner().digests { + let digest: DigestInfo = digest.try_into()?; let digest_copy = digest.clone(); let store = self.store.clone(); @@ -121,11 +125,7 @@ impl CasServer { // TODO(allada) There is a security risk here of someone taking all the memory on the instance. let mut store_data = Vec::with_capacity(size_bytes); store - .get( - &digest_copy.hash, - size_bytes, - &mut Cursor::new(&mut store_data), - ) + .get(&digest_copy, &mut Cursor::new(&mut store_data)) .await .err_tip(|| "Error reading from store")?; Ok(store_data) @@ -135,7 +135,7 @@ impl CasServer { result.map_or_else(|e| (e.into(), vec![]), |v| (GrpcStatus::default(), v)); batch_read_blobs_response::Response { status: Some(status), - digest: Some(digest), + digest: Some(digest.into()), data: data, } }), diff --git a/cas/grpc_service/tests/ac_server_test.rs b/cas/grpc_service/tests/ac_server_test.rs index 4f7ed31e8..dda361a56 100644 --- a/cas/grpc_service/tests/ac_server_test.rs +++ b/cas/grpc_service/tests/ac_server_test.rs @@ -10,6 +10,7 @@ use proto::build::bazel::remote::execution::v2::{ }; use ac_server::AcServer; +use common::DigestInfo; use store::{create_store, Store, StoreType}; const INSTANCE_NAME: &str = "foo"; @@ -22,11 +23,11 @@ async fn insert_into_store( ) -> Result> { let mut store_data = Vec::new(); action_result.encode(&mut store_data)?; - let digest_size = store_data.len() as i64; + let digest = DigestInfo::try_new(&hash, store_data.len() as i64)?; store - .update(&hash, store_data.len(), Box::new(Cursor::new(store_data))) + .update(&digest, Box::new(Cursor::new(store_data))) .await?; - Ok(digest_size) + Ok(digest.size_bytes as i64) } #[cfg(test)] @@ -146,13 +147,13 @@ mod update_action_result { let mut action_result = ActionResult::default(); action_result.exit_code = 45; - let size_bytes = get_encoded_proto_size(&action_result)?; + let size_bytes = get_encoded_proto_size(&action_result)? as i64; let raw_response = update_action_result( &ac_server, Digest { hash: HASH1.to_string(), - size_bytes: size_bytes as i64, + size_bytes: size_bytes, }, action_result.clone(), ) @@ -166,8 +167,9 @@ mod update_action_result { assert_eq!(raw_response.unwrap().into_inner(), action_result); let mut raw_data = Vec::new(); + let digest = DigestInfo::try_new(&HASH1, size_bytes)?; ac_store - .get(&HASH1, size_bytes, &mut Cursor::new(&mut raw_data)) + .get(&digest, &mut Cursor::new(&mut raw_data)) .await?; let decoded_action_result = ActionResult::decode(Cursor::new(&raw_data))?; diff --git a/cas/grpc_service/tests/bytestream_server_test.rs b/cas/grpc_service/tests/bytestream_server_test.rs index 38c46f5b7..b4ac8517b 100644 --- a/cas/grpc_service/tests/bytestream_server_test.rs +++ b/cas/grpc_service/tests/bytestream_server_test.rs @@ -6,6 +6,7 @@ use std::io::Cursor; use bytestream_server::ByteStreamServer; use tonic::Request; +use common::DigestInfo; use store::{create_store, StoreType}; const INSTANCE_NAME: &str = "foo"; @@ -122,10 +123,15 @@ pub mod write_tests { assert_eq!(committed_size as usize, raw_data.len()); // Now lets check our store to ensure it was written with proper data. - store.has(HASH1, raw_data.len()).await?; + store + .has(&DigestInfo::try_new(&HASH1, raw_data.len())?) + .await?; let mut store_data = Vec::new(); store - .get(HASH1, raw_data.len(), &mut Cursor::new(&mut store_data)) + .get( + &DigestInfo::try_new(&HASH1, raw_data.len())?, + &mut Cursor::new(&mut store_data), + ) .await?; assert_eq!( std::str::from_utf8(&store_data), diff --git a/cas/grpc_service/tests/cas_server_test.rs b/cas/grpc_service/tests/cas_server_test.rs index f28609310..455d2a396 100644 --- a/cas/grpc_service/tests/cas_server_test.rs +++ b/cas/grpc_service/tests/cas_server_test.rs @@ -2,11 +2,13 @@ use tonic::Request; -use proto::build::bazel::remote::execution::v2::Digest; +use proto::build::bazel::remote::execution::v2::{ + content_addressable_storage_server::ContentAddressableStorage, Digest, +}; use proto::google::rpc::Status as GrpcStatus; use cas_server::CasServer; -use proto::build::bazel::remote::execution::v2::content_addressable_storage_server::ContentAddressableStorage; +use common::DigestInfo; use store::{create_store, StoreType}; const INSTANCE_NAME: &str = "foo"; @@ -51,7 +53,10 @@ mod find_missing_blobs { const VALUE: &str = "1"; store - .update(&HASH1, VALUE.len(), Box::new(Cursor::new(VALUE))) + .update( + &DigestInfo::try_new(HASH1, VALUE.len())?, + Box::new(Cursor::new(VALUE)), + ) .await?; let raw_response = cas_server .find_missing_blobs(Request::new(FindMissingBlobsRequest { @@ -69,7 +74,6 @@ mod find_missing_blobs { } #[tokio::test] - #[ignore] // TODO(allada) Disabled until we finish moving to a Hash struct. async fn has_three_requests_one_bad_hash() -> Result<(), Box> { let store = create_store(&StoreType::Memory); let cas_server = CasServer::new(store.clone()); @@ -77,7 +81,10 @@ mod find_missing_blobs { const VALUE: &str = "1"; store - .update(&HASH1, VALUE.len(), Box::new(Cursor::new(VALUE))) + .update( + &DigestInfo::try_new(HASH1, VALUE.len())?, + Box::new(Cursor::new(VALUE)), + ) .await?; let raw_response = cas_server .find_missing_blobs(Request::new(FindMissingBlobsRequest { @@ -100,10 +107,8 @@ mod find_missing_blobs { .await; let error = raw_response.unwrap_err(); assert!( - error - .to_string() - .contains("Hex length is not 64 hex characters"), - "'Hex length is not 64 hex characters' not found in: {:?}", + error.to_string().contains("Invalid sha256 hash: BAD_HASH"), + "'Invalid sha256 hash: BAD_HASH' not found in: {:?}", error ); Ok(()) @@ -123,7 +128,7 @@ mod batch_update_blobs { }; #[tokio::test] - async fn update_existing_item() { + async fn update_existing_item() -> Result<(), Box> { let store = create_store(&StoreType::Memory); let cas_server = CasServer::new(store.clone()); @@ -136,7 +141,10 @@ mod batch_update_blobs { }; store - .update(&HASH1, VALUE1.len(), Box::new(Cursor::new(VALUE1))) + .update( + &DigestInfo::try_new(&HASH1, VALUE1.len())?, + Box::new(Cursor::new(VALUE1)), + ) .await .expect("Update should have succeeded"); @@ -165,7 +173,10 @@ mod batch_update_blobs { ); let mut new_data = Vec::new(); store - .get(&HASH1, HASH1.len(), &mut Cursor::new(&mut new_data)) + .get( + &DigestInfo::try_new(&HASH1, HASH1.len())?, + &mut Cursor::new(&mut new_data), + ) .await .expect("Get should have succeeded"); assert_eq!( @@ -173,6 +184,7 @@ mod batch_update_blobs { VALUE2.as_bytes(), "Expected store to have been updated to new value" ); + Ok(()) } } @@ -208,11 +220,17 @@ mod batch_read_blobs { { // Insert dummy data. store - .update(&HASH1, VALUE1.len(), Box::new(Cursor::new(VALUE1))) + .update( + &DigestInfo::try_new(&HASH1, VALUE1.len())?, + Box::new(Cursor::new(VALUE1)), + ) .await .expect("Update should have succeeded"); store - .update(&HASH2, VALUE2.len(), Box::new(Cursor::new(VALUE2))) + .update( + &DigestInfo::try_new(&HASH2, VALUE2.len())?, + Box::new(Cursor::new(VALUE2)), + ) .await .expect("Update should have succeeded"); } diff --git a/cas/store/BUILD b/cas/store/BUILD index 9130bd027..04c6eed76 100644 --- a/cas/store/BUILD +++ b/cas/store/BUILD @@ -18,6 +18,7 @@ rust_library( deps = [ "//third_party:tokio", "//util:error", + "//util:common", ], proc_macro_deps = ["//third_party:async_trait"], visibility = ["//cas:__pkg__"] @@ -31,7 +32,7 @@ rust_library( "//util:error", "//third_party:async_mutex", "//third_party:tokio", - "//third_party:hex", + "//util:common", ], proc_macro_deps = ["//third_party:async_trait"], visibility = ["//cas:__pkg__"] @@ -44,8 +45,8 @@ rust_test( ":memory_store", ":traits", "//util:error", + "//util:common", "//third_party:tokio", - "//third_party:tokio_test", "//third_party:pretty_assertions", ], ) diff --git a/cas/store/memory_store.rs b/cas/store/memory_store.rs index 0e92603e5..8fa3117a1 100644 --- a/cas/store/memory_store.rs +++ b/cas/store/memory_store.rs @@ -5,9 +5,9 @@ use std::sync::Arc; use async_mutex::Mutex; use async_trait::async_trait; -use hex::FromHex; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use common::DigestInfo; use error::{error_if, Code, Error, ResultExt}; use traits::StoreTrait; @@ -26,47 +26,39 @@ impl MemoryStore { #[async_trait] impl StoreTrait for MemoryStore { - async fn has(&self, hash: &str, _expected_size: usize) -> Result { - let raw_key = - <[u8; 32]>::from_hex(&hash).err_tip(|| "Hex length is not 64 hex characters")?; + async fn has(&self, digest: &DigestInfo) -> Result { let map = self.map.lock().await; - Ok(map.contains_key(&raw_key)) + Ok(map.contains_key(&digest.packed_hash)) } async fn update<'a, 'b>( &'a self, - hash: &'a str, - expected_size: usize, + digest: &'a DigestInfo, mut reader: Box, ) -> Result<(), Error> { - let raw_key = - <[u8; 32]>::from_hex(&hash).err_tip(|| "Hex length is not 64 hex characters")?; let mut buffer = Vec::new(); - let read_size = reader.read_to_end(&mut buffer).await?; + let read_size = reader.read_to_end(&mut buffer).await? as i64; error_if!( - read_size != expected_size, + read_size != digest.size_bytes, "Expected size {} but got size {} for hash {} CAS insert", - expected_size, + digest.size_bytes, read_size, - hash + digest.str() ); let mut map = self.map.lock().await; - map.insert(raw_key, Arc::new(buffer)); + map.insert(digest.packed_hash, Arc::new(buffer)); Ok(()) } async fn get( &self, - hash: &str, - _expected_size: usize, + digest: &DigestInfo, writer: &mut (dyn AsyncWrite + Send + Unpin), ) -> Result<(), Error> { - let raw_key = - <[u8; 32]>::from_hex(&hash).err_tip(|| "Hex length is not 64 hex characters")?; let map = self.map.lock().await; let value = map - .get(&raw_key) - .err_tip_with_code(|_| (Code::NotFound, format!("Hash {} not found", hash)))?; + .get(&digest.packed_hash) + .err_tip_with_code(|_| (Code::NotFound, format!("Hash {} not found", digest.str())))?; writer.write_all(value).await?; Ok(()) } diff --git a/cas/store/store_trait.rs b/cas/store/store_trait.rs index c4efb92dc..ac39c1373 100644 --- a/cas/store/store_trait.rs +++ b/cas/store/store_trait.rs @@ -3,23 +3,22 @@ use async_trait::async_trait; use tokio::io::{AsyncRead, AsyncWrite}; +use common::DigestInfo; use error::Error; #[async_trait] pub trait StoreTrait: Sync + Send { - async fn has(&self, hash: &str, expected_size: usize) -> Result; + async fn has(&self, digest: &DigestInfo) -> Result; async fn update<'a, 'b>( &'a self, - hash: &'a str, - expected_size: usize, + digest: &'a DigestInfo, mut reader: Box, ) -> Result<(), Error>; async fn get( &self, - hash: &str, - expected_size: usize, + digest: &DigestInfo, writer: &mut (dyn AsyncWrite + Send + Unpin), ) -> Result<(), Error>; } diff --git a/cas/store/tests/memory_store_test.rs b/cas/store/tests/memory_store_test.rs index 4736bbfee..2b99dc3cc 100644 --- a/cas/store/tests/memory_store_test.rs +++ b/cas/store/tests/memory_store_test.rs @@ -6,8 +6,8 @@ mod memory_store_tests { use error::Error; use std::io::Cursor; - use tokio_test::assert_err; + use common::DigestInfo; use memory_store::MemoryStore; use traits::StoreTrait; @@ -21,10 +21,15 @@ mod memory_store_tests { // Insert dummy value into store. const VALUE1: &str = "1"; store - .update(&VALID_HASH1, VALUE1.len(), Box::new(Cursor::new(VALUE1))) + .update( + &DigestInfo::try_new(&VALID_HASH1, VALUE1.len())?, + Box::new(Cursor::new(VALUE1)), + ) .await?; assert!( - store.has(&VALID_HASH1, VALID_HASH1.len()).await?, + store + .has(&DigestInfo::try_new(&VALID_HASH1, VALID_HASH1.len())?) + .await?, "Expected memory store to have hash: {}", VALID_HASH1 ); @@ -35,12 +40,14 @@ mod memory_store_tests { { // Now change value we just inserted. store - .update(&VALID_HASH1, VALUE2.len(), Box::new(Cursor::new(VALUE2))) + .update( + &DigestInfo::try_new(&VALID_HASH1, VALUE2.len())?, + Box::new(Cursor::new(VALUE2)), + ) .await?; store .get( - &VALID_HASH1, - VALID_HASH1.len(), + &DigestInfo::try_new(&VALID_HASH1, VALID_HASH1.len())?, &mut Cursor::new(&mut store_data), ) .await?; @@ -69,8 +76,9 @@ mod memory_store_tests { { // .has() tests. async fn has_should_fail(store: &MemoryStore, hash: &str, expected_size: usize) { - assert_err!( - store.has(&hash, expected_size).await, + let digest = DigestInfo::try_new(&hash, expected_size); + assert!( + digest.is_err() || store.has(&digest.unwrap()).await.is_err(), ".has() should have failed: {} {}", hash, expected_size @@ -88,10 +96,13 @@ mod memory_store_tests { expected_size: usize, value: &'b str, ) { - assert_err!( - store - .update(&hash, expected_size, Box::new(Cursor::new(&value))) - .await, + let digest = DigestInfo::try_new(&hash, expected_size); + assert!( + digest.is_err() + || store + .update(&digest.unwrap(), Box::new(Cursor::new(&value))) + .await + .is_err(), ".has() should have failed: {} {} {}", hash, expected_size, @@ -112,10 +123,13 @@ mod memory_store_tests { expected_size: usize, out_data: &'a mut Vec, ) { - assert_err!( - store - .get(&hash, expected_size, &mut Cursor::new(out_data)) - .await, + let digest = DigestInfo::try_new(&hash, expected_size); + assert!( + digest.is_err() + || store + .get(&digest.unwrap(), &mut Cursor::new(out_data)) + .await + .is_err(), ".get() should have failed: {} {}", hash, expected_size diff --git a/third_party/BUILD.bazel b/third_party/BUILD.bazel index 548b59511..777e83f5d 100644 --- a/third_party/BUILD.bazel +++ b/third_party/BUILD.bazel @@ -75,6 +75,15 @@ alias( ], ) +alias( + name = "lazy_init", + actual = "@raze__lazy_init__0_4_0//:lazy_init", + tags = [ + "cargo-raze", + "manual", + ], +) + alias( name = "pretty_assertions", actual = "@raze__pretty_assertions__0_6_1//:pretty_assertions", @@ -149,15 +158,6 @@ alias( ], ) -alias( - name = "tokio_test", - actual = "@raze__tokio_test__0_4_0//:tokio_test", - tags = [ - "cargo-raze", - "manual", - ], -) - alias( name = "tonic", actual = "@raze__tonic__0_3_1//:tonic", diff --git a/third_party/crates.bzl b/third_party/crates.bzl index 2c9f696a5..0da839448 100644 --- a/third_party/crates.bzl +++ b/third_party/crates.bzl @@ -101,16 +101,6 @@ def raze_fetch_remote_crates(): build_file = Label("//third_party/remote:BUILD.async-stream-0.2.1.bazel"), ) - maybe( - http_archive, - name = "raze__async_stream__0_3_0", - url = "https://crates.io/api/v1/crates/async-stream/0.3.0/download", - type = "tar.gz", - sha256 = "3670df70cbc01729f901f94c887814b3c68db038aad1329a418bae178bc5295c", - strip_prefix = "async-stream-0.3.0", - build_file = Label("//third_party/remote:BUILD.async-stream-0.3.0.bazel"), - ) - maybe( http_archive, name = "raze__async_stream_impl__0_2_1", @@ -121,16 +111,6 @@ def raze_fetch_remote_crates(): build_file = Label("//third_party/remote:BUILD.async-stream-impl-0.2.1.bazel"), ) - maybe( - http_archive, - name = "raze__async_stream_impl__0_3_0", - url = "https://crates.io/api/v1/crates/async-stream-impl/0.3.0/download", - type = "tar.gz", - sha256 = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70", - strip_prefix = "async-stream-impl-0.3.0", - build_file = Label("//third_party/remote:BUILD.async-stream-impl-0.3.0.bazel"), - ) - maybe( http_archive, name = "raze__async_trait__0_1_42", @@ -271,16 +251,6 @@ def raze_fetch_remote_crates(): build_file = Label("//third_party/remote:BUILD.bytes-0.5.6.bazel"), ) - maybe( - http_archive, - name = "raze__bytes__1_0_0", - url = "https://crates.io/api/v1/crates/bytes/1.0.0/download", - type = "tar.gz", - sha256 = "ad1f8e949d755f9d79112b5bb46938e0ef9d3804a0b16dfab13aafcaa5f0fa72", - strip_prefix = "bytes-1.0.0", - build_file = Label("//third_party/remote:BUILD.bytes-1.0.0.bazel"), - ) - maybe( http_archive, name = "raze__cargo_metadata__0_8_2", @@ -891,6 +861,16 @@ def raze_fetch_remote_crates(): build_file = Label("//third_party/remote:BUILD.kernel32-sys-0.2.2.bazel"), ) + maybe( + http_archive, + name = "raze__lazy_init__0_4_0", + url = "https://crates.io/api/v1/crates/lazy-init/0.4.0/download", + type = "tar.gz", + sha256 = "671025f3cc9b5d5188eca51493d8097e6cd1dc3828e0a59f7befbaaf56e930ab", + strip_prefix = "lazy-init-0.4.0", + build_file = Label("//third_party/remote:BUILD.lazy-init-0.4.0.bazel"), + ) + maybe( http_archive, name = "raze__lazy_static__1_4_0", @@ -2041,16 +2021,6 @@ def raze_fetch_remote_crates(): build_file = Label("//third_party/remote:BUILD.tokio-0.2.24.bazel"), ) - maybe( - http_archive, - name = "raze__tokio__1_0_1", - url = "https://crates.io/api/v1/crates/tokio/1.0.1/download", - type = "tar.gz", - sha256 = "d258221f566b6c803c7b4714abadc080172b272090cdc5e244a6d4dd13c3a6bd", - strip_prefix = "tokio-1.0.1", - build_file = Label("//third_party/remote:BUILD.tokio-1.0.1.bazel"), - ) - maybe( http_archive, name = "raze__tokio_macros__0_2_6", @@ -2061,26 +2031,6 @@ def raze_fetch_remote_crates(): build_file = Label("//third_party/remote:BUILD.tokio-macros-0.2.6.bazel"), ) - maybe( - http_archive, - name = "raze__tokio_stream__0_1_0", - url = "https://crates.io/api/v1/crates/tokio-stream/0.1.0/download", - type = "tar.gz", - sha256 = "3f3be913b74b13210c8fe04b17ab833f5a124f45b93d0f99f59fff621f64392a", - strip_prefix = "tokio-stream-0.1.0", - build_file = Label("//third_party/remote:BUILD.tokio-stream-0.1.0.bazel"), - ) - - maybe( - http_archive, - name = "raze__tokio_test__0_4_0", - url = "https://crates.io/api/v1/crates/tokio-test/0.4.0/download", - type = "tar.gz", - sha256 = "7c7d205f6f59b03f9e824ac86eaba635a98395f287756ecc8a06464779c399bf", - strip_prefix = "tokio-test-0.4.0", - build_file = Label("//third_party/remote:BUILD.tokio-test-0.4.0.bazel"), - ) - maybe( http_archive, name = "raze__tokio_util__0_3_1", diff --git a/third_party/remote/BUILD.async-stream-0.3.0.bazel b/third_party/remote/BUILD.async-stream-0.3.0.bazel deleted file mode 100644 index 440161a09..000000000 --- a/third_party/remote/BUILD.async-stream-0.3.0.bazel +++ /dev/null @@ -1,65 +0,0 @@ -""" -@generated -cargo-raze crate build file. - -DO NOT EDIT! Replaced on runs of cargo-raze -""" - -# buildifier: disable=load -load( - "@io_bazel_rules_rust//rust:rust.bzl", - "rust_binary", - "rust_library", - "rust_test", -) - -# buildifier: disable=load -load("@bazel_skylib//lib:selects.bzl", "selects") - -package(default_visibility = [ - # Public for visibility by "@raze__crate__version//" targets. - # - # Prefer access through "//third_party", which limits external - # visibility to explicit Cargo.toml dependencies. - "//visibility:public", -]) - -licenses([ - "notice", # MIT from expression "MIT" -]) - -# Generated Targets - -# Unsupported target "tcp_accept" with type "example" omitted - -rust_library( - name = "async_stream", - srcs = glob(["**/*.rs"]), - crate_features = [ - ], - crate_root = "src/lib.rs", - crate_type = "lib", - data = [], - edition = "2018", - proc_macro_deps = [ - "@raze__async_stream_impl__0_3_0//:async_stream_impl", - ], - rustc_flags = [ - "--cap-lints=allow", - ], - tags = [ - "cargo-raze", - "manual", - ], - version = "0.3.0", - # buildifier: leave-alone - deps = [ - "@raze__futures_core__0_3_8//:futures_core", - ], -) - -# Unsupported target "for_await" with type "test" omitted - -# Unsupported target "stream" with type "test" omitted - -# Unsupported target "try_stream" with type "test" omitted diff --git a/third_party/remote/BUILD.bytes-1.0.0.bazel b/third_party/remote/BUILD.bytes-1.0.0.bazel deleted file mode 100644 index 6723d6844..000000000 --- a/third_party/remote/BUILD.bytes-1.0.0.bazel +++ /dev/null @@ -1,83 +0,0 @@ -""" -@generated -cargo-raze crate build file. - -DO NOT EDIT! Replaced on runs of cargo-raze -""" - -# buildifier: disable=load -load( - "@io_bazel_rules_rust//rust:rust.bzl", - "rust_binary", - "rust_library", - "rust_test", -) - -# buildifier: disable=load -load("@bazel_skylib//lib:selects.bzl", "selects") - -package(default_visibility = [ - # Public for visibility by "@raze__crate__version//" targets. - # - # Prefer access through "//third_party", which limits external - # visibility to explicit Cargo.toml dependencies. - "//visibility:public", -]) - -licenses([ - "notice", # MIT from expression "MIT" -]) - -# Generated Targets - -# Unsupported target "buf" with type "bench" omitted - -# Unsupported target "bytes" with type "bench" omitted - -# Unsupported target "bytes_mut" with type "bench" omitted - -rust_library( - name = "bytes", - srcs = glob(["**/*.rs"]), - crate_features = [ - "default", - "std", - ], - crate_root = "src/lib.rs", - crate_type = "lib", - data = [], - edition = "2018", - rustc_flags = [ - "--cap-lints=allow", - ], - tags = [ - "cargo-raze", - "manual", - ], - version = "1.0.0", - # buildifier: leave-alone - deps = [ - ], -) - -# Unsupported target "test_buf" with type "test" omitted - -# Unsupported target "test_buf_mut" with type "test" omitted - -# Unsupported target "test_bytes" with type "test" omitted - -# Unsupported target "test_bytes_odd_alloc" with type "test" omitted - -# Unsupported target "test_bytes_vec_alloc" with type "test" omitted - -# Unsupported target "test_chain" with type "test" omitted - -# Unsupported target "test_debug" with type "test" omitted - -# Unsupported target "test_iter" with type "test" omitted - -# Unsupported target "test_reader" with type "test" omitted - -# Unsupported target "test_serde" with type "test" omitted - -# Unsupported target "test_take" with type "test" omitted diff --git a/third_party/remote/BUILD.async-stream-impl-0.3.0.bazel b/third_party/remote/BUILD.lazy-init-0.4.0.bazel similarity index 76% rename from third_party/remote/BUILD.async-stream-impl-0.3.0.bazel rename to third_party/remote/BUILD.lazy-init-0.4.0.bazel index 074a5f4c6..8956aff6b 100644 --- a/third_party/remote/BUILD.async-stream-impl-0.3.0.bazel +++ b/third_party/remote/BUILD.lazy-init-0.4.0.bazel @@ -25,20 +25,20 @@ package(default_visibility = [ ]) licenses([ - "notice", # MIT from expression "MIT" + "notice", # Apache-2.0 from expression "Apache-2.0 OR MIT" ]) # Generated Targets rust_library( - name = "async_stream_impl", + name = "lazy_init", srcs = glob(["**/*.rs"]), crate_features = [ ], crate_root = "src/lib.rs", - crate_type = "proc-macro", + crate_type = "lib", data = [], - edition = "2018", + edition = "2015", rustc_flags = [ "--cap-lints=allow", ], @@ -46,11 +46,8 @@ rust_library( "cargo-raze", "manual", ], - version = "0.3.0", + version = "0.4.0", # buildifier: leave-alone deps = [ - "@raze__proc_macro2__1_0_24//:proc_macro2", - "@raze__quote__1_0_8//:quote", - "@raze__syn__1_0_57//:syn", ], ) diff --git a/third_party/remote/BUILD.tokio-1.0.1.bazel b/third_party/remote/BUILD.tokio-1.0.1.bazel deleted file mode 100644 index ca7b862b1..000000000 --- a/third_party/remote/BUILD.tokio-1.0.1.bazel +++ /dev/null @@ -1,292 +0,0 @@ -""" -@generated -cargo-raze crate build file. - -DO NOT EDIT! Replaced on runs of cargo-raze -""" - -# buildifier: disable=load -load( - "@io_bazel_rules_rust//rust:rust.bzl", - "rust_binary", - "rust_library", - "rust_test", -) - -# buildifier: disable=load -load("@bazel_skylib//lib:selects.bzl", "selects") - -package(default_visibility = [ - # Public for visibility by "@raze__crate__version//" targets. - # - # Prefer access through "//third_party", which limits external - # visibility to explicit Cargo.toml dependencies. - "//visibility:public", -]) - -licenses([ - "notice", # MIT from expression "MIT" -]) - -# Generated Targets -# buildifier: disable=load-on-top -load( - "@io_bazel_rules_rust//cargo:cargo_build_script.bzl", - "cargo_build_script", -) - -cargo_build_script( - name = "tokio_build_script", - srcs = glob(["**/*.rs"]), - build_script_env = { - }, - crate_features = [ - "default", - "rt", - "sync", - "test-util", - "time", - ], - crate_root = "build.rs", - data = glob(["**"]), - edition = "2018", - rustc_flags = [ - "--cap-lints=allow", - ], - tags = [ - "cargo-raze", - "manual", - ], - version = "1.0.1", - visibility = ["//visibility:private"], - deps = [ - "@raze__autocfg__1_0_1//:autocfg", - ] + selects.with_or({ - # cfg(unix) - ( - "@io_bazel_rules_rust//rust/platform:x86_64-unknown-linux-gnu", - ): [ - ], - "//conditions:default": [], - }), -) - -rust_library( - name = "tokio", - srcs = glob(["**/*.rs"]), - aliases = { - }, - crate_features = [ - "default", - "rt", - "sync", - "test-util", - "time", - ], - crate_root = "src/lib.rs", - crate_type = "lib", - data = [], - edition = "2018", - rustc_flags = [ - "--cap-lints=allow", - ], - tags = [ - "cargo-raze", - "manual", - ], - version = "1.0.1", - # buildifier: leave-alone - deps = [ - ":tokio_build_script", - "@raze__pin_project_lite__0_2_0//:pin_project_lite", - ] + selects.with_or({ - # cfg(unix) - ( - "@io_bazel_rules_rust//rust/platform:x86_64-unknown-linux-gnu", - ): [ - ], - "//conditions:default": [], - }), -) - -# Unsupported target "_require_full" with type "test" omitted - -# Unsupported target "async_send_sync" with type "test" omitted - -# Unsupported target "buffered" with type "test" omitted - -# Unsupported target "fs" with type "test" omitted - -# Unsupported target "fs_copy" with type "test" omitted - -# Unsupported target "fs_dir" with type "test" omitted - -# Unsupported target "fs_file" with type "test" omitted - -# Unsupported target "fs_file_mocked" with type "test" omitted - -# Unsupported target "fs_link" with type "test" omitted - -# Unsupported target "io_async_fd" with type "test" omitted - -# Unsupported target "io_async_read" with type "test" omitted - -# Unsupported target "io_chain" with type "test" omitted - -# Unsupported target "io_copy" with type "test" omitted - -# Unsupported target "io_driver" with type "test" omitted - -# Unsupported target "io_driver_drop" with type "test" omitted - -# Unsupported target "io_lines" with type "test" omitted - -# Unsupported target "io_mem_stream" with type "test" omitted - -# Unsupported target "io_read" with type "test" omitted - -# Unsupported target "io_read_buf" with type "test" omitted - -# Unsupported target "io_read_exact" with type "test" omitted - -# Unsupported target "io_read_line" with type "test" omitted - -# Unsupported target "io_read_to_end" with type "test" omitted - -# Unsupported target "io_read_to_string" with type "test" omitted - -# Unsupported target "io_read_until" with type "test" omitted - -# Unsupported target "io_split" with type "test" omitted - -# Unsupported target "io_take" with type "test" omitted - -# Unsupported target "io_write" with type "test" omitted - -# Unsupported target "io_write_all" with type "test" omitted - -# Unsupported target "io_write_buf" with type "test" omitted - -# Unsupported target "io_write_int" with type "test" omitted - -# Unsupported target "macros_join" with type "test" omitted - -# Unsupported target "macros_pin" with type "test" omitted - -# Unsupported target "macros_select" with type "test" omitted - -# Unsupported target "macros_test" with type "test" omitted - -# Unsupported target "macros_try_join" with type "test" omitted - -# Unsupported target "net_bind_resource" with type "test" omitted - -# Unsupported target "net_lookup_host" with type "test" omitted - -# Unsupported target "no_rt" with type "test" omitted - -# Unsupported target "process_issue_2174" with type "test" omitted - -# Unsupported target "process_issue_42" with type "test" omitted - -# Unsupported target "process_kill_on_drop" with type "test" omitted - -# Unsupported target "process_smoke" with type "test" omitted - -# Unsupported target "rt_basic" with type "test" omitted - -# Unsupported target "rt_common" with type "test" omitted - -# Unsupported target "rt_threaded" with type "test" omitted - -# Unsupported target "signal_ctrl_c" with type "test" omitted - -# Unsupported target "signal_drop_recv" with type "test" omitted - -# Unsupported target "signal_drop_rt" with type "test" omitted - -# Unsupported target "signal_drop_signal" with type "test" omitted - -# Unsupported target "signal_multi_rt" with type "test" omitted - -# Unsupported target "signal_no_rt" with type "test" omitted - -# Unsupported target "signal_notify_both" with type "test" omitted - -# Unsupported target "signal_twice" with type "test" omitted - -# Unsupported target "signal_usr1" with type "test" omitted - -# Unsupported target "sync_barrier" with type "test" omitted - -# Unsupported target "sync_broadcast" with type "test" omitted - -# Unsupported target "sync_errors" with type "test" omitted - -# Unsupported target "sync_mpsc" with type "test" omitted - -# Unsupported target "sync_mutex" with type "test" omitted - -# Unsupported target "sync_mutex_owned" with type "test" omitted - -# Unsupported target "sync_notify" with type "test" omitted - -# Unsupported target "sync_oneshot" with type "test" omitted - -# Unsupported target "sync_rwlock" with type "test" omitted - -# Unsupported target "sync_semaphore" with type "test" omitted - -# Unsupported target "sync_semaphore_owned" with type "test" omitted - -# Unsupported target "sync_watch" with type "test" omitted - -# Unsupported target "task_abort" with type "test" omitted - -# Unsupported target "task_blocking" with type "test" omitted - -# Unsupported target "task_local" with type "test" omitted - -# Unsupported target "task_local_set" with type "test" omitted - -# Unsupported target "tcp_accept" with type "test" omitted - -# Unsupported target "tcp_connect" with type "test" omitted - -# Unsupported target "tcp_echo" with type "test" omitted - -# Unsupported target "tcp_into_split" with type "test" omitted - -# Unsupported target "tcp_into_std" with type "test" omitted - -# Unsupported target "tcp_peek" with type "test" omitted - -# Unsupported target "tcp_shutdown" with type "test" omitted - -# Unsupported target "tcp_socket" with type "test" omitted - -# Unsupported target "tcp_split" with type "test" omitted - -# Unsupported target "tcp_stream" with type "test" omitted - -# Unsupported target "test_clock" with type "test" omitted - -# Unsupported target "time_interval" with type "test" omitted - -# Unsupported target "time_pause" with type "test" omitted - -# Unsupported target "time_rt" with type "test" omitted - -# Unsupported target "time_sleep" with type "test" omitted - -# Unsupported target "time_timeout" with type "test" omitted - -# Unsupported target "udp" with type "test" omitted - -# Unsupported target "uds_cred" with type "test" omitted - -# Unsupported target "uds_datagram" with type "test" omitted - -# Unsupported target "uds_split" with type "test" omitted - -# Unsupported target "uds_stream" with type "test" omitted diff --git a/third_party/remote/BUILD.tokio-stream-0.1.0.bazel b/third_party/remote/BUILD.tokio-stream-0.1.0.bazel deleted file mode 100644 index d77411b59..000000000 --- a/third_party/remote/BUILD.tokio-stream-0.1.0.bazel +++ /dev/null @@ -1,83 +0,0 @@ -""" -@generated -cargo-raze crate build file. - -DO NOT EDIT! Replaced on runs of cargo-raze -""" - -# buildifier: disable=load -load( - "@io_bazel_rules_rust//rust:rust.bzl", - "rust_binary", - "rust_library", - "rust_test", -) - -# buildifier: disable=load -load("@bazel_skylib//lib:selects.bzl", "selects") - -package(default_visibility = [ - # Public for visibility by "@raze__crate__version//" targets. - # - # Prefer access through "//third_party", which limits external - # visibility to explicit Cargo.toml dependencies. - "//visibility:public", -]) - -licenses([ - "notice", # MIT from expression "MIT" -]) - -# Generated Targets - -rust_library( - name = "tokio_stream", - srcs = glob(["**/*.rs"]), - crate_features = [ - "default", - "time", - ], - crate_root = "src/lib.rs", - crate_type = "lib", - data = [], - edition = "2018", - rustc_flags = [ - "--cap-lints=allow", - ], - tags = [ - "cargo-raze", - "manual", - ], - version = "0.1.0", - # buildifier: leave-alone - deps = [ - "@raze__async_stream__0_3_0//:async_stream", - "@raze__futures_core__0_3_8//:futures_core", - "@raze__pin_project_lite__0_2_0//:pin_project_lite", - "@raze__tokio__1_0_1//:tokio", - ], -) - -# Unsupported target "async_send_sync" with type "test" omitted - -# Unsupported target "stream_chain" with type "test" omitted - -# Unsupported target "stream_collect" with type "test" omitted - -# Unsupported target "stream_empty" with type "test" omitted - -# Unsupported target "stream_fuse" with type "test" omitted - -# Unsupported target "stream_iter" with type "test" omitted - -# Unsupported target "stream_merge" with type "test" omitted - -# Unsupported target "stream_once" with type "test" omitted - -# Unsupported target "stream_pending" with type "test" omitted - -# Unsupported target "stream_stream_map" with type "test" omitted - -# Unsupported target "stream_timeout" with type "test" omitted - -# Unsupported target "time_throttle" with type "test" omitted diff --git a/third_party/remote/BUILD.tokio-test-0.4.0.bazel b/third_party/remote/BUILD.tokio-test-0.4.0.bazel deleted file mode 100644 index db990bb38..000000000 --- a/third_party/remote/BUILD.tokio-test-0.4.0.bazel +++ /dev/null @@ -1,64 +0,0 @@ -""" -@generated -cargo-raze crate build file. - -DO NOT EDIT! Replaced on runs of cargo-raze -""" - -# buildifier: disable=load -load( - "@io_bazel_rules_rust//rust:rust.bzl", - "rust_binary", - "rust_library", - "rust_test", -) - -# buildifier: disable=load -load("@bazel_skylib//lib:selects.bzl", "selects") - -package(default_visibility = [ - # Public for visibility by "@raze__crate__version//" targets. - # - # Prefer access through "//third_party", which limits external - # visibility to explicit Cargo.toml dependencies. - "//visibility:public", -]) - -licenses([ - "notice", # MIT from expression "MIT" -]) - -# Generated Targets - -rust_library( - name = "tokio_test", - srcs = glob(["**/*.rs"]), - crate_features = [ - ], - crate_root = "src/lib.rs", - crate_type = "lib", - data = [], - edition = "2018", - rustc_flags = [ - "--cap-lints=allow", - ], - tags = [ - "cargo-raze", - "manual", - ], - version = "0.4.0", - # buildifier: leave-alone - deps = [ - "@raze__async_stream__0_3_0//:async_stream", - "@raze__bytes__1_0_0//:bytes", - "@raze__futures_core__0_3_8//:futures_core", - "@raze__tokio__1_0_1//:tokio", - "@raze__tokio_stream__0_1_0//:tokio_stream", - ], -) - -# Unsupported target "block_on" with type "test" omitted - -# Unsupported target "io" with type "test" omitted - -# Unsupported target "macros" with type "test" omitted diff --git a/util/BUILD b/util/BUILD index 362fd8785..aa25942d9 100644 --- a/util/BUILD +++ b/util/BUILD @@ -15,6 +15,18 @@ rust_library( visibility = ["//visibility:public"], ) +rust_library( + name = "common", + srcs = ["common.rs"], + deps = [ + ":error", + "//proto", + "//third_party:hex", + "//third_party:lazy_init", + ], + visibility = ["//visibility:public"], +) + rust_library( name = "async_fixed_buffer", srcs = ["async_fixed_buffer.rs"], diff --git a/util/common.rs b/util/common.rs new file mode 100644 index 000000000..1fe253432 --- /dev/null +++ b/util/common.rs @@ -0,0 +1,77 @@ +// Copyright 2020 Nathan (Blaise) Bruer. All rights reserved. + +use std::convert::TryFrom; +use std::convert::TryInto; + +use hex::FromHex; +use lazy_init::LazyTransform; +use proto::build::bazel::remote::execution::v2::Digest; + +use error::{make_input_err, Error, ResultExt}; + +pub struct DigestInfo { + pub size_bytes: i64, + pub packed_hash: [u8; 32], + str_hash: LazyTransform, String>, +} + +impl DigestInfo { + pub fn try_new(hash: &str, size_bytes: T) -> Result + where + T: TryInto + std::fmt::Display + Copy, + { + let packed_hash = + <[u8; 32]>::from_hex(hash).err_tip(|| format!("Invalid sha256 hash: {}", hash))?; + let size_bytes = size_bytes + .try_into() + .or_else(|_| Err(make_input_err!("Could not convert {} into i64", size_bytes)))?; + Ok(DigestInfo { + size_bytes: size_bytes, + packed_hash: packed_hash, + str_hash: LazyTransform::new(None), + }) + } + + pub fn str<'a>(&'a self) -> &'a str { + &self + .str_hash + .get_or_create(|v| v.unwrap_or_else(|| hex::encode(self.packed_hash))) + } +} + +impl Clone for DigestInfo { + fn clone(&self) -> Self { + DigestInfo { + size_bytes: self.size_bytes, + packed_hash: self.packed_hash, + str_hash: LazyTransform::new(None), + } + } +} + +impl TryFrom for DigestInfo { + type Error = Error; + fn try_from(digest: Digest) -> Result { + let packed_hash = <[u8; 32]>::from_hex(&digest.hash) + .err_tip(|| format!("Invalid sha256 hash: {}", digest.hash))?; + Ok(DigestInfo { + size_bytes: digest.size_bytes, + packed_hash: packed_hash, + str_hash: LazyTransform::new(Some(digest.hash)), + }) + } +} + +impl Into for DigestInfo { + fn into(self) -> Digest { + let packed_hash = self.packed_hash; + let hash = self + .str_hash + .into_inner() + .unwrap_or_else(|v| v.unwrap_or_else(|| hex::encode(packed_hash))); + Digest { + hash: hash, + size_bytes: self.size_bytes, + } + } +}