From 75c080ebae22a70578c06ddf1eda0b18ef091845 Mon Sep 17 00:00:00 2001 From: Leon Tan Date: Fri, 2 Aug 2024 15:14:22 +0000 Subject: [PATCH 1/7] Return batch id and evidence from prepare_sync_for_proposal --- src/canisters/frontend/ic-asset/src/sync.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/canisters/frontend/ic-asset/src/sync.rs b/src/canisters/frontend/ic-asset/src/sync.rs index 4950823966..c229416b3c 100644 --- a/src/canisters/frontend/ic-asset/src/sync.rs +++ b/src/canisters/frontend/ic-asset/src/sync.rs @@ -34,6 +34,7 @@ use candid::Nat; use ic_agent::AgentError; use ic_utils::Canister; use itertools::Itertools; +use serde_bytes::ByteBuf; use slog::{debug, info, trace, warn, Logger}; use std::collections::HashMap; use std::path::Path; @@ -194,7 +195,7 @@ pub async fn prepare_sync_for_proposal( canister: &Canister<'_>, dirs: &[&Path], logger: &Logger, -) -> Result<(), PrepareSyncForProposalError> { +) -> Result<(Nat, ByteBuf), PrepareSyncForProposalError> { let arg = upload_content_and_assemble_sync_operations(canister, dirs, false, logger).await?; let arg = sort_batch_operations(arg); let batch_id = arg.batch_id.clone(); @@ -218,9 +219,9 @@ pub async fn prepare_sync_for_proposal( } }; - info!(logger, "Proposed commit of batch {} with evidence {}. Either commit it by proposal, or delete it.", batch_id, hex::encode(evidence)); + info!(logger, "Proposed commit of batch {} with evidence {}. Either commit it by proposal, or delete it.", batch_id, hex::encode(&evidence)); - Ok(()) + Ok((batch_id, evidence)) } fn sort_batch_operations(mut args: CommitBatchArguments) -> CommitBatchArguments { From 6dd3f036eb5dfb1adb9d3709b286648d241313cc Mon Sep 17 00:00:00 2001 From: Leon Tan Date: Tue, 1 Oct 2024 14:19:20 +0000 Subject: [PATCH 2/7] Sort chunk_ids --- .../frontend/ic-asset/src/batch_upload/operations.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/canisters/frontend/ic-asset/src/batch_upload/operations.rs b/src/canisters/frontend/ic-asset/src/batch_upload/operations.rs index b3363b6e39..89f01b2b76 100644 --- a/src/canisters/frontend/ic-asset/src/batch_upload/operations.rs +++ b/src/canisters/frontend/ic-asset/src/batch_upload/operations.rs @@ -163,11 +163,14 @@ pub(crate) fn set_encodings( continue; } + let mut chunk_ids = v.chunk_ids.clone(); + chunk_ids.sort(); + operations.push(BatchOperationKind::SetAssetContent( SetAssetContentArguments { key: key.clone(), content_encoding: content_encoding.clone(), - chunk_ids: v.chunk_ids.clone(), + chunk_ids, sha256: Some(v.sha256.clone()), }, )); From 921b8b9e985e3bd334a2fae110fe6cec4a5afef4 Mon Sep 17 00:00:00 2001 From: Leon Tan Date: Tue, 1 Oct 2024 16:19:49 +0000 Subject: [PATCH 3/7] Add operations trage --- src/canisters/frontend/ic-asset/src/evidence/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/canisters/frontend/ic-asset/src/evidence/mod.rs b/src/canisters/frontend/ic-asset/src/evidence/mod.rs index 2a64a032e2..463a8486cd 100644 --- a/src/canisters/frontend/ic-asset/src/evidence/mod.rs +++ b/src/canisters/frontend/ic-asset/src/evidence/mod.rs @@ -17,7 +17,7 @@ use crate::error::HashContentError::{EncodeContentFailed, LoadContentFailed}; use crate::sync::gather_asset_descriptors; use ic_utils::Canister; use sha2::{Digest, Sha256}; -use slog::{info, Logger}; +use slog::{info, trace, Logger}; use std::collections::{BTreeMap, HashMap}; use std::path::Path; @@ -65,6 +65,7 @@ pub async fn compute_evidence( canister_asset_properties, ); operations.sort(); + trace!(logger, "{:?}", operations); let mut sha = Sha256::new(); for op in operations { From 5949a567782c8101a70dbfec030f41a921073fb4 Mon Sep 17 00:00:00 2001 From: Leon Tan Date: Tue, 1 Oct 2024 16:22:34 +0000 Subject: [PATCH 4/7] Improve --- src/canisters/frontend/ic-asset/src/evidence/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/canisters/frontend/ic-asset/src/evidence/mod.rs b/src/canisters/frontend/ic-asset/src/evidence/mod.rs index 463a8486cd..3ab0751d1c 100644 --- a/src/canisters/frontend/ic-asset/src/evidence/mod.rs +++ b/src/canisters/frontend/ic-asset/src/evidence/mod.rs @@ -65,7 +65,7 @@ pub async fn compute_evidence( canister_asset_properties, ); operations.sort(); - trace!(logger, "{:?}", operations); + trace!(logger, "{:#?}", operations); let mut sha = Sha256::new(); for op in operations { From bb5f8b58afa94b1950f5e1a750e0491457ad88d1 Mon Sep 17 00:00:00 2001 From: Leon Tan Date: Wed, 2 Oct 2024 11:55:30 +0000 Subject: [PATCH 5/7] Try a different gzip backend --- Cargo.lock | 20 -------------------- Cargo.toml | 2 +- src/dfx-core/Cargo.toml | 2 +- src/dfx/Cargo.toml | 6 ++---- 4 files changed, 4 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40b52b04e1..dcb52b1728 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1020,15 +1020,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" -[[package]] -name = "cmake" -version = "0.1.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" -dependencies = [ - "cc", -] - [[package]] name = "codespan-reporting" version = "0.11.1" @@ -2075,7 +2066,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", - "libz-ng-sys", "miniz_oxide", ] @@ -3809,16 +3799,6 @@ dependencies = [ "libc", ] -[[package]] -name = "libz-ng-sys" -version = "1.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6409efc61b12687963e602df8ecf70e8ddacf95bc6576bcf16e3ac6328083c5" -dependencies = [ - "cmake", - "libc", -] - [[package]] name = "link-cplusplus" version = "1.0.9" diff --git a/Cargo.toml b/Cargo.toml index 0c2b83509d..0a44a0e4dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,7 +39,7 @@ clap = "4.5" clap_complete = "4.5" dialoguer = "0.11.0" directories-next = "2.0.0" -flate2 = { version = "1.0.11", default-features = false } +flate2 = { version = "1.0.11" } futures = "0.3.21" handlebars = "4.3.3" hex = "0.4.3" diff --git a/src/dfx-core/Cargo.toml b/src/dfx-core/Cargo.toml index 008b3c11ee..c878ec460c 100644 --- a/src/dfx-core/Cargo.toml +++ b/src/dfx-core/Cargo.toml @@ -19,7 +19,7 @@ clap = { workspace = true, features = ["string"] } dialoguer = { workspace = true } directories-next.workspace = true dunce = "1.0" -flate2 = { workspace = true, default-features = false, features = ["zlib-ng"] } +flate2 = { workspace = true } handlebars.workspace = true hex = { workspace = true, features = ["serde"] } humantime-serde = "1.1.1" diff --git a/src/dfx/Cargo.toml b/src/dfx/Cargo.toml index fee8326a21..14d0c591e0 100644 --- a/src/dfx/Cargo.toml +++ b/src/dfx/Cargo.toml @@ -15,9 +15,7 @@ path = "src/main.rs" [build-dependencies] backoff = { version = "0.4.0", features = ["futures", "tokio"] } bytes = "1" -flate2 = { version = "1.0.11", default-features = false, features = [ - "zlib-ng", -] } +flate2 = { version = "1.0.11" } hex = "0.4.3" reqwest.workspace = true serde = { version = "1.0", features = ["derive"] } @@ -55,7 +53,7 @@ ctrlc = { version = "3.2.1", features = ["termination"] } dfx-core = { path = "../dfx-core" } dialoguer = { workspace = true, features = ["fuzzy-select"] } directories-next.workspace = true -flate2 = { workspace = true, default-features = false, features = ["zlib-ng"] } +flate2 = { workspace = true } fn-error-context = "0.2.0" futures-util = "0.3.21" futures.workspace = true From e57b2d334f310e12db516367e1e7db4e5813eb35 Mon Sep 17 00:00:00 2001 From: Leon Tan Date: Tue, 22 Oct 2024 14:01:49 +0000 Subject: [PATCH 6/7] Remove chunk ids fix, will do that separetely --- .../frontend/ic-asset/src/batch_upload/operations.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/canisters/frontend/ic-asset/src/batch_upload/operations.rs b/src/canisters/frontend/ic-asset/src/batch_upload/operations.rs index 31ed3665ee..76d5e03066 100644 --- a/src/canisters/frontend/ic-asset/src/batch_upload/operations.rs +++ b/src/canisters/frontend/ic-asset/src/batch_upload/operations.rs @@ -172,8 +172,7 @@ pub(crate) async fn set_encodings( if v.already_in_place { continue; } - - let (mut chunk_ids, last_chunk) = match chunk_uploader { + let (chunk_ids, last_chunk) = match chunk_uploader { Some(uploader) => { uploader .uploader_ids_to_canister_chunk_ids(&v.uploader_chunk_ids) @@ -181,8 +180,6 @@ pub(crate) async fn set_encodings( } None => (vec![], None), }; - - chunk_ids.sort(); operations.push(BatchOperationKind::SetAssetContent( SetAssetContentArguments { key: key.clone(), From cb7696dadcc4daec36b0bdb8c1ccb7560c0329c0 Mon Sep 17 00:00:00 2001 From: Leon Tan Date: Tue, 22 Oct 2024 15:09:10 +0000 Subject: [PATCH 7/7] Reintroduce `zlib-ng` in build-dependencies --- Cargo.lock | 20 ++++++++++++++++++++ src/dfx/Cargo.toml | 4 +++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 178ec84455..0da609f011 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1006,6 +1006,15 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" +[[package]] +name = "cmake" +version = "0.1.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" +dependencies = [ + "cc", +] + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -2053,6 +2062,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", + "libz-ng-sys", "miniz_oxide", ] @@ -3916,6 +3926,16 @@ dependencies = [ "libc", ] +[[package]] +name = "libz-ng-sys" +version = "1.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4436751a01da56f1277f323c80d584ffad94a3d14aecd959dd0dff75aa73a438" +dependencies = [ + "cmake", + "libc", +] + [[package]] name = "link-cplusplus" version = "1.0.9" diff --git a/src/dfx/Cargo.toml b/src/dfx/Cargo.toml index 284d59c345..3fd8b6aeb8 100644 --- a/src/dfx/Cargo.toml +++ b/src/dfx/Cargo.toml @@ -15,7 +15,9 @@ path = "src/main.rs" [build-dependencies] backoff = { version = "0.4.0", features = ["futures", "tokio"] } bytes = "1" -flate2 = { version = "1.0.11" } +flate2 = { version = "1.0.11", default-features = false, features = [ + "zlib-ng", +] } hex = "0.4.3" reqwest.workspace = true serde = { version = "1.0", features = ["derive"] }