From b3eafd3a6510442f935b49cf88b4b5f1ce6dc2a2 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Mon, 5 Jun 2023 15:10:46 +0100 Subject: [PATCH 01/64] docker image + tokio --- Cargo.lock | 222 +----------------- build-image/.dockerignore | 1 + build-image/Dockerfile | 72 ++++++ build-image/README.md | 2 + build-image/build.sh | 4 + crates/build/src/lib.rs | 14 +- crates/build/src/metadata.rs | 3 +- crates/cargo-contract/Cargo.toml | 2 +- crates/cargo-contract/src/cmd/build.rs | 6 + .../cargo-contract/src/cmd/extrinsics/call.rs | 5 +- .../src/cmd/extrinsics/instantiate.rs | 3 +- .../src/cmd/extrinsics/integration_tests.rs | 6 +- .../src/cmd/extrinsics/remove.rs | 5 +- .../src/cmd/extrinsics/upload.rs | 5 +- crates/cargo-contract/src/cmd/info.rs | 5 +- crates/metadata/src/lib.rs | 19 +- 16 files changed, 136 insertions(+), 238 deletions(-) create mode 100644 build-image/.dockerignore create mode 100644 build-image/Dockerfile create mode 100644 build-image/README.md create mode 100644 build-image/build.sh diff --git a/Cargo.lock b/Cargo.lock index cc6c73c91..2cb5dce1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -192,77 +192,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" -[[package]] -name = "async-attributes" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5" -dependencies = [ - "quote", - "syn 1.0.109", -] - -[[package]] -name = "async-channel" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833" -dependencies = [ - "concurrent-queue", - "event-listener", - "futures-core", -] - -[[package]] -name = "async-executor" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17adb73da160dfb475c183343c8cccd80721ea5a605d3eb57125f0a7b7a92d0b" -dependencies = [ - "async-lock", - "async-task", - "concurrent-queue", - "fastrand", - "futures-lite", - "slab", -] - -[[package]] -name = "async-global-executor" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" -dependencies = [ - "async-channel", - "async-executor", - "async-io", - "async-lock", - "blocking", - "futures-lite", - "once_cell", - "tokio", -] - -[[package]] -name = "async-io" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c374dda1ed3e7d8f0d9ba58715f924862c63eae6849c92d3a18e7fbde9e2794" -dependencies = [ - "async-lock", - "autocfg", - "concurrent-queue", - "futures-lite", - "libc", - "log", - "parking", - "polling", - "slab", - "socket2", - "waker-fn", - "windows-sys 0.42.0", -] - [[package]] name = "async-lock" version = "2.7.0" @@ -272,39 +201,6 @@ dependencies = [ "event-listener", ] -[[package]] -name = "async-std" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" -dependencies = [ - "async-attributes", - "async-channel", - "async-global-executor", - "async-io", - "async-lock", - "crossbeam-utils", - "futures-channel", - "futures-core", - "futures-io", - "futures-lite", - "gloo-timers", - "kv-log-macro", - "log", - "memchr", - "once_cell", - "pin-project-lite", - "pin-utils", - "slab", - "wasm-bindgen-futures", -] - -[[package]] -name = "async-task" -version = "4.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a40729d2133846d9ed0ea60a8b9541bccddab49cd30f0715a1da672fe9a2524" - [[package]] name = "async-trait" version = "0.1.66" @@ -316,12 +212,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "atomic-waker" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "debc29dde2e69f9e47506b525f639ed42300fc014a3e007832592448fa8e4599" - [[package]] name = "atty" version = "0.2.14" @@ -467,20 +357,6 @@ dependencies = [ "byte-tools", ] -[[package]] -name = "blocking" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c67b173a56acffd6d2326fb7ab938ba0b00a71480e14902b2591c87bc5741e8" -dependencies = [ - "async-channel", - "async-lock", - "async-task", - "atomic-waker", - "fastrand", - "futures-lite", -] - [[package]] name = "borsh" version = "0.10.2" @@ -632,7 +508,6 @@ version = "3.0.1" dependencies = [ "anyhow", "assert_cmd", - "async-std", "clap", "colored", "contract-build", @@ -655,6 +530,7 @@ dependencies = [ "substrate-build-script-utils", "subxt", "tempfile", + "tokio", "tracing", "tracing-subscriber 0.3.17", "url", @@ -789,15 +665,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "concurrent-queue" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c278839b831783b70278b14df4d45e1beb1aad306c07bb796637de9a0e323e8e" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "constant_time_eq" version = "0.2.5" @@ -1443,21 +1310,6 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" -[[package]] -name = "futures-lite" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" -dependencies = [ - "fastrand", - "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite", - "waker-fn", -] - [[package]] name = "futures-macro" version = "0.3.28" @@ -1570,18 +1422,6 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" -[[package]] -name = "gloo-timers" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "h2" version = "0.3.18" @@ -2322,15 +2162,6 @@ dependencies = [ "cpufeatures", ] -[[package]] -name = "kv-log-macro" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" -dependencies = [ - "log", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -2435,7 +2266,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if", - "value-bag", ] [[package]] @@ -2767,12 +2597,6 @@ version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1ad0aff30c1da14b1254fcb2af73e1fa9a28670e584a626f53a369d0e157304" -[[package]] -name = "parking" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" - [[package]] name = "parking_lot" version = "0.12.1" @@ -2864,22 +2688,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8d0eef3571242013a0d5dc84861c3ae4a652e56e12adf8bdc26ff5f8cb34c94" -[[package]] -name = "polling" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e1f879b2998099c2d69ab9605d145d5b661195627eccc680002c4918a7fb6fa" -dependencies = [ - "autocfg", - "bitflags", - "cfg-if", - "concurrent-queue", - "libc", - "log", - "pin-project-lite", - "windows-sys 0.45.0", -] - [[package]] name = "ppv-lite86" version = "0.2.17" @@ -4814,16 +4622,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" -[[package]] -name = "value-bag" -version = "1.0.0-alpha.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55" -dependencies = [ - "ctor", - "version_check", -] - [[package]] name = "version_check" version = "0.9.4" @@ -4862,12 +4660,6 @@ dependencies = [ "libc", ] -[[package]] -name = "waker-fn" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" - [[package]] name = "walkdir" version = "2.3.3" @@ -4925,18 +4717,6 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "wasm-bindgen-macro" version = "0.2.84" diff --git a/build-image/.dockerignore b/build-image/.dockerignore new file mode 100644 index 000000000..2f7896d1d --- /dev/null +++ b/build-image/.dockerignore @@ -0,0 +1 @@ +target/ diff --git a/build-image/Dockerfile b/build-image/Dockerfile new file mode 100644 index 000000000..ae07af6c6 --- /dev/null +++ b/build-image/Dockerfile @@ -0,0 +1,72 @@ +ARG VCS_REF=master +ARG REGISTRY_PATH=docker.io/paritytech +ARG BUILD_DATE + +FROM ${REGISTRY_PATH}/base-ci-linux:latest + +# metadata +LABEL io.parity.image.vendor="Parity Technologies" \ + io.parity.image.title="${REGISTRY_PATH}/ink-build-image" \ + io.parity.image.documentation="https://github.com/paritytech/scripts/blob/${VCS_REF}/\ + dockerfiles/ink-ci-linux/README.md" \ + io.parity.image.description="Inherits from docker.io/paritytech/base-ci-linux. \ + rust nightly, clippy, rustfmt, miri, rust-src, rustc-dev, grcov, rust-covfix, \ + llvm-tools-preview, cargo-contract, xargo, binaryen, parallel, codecov, ink, solang" \ + io.parity.image.revision="${VCS_REF}" \ + io.parity.image.documentation="https://github.com/paritytech/cargo-contract/blob/${VCS_REF}/\ + build-image/README.md" \ + io.parity.image.created="${BUILD_DATE}" + +WORKDIR /builds + +RUN set -eux; \ + apt-get -y update && \ + + # `binaryen` is needed by `cargo-contract` for optimizing Wasm files. + # We fetch the latest release which contains a Linux binary. + curl -L $(curl --silent https://api.github.com/repos/WebAssembly/binaryen/releases \ + | jq -r '.[0].assets | [.[] | .browser_download_url] | map(select(match("x86_64-linux\\.tar\\.gz$"))) | .[0]' \ + ) | tar -xz -C /usr/local/bin/ --wildcards --strip-components=2 'binaryen-*/bin/wasm-opt' && \ + + # Install LLVM for solang + curl -L https://github.com/hyperledger/solang-llvm/releases/download/llvm15-1/llvm15.0-linux-x86-64.tar.xz > llvm15.0-linux-x86-64.tar.xz && \ + tar Jxf llvm15.0-linux-x86-64.tar.xz && \ + export PATH=$(pwd)/llvm15.0/bin:$PATH && \ + + # The stable toolchain is used to build ink! itself through the use of the + # `RUSTC_BOOSTRAP=1` environment variable. We also need to install the + # `wasm32-unknown-unknown` target since that's the platform that ink! smart contracts + # run on. + rustup target add wasm32-unknown-unknown --toolchain stable && \ + rustup component add rust-src clippy rustfmt --toolchain stable && \ + rustup default stable && \ + + # `cargo-dylint` and `dylint-link` are dependencies needed to run `cargo-contract`. + cargo install cargo-dylint dylint-link && \ + + # Install the latest stable release of `cargo-contract` + cargo install cargo-contract && \ + + # Install the latest `solang` + cargo install solang && \ + + # Versions + rustup show && \ + cargo --version && \ + cargo-contract --version && \ + wasm-opt --version && \ + + # Clean up and remove compilation artifacts that a cargo install creates (>250M). + rm -rf "${CARGO_HOME}/registry" "${CARGO_HOME}/git" /root/.cache/sccache && \ + + # apt clean up + apt-get remove -y gnupg && \ + apt-get autoremove -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# A contract workdir which should be bind mounted to the container. See README for detailed documentation. +WORKDIR /contract + +# Default entry point to the container. Can be overwritten by the child's image +ENTRYPOINT [ "cargo", "contract", "build", "--release", "--generate", "code-only" ] diff --git a/build-image/README.md b/build-image/README.md new file mode 100644 index 000000000..e74e5db25 --- /dev/null +++ b/build-image/README.md @@ -0,0 +1,2 @@ +# Verifiable build using Docker + diff --git a/build-image/build.sh b/build-image/build.sh new file mode 100644 index 000000000..4d3c241c8 --- /dev/null +++ b/build-image/build.sh @@ -0,0 +1,4 @@ +docker run -d \ + --name flipper \ + --mount type=bind,source="$(pwd)",target="/contract" \ + parity/ver-build diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index 472e0f0ac..d462c9fbd 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -119,6 +119,7 @@ pub struct ExecuteArgs { pub skip_wasm_validation: bool, pub target: Target, pub max_memory_pages: u32, + pub verifiable: bool, } impl Default for ExecuteArgs { @@ -138,6 +139,7 @@ impl Default for ExecuteArgs { skip_wasm_validation: Default::default(), target: Default::default(), max_memory_pages: DEFAULT_MAX_MEMORY_PAGES, + verifiable: Default::default(), } } } @@ -678,6 +680,7 @@ pub fn execute(args: ExecuteArgs) -> Result { skip_wasm_validation, target, max_memory_pages, + verifiable, } = args; // The CLI flag `optimization-passes` overwrites optimization passes which are @@ -721,7 +724,7 @@ pub fn execute(args: ExecuteArgs) -> Result { } }; - let build = + let local_build = || -> Result<(Option, BuildInfo, PathBuf, BuildSteps)> { let mut build_steps = BuildSteps::new(); let pre_fingerprint = Fingerprint::new(&crate_metadata)?; @@ -882,12 +885,12 @@ pub fn execute(args: ExecuteArgs) -> Result { BuildArtifacts::CodeOnly => { // when building only the code metadata will become stale clean_metadata(); - let (opt_result, _, dest_wasm, _) = build()?; + let (opt_result, _, dest_wasm, _) = local_build()?; (opt_result, None, Some(dest_wasm)) } BuildArtifacts::All => { - let (opt_result, build_info, dest_wasm, build_steps) = - build().map_err(|e| { + let (opt_result, build_info, dest_wasm, build_steps) = local_build() + .map_err(|e| { // build error -> bundle is stale clean_metadata(); e @@ -916,6 +919,7 @@ pub fn execute(args: ExecuteArgs) -> Result { build_steps, &unstable_flags, build_info, + None, )?; } (opt_result, Some(metadata_result), Some(dest_wasm)) @@ -934,6 +938,8 @@ pub fn execute(args: ExecuteArgs) -> Result { }) } +fn docker_build() {} + /// Unique fingerprint for a file to detect whether it has changed. #[derive(Debug, Eq, PartialEq)] struct Fingerprint { diff --git a/crates/build/src/metadata.rs b/crates/build/src/metadata.rs index d6cad3fa2..49a856f72 100644 --- a/crates/build/src/metadata.rs +++ b/crates/build/src/metadata.rs @@ -119,6 +119,7 @@ pub(crate) fn execute( mut build_steps: BuildSteps, unstable_options: &UnstableFlags, build_info: BuildInfo, + image: Option, ) -> Result<()> { // build the extended contract project metadata let ExtendedMetadataResult { @@ -160,7 +161,7 @@ pub(crate) fn execute( let ink_meta: serde_json::Map = serde_json::from_slice(&output.stdout)?; - let metadata = ContractMetadata::new(source, contract, user, ink_meta); + let metadata = ContractMetadata::new(source, contract, image, user, ink_meta); { let mut metadata = metadata.clone(); metadata.remove_source_wasm_attribute(); diff --git a/crates/cargo-contract/Cargo.toml b/crates/cargo-contract/Cargo.toml index 9f2f9663b..10f27b9f9 100644 --- a/crates/cargo-contract/Cargo.toml +++ b/crates/cargo-contract/Cargo.toml @@ -35,7 +35,7 @@ url = { version = "2.3.1", features = ["serde"] } rust_decimal = "1.29" # dependencies for extrinsics (deploying and calling a contract) -async-std = { version = "1.12.0", features = ["attributes", "tokio1"] } +tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } sp-core = "20.0.0" sp-runtime = "23.0.0" sp-weights = "19.0.0" diff --git a/crates/cargo-contract/src/cmd/build.rs b/crates/cargo-contract/src/cmd/build.rs index b79b7a67b..ee0c2d4ae 100644 --- a/crates/cargo-contract/src/cmd/build.rs +++ b/crates/cargo-contract/src/cmd/build.rs @@ -121,6 +121,10 @@ pub struct BuildCommand { /// The maximum number of pages available for a wasm contract to allocate. #[clap(long, default_value_t = contract_build::DEFAULT_MAX_MEMORY_PAGES)] max_memory_pages: u32, + /// Executes the build inside a docker container to produce a verifiable bundle. + /// Requires docker daemon running. + #[clap(long, short = 'V', default_value_t = false)] + verifiable: bool, } impl BuildCommand { @@ -165,6 +169,7 @@ impl BuildCommand { skip_wasm_validation: self.skip_wasm_validation, target: self.target, max_memory_pages: self.max_memory_pages, + verifiable: self.verifiable, }; contract_build::execute(args) @@ -209,6 +214,7 @@ impl CheckCommand { skip_wasm_validation: false, target: self.target, max_memory_pages: 0, + verifiable: false, }; contract_build::execute(args) diff --git a/crates/cargo-contract/src/cmd/extrinsics/call.rs b/crates/cargo-contract/src/cmd/extrinsics/call.rs index 2e8bb5dc3..651edd948 100644 --- a/crates/cargo-contract/src/cmd/extrinsics/call.rs +++ b/crates/cargo-contract/src/cmd/extrinsics/call.rs @@ -55,6 +55,9 @@ use contract_transcode::Value; use pallet_contracts_primitives::ContractExecResult; use scale::Encode; use sp_weights::Weight; +use tokio::runtime::{ + Runtime, +}; use std::fmt::Debug; use subxt::{ @@ -108,7 +111,7 @@ impl CallCommand { let signer = super::pair_signer(self.extrinsic_opts.signer()?); - async_std::task::block_on(async { + Runtime::new().unwrap().block_on(async { let url = self.extrinsic_opts.url_to_string(); let client = OnlineClient::from_url(url.clone()).await?; diff --git a/crates/cargo-contract/src/cmd/extrinsics/instantiate.rs b/crates/cargo-contract/src/cmd/extrinsics/instantiate.rs index 2f0daea20..9c3e5d517 100644 --- a/crates/cargo-contract/src/cmd/extrinsics/instantiate.rs +++ b/crates/cargo-contract/src/cmd/extrinsics/instantiate.rs @@ -63,6 +63,7 @@ use subxt::{ Config, OnlineClient, }; +use tokio::runtime::Runtime; #[derive(Debug, clap::Args)] pub struct InstantiateCommand { @@ -125,7 +126,7 @@ impl InstantiateCommand { }; let salt = self.salt.clone().map(|s| s.0).unwrap_or_default(); - async_std::task::block_on(async move { + Runtime::new().unwrap().block_on(async { let client = OnlineClient::from_url(url.clone()).await?; let token_metadata = TokenMetadata::query(&client).await?; diff --git a/crates/cargo-contract/src/cmd/extrinsics/integration_tests.rs b/crates/cargo-contract/src/cmd/extrinsics/integration_tests.rs index bf0a8e328..d260a08b8 100644 --- a/crates/cargo-contract/src/cmd/extrinsics/integration_tests.rs +++ b/crates/cargo-contract/src/cmd/extrinsics/integration_tests.rs @@ -155,7 +155,7 @@ pub fn init_tracing_subscriber() { /// Requires [`substrate-contracts-node`](https://github.com/paritytech/substrate-contracts-node/) to /// be installed and available on the `PATH`, and the no other process running using the /// default port `9944`. -#[async_std::test] +#[tokio::test] async fn build_upload_instantiate_call() { init_tracing_subscriber(); @@ -235,7 +235,7 @@ async fn build_upload_instantiate_call() { /// Sanity test the whole lifecycle of: /// build -> upload -> remove -#[async_std::test] +#[tokio::test] async fn build_upload_remove() { init_tracing_subscriber(); @@ -300,7 +300,7 @@ async fn build_upload_remove() { /// Requires [`substrate-contracts-node`](https://github.com/paritytech/substrate-contracts-node/) to /// be installed and available on the `PATH`, and the no other process running using the /// default port `9944`. -#[async_std::test] +#[tokio::test] async fn build_upload_instantiate_info() { init_tracing_subscriber(); diff --git a/crates/cargo-contract/src/cmd/extrinsics/remove.rs b/crates/cargo-contract/src/cmd/extrinsics/remove.rs index ae61a315d..238ec1f2e 100644 --- a/crates/cargo-contract/src/cmd/extrinsics/remove.rs +++ b/crates/cargo-contract/src/cmd/extrinsics/remove.rs @@ -44,6 +44,9 @@ use subxt::{ Config, OnlineClient, }; +use tokio::runtime::{ + Runtime, +}; #[derive(Debug, clap::Args)] #[clap(name = "remove", about = "Remove a contract's code")] @@ -87,7 +90,7 @@ impl RemoveCommand { } }?; - async_std::task::block_on(async { + Runtime::new().unwrap().block_on(async { let url = self.extrinsic_opts.url_to_string(); let client = OnlineClient::from_url(url.clone()).await?; if let Some(code_removed) = self diff --git a/crates/cargo-contract/src/cmd/extrinsics/upload.rs b/crates/cargo-contract/src/cmd/extrinsics/upload.rs index 0fa31db90..78bc8e420 100644 --- a/crates/cargo-contract/src/cmd/extrinsics/upload.rs +++ b/crates/cargo-contract/src/cmd/extrinsics/upload.rs @@ -48,6 +48,9 @@ use subxt::{ Config, OnlineClient, }; +use tokio::runtime::{ + Runtime, +}; #[derive(Debug, clap::Args)] #[clap(name = "upload", about = "Upload a contract's code")] @@ -77,7 +80,7 @@ impl UploadCommand { })?; let code_hash = code.code_hash(); - async_std::task::block_on(async { + Runtime::new().unwrap().block_on(async { let url = self.extrinsic_opts.url_to_string(); let client = OnlineClient::from_url(url.clone()).await?; diff --git a/crates/cargo-contract/src/cmd/info.rs b/crates/cargo-contract/src/cmd/info.rs index b682bd962..753960f20 100644 --- a/crates/cargo-contract/src/cmd/info.rs +++ b/crates/cargo-contract/src/cmd/info.rs @@ -38,6 +38,9 @@ use subxt::{ Config, OnlineClient, }; +use tokio::runtime::{ + Runtime, +}; #[derive(Debug, clap::Args)] #[clap(name = "info", about = "Get infos from a contract")] @@ -65,7 +68,7 @@ impl InfoCommand { self.contract ); - async_std::task::block_on(async { + Runtime::new().unwrap().block_on(async { let url = self.url.clone(); let client = OnlineClient::::from_url(url).await?; diff --git a/crates/metadata/src/lib.rs b/crates/metadata/src/lib.rs index 474fdb5a8..d3e7c88ba 100644 --- a/crates/metadata/src/lib.rs +++ b/crates/metadata/src/lib.rs @@ -99,6 +99,8 @@ pub struct ContractMetadata { pub source: Source, /// Metadata about the contract. pub contract: Contract, + /// If the contract is verifiable, then the Docker image id is specified. + pub image: Option, /// Additional user-defined metadata. #[serde(skip_serializing_if = "Option::is_none")] pub user: Option, @@ -112,12 +114,14 @@ impl ContractMetadata { pub fn new( source: Source, contract: Contract, + image: Option, user: Option, abi: Map, ) -> Self { Self { source, contract, + image, user, abi, } @@ -737,7 +741,13 @@ mod tests { .unwrap() .clone(); - let metadata = ContractMetadata::new(source, contract, Some(user), abi_json); + let metadata = ContractMetadata::new( + source, + contract, + Some(String::from("parity/ver-build:latest")), + Some(user), + abi_json, + ); let json = serde_json::to_value(&metadata).unwrap(); let expected = json! { @@ -753,6 +763,7 @@ mod tests { "example_name": "increment" } }, + "image": "parity/ver-build:latest", "contract": { "name": "incrementer", "version": "2.1.0", @@ -808,7 +819,7 @@ mod tests { .unwrap() .clone(); - let metadata = ContractMetadata::new(source, contract, None, abi_json); + let metadata = ContractMetadata::new(source, contract, None, None, abi_json); let json = serde_json::to_value(&metadata).unwrap(); let expected = json! { @@ -820,6 +831,7 @@ mod tests { "Parity Technologies " ], }, + "image": serde_json::Value::Null, "source": { "hash": "0x0000000000000000000000000000000000000000000000000000000000000000", "language": "ink! 2.1.0", @@ -895,7 +907,8 @@ mod tests { .unwrap() .clone(); - let metadata = ContractMetadata::new(source, contract, Some(user), abi_json); + let metadata = + ContractMetadata::new(source, contract, None, Some(user), abi_json); let json = serde_json::to_value(&metadata).unwrap(); let decoded = serde_json::from_value::(json); From a11bb5b7cd00162160a0aa3c5a181b57b339cf53 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Mon, 5 Jun 2023 18:02:44 +0100 Subject: [PATCH 02/64] docker client draft --- Cargo.lock | 121 +++++++++++ crates/build/Cargo.toml | 3 + crates/build/src/args.rs | 12 +- crates/build/src/lib.rs | 197 +++++++++++++++++- crates/build/src/metadata.rs | 4 +- crates/build/src/wasm_opt.rs | 2 +- crates/cargo-contract/Cargo.toml | 3 +- crates/cargo-contract/src/cmd/build.rs | 8 +- .../cargo-contract/src/cmd/extrinsics/call.rs | 4 +- .../src/cmd/extrinsics/remove.rs | 4 +- .../src/cmd/extrinsics/upload.rs | 4 +- crates/cargo-contract/src/cmd/info.rs | 4 +- 12 files changed, 339 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2cb5dce1d..cd9960f97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -357,6 +357,45 @@ dependencies = [ "byte-tools", ] +[[package]] +name = "bollard" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af254ed2da4936ef73309e9597180558821cb16ae9bba4cb24ce6b612d8d80ed" +dependencies = [ + "base64 0.21.0", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "http", + "hyper", + "hyperlocal", + "log", + "pin-project-lite", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror", + "tokio", + "tokio-util", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.42.0-rc.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "602bda35f33aeb571cef387dcd4042c643a8bf689d8aaac2cc47ea24cb7bc7e0" +dependencies = [ + "serde", + "serde_with", +] + [[package]] name = "borsh" version = "0.10.2" @@ -584,6 +623,7 @@ dependencies = [ "iana-time-zone", "num-integer", "num-traits", + "serde", "winapi", ] @@ -677,6 +717,7 @@ version = "3.0.1" dependencies = [ "anyhow", "blake2", + "bollard", "cargo_metadata", "clap", "colored", @@ -695,6 +736,8 @@ dependencies = [ "strum", "tempfile", "term_size", + "tokio", + "tokio-stream", "toml 0.7.4", "tracing", "url", @@ -1630,6 +1673,19 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "hyperlocal" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fafdf7b2b2de7c9784f76e02c0935e65a8117ec3b768644379983ab333ac98c" +dependencies = [ + "futures-util", + "hex", + "hyper", + "pin-project", + "tokio", +] + [[package]] name = "iana-time-zone" version = "0.1.53" @@ -3474,6 +3530,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcec881020c684085e55a25f7fd888954d56609ef363479dc5a1305eb0d40cab" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.11", +] + [[package]] name = "serde_spanned" version = "0.6.2" @@ -3483,6 +3550,33 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ff71d2c147a7b57362cead5e22f772cd52f6ab31cfcd9edcd7f6aeb2a0afbe" +dependencies = [ + "base64 0.13.1", + "chrono", + "hex", + "indexmap", + "serde", + "serde_json", + "time", +] + [[package]] name = "sha-1" version = "0.9.8" @@ -4249,6 +4343,33 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3403384eaacbca9923fa06940178ac13e4edb725486d70e8e15881d0c836cc" +dependencies = [ + "itoa", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" + +[[package]] +name = "time-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" +dependencies = [ + "time-core", +] + [[package]] name = "tiny-bip39" version = "1.0.0" diff --git a/crates/build/Cargo.toml b/crates/build/Cargo.toml index df13faa40..92a931897 100644 --- a/crates/build/Cargo.toml +++ b/crates/build/Cargo.toml @@ -38,6 +38,9 @@ wasm-opt = "0.112.0" which = "4.4.0" zip = { version = "0.6.6", default-features = false } strum = { version = "0.24", features = ["derive"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +tokio-stream = "0.1" +bollard = "0.14" contract-metadata = { version = "3.0.1", path = "../metadata" } diff --git a/crates/build/src/args.rs b/crates/build/src/args.rs index 80169e082..f238302a4 100644 --- a/crates/build/src/args.rs +++ b/crates/build/src/args.rs @@ -45,7 +45,7 @@ impl TryFrom<&VerbosityFlags> for Verbosity { } /// Denotes if output should be printed to stdout. -#[derive(Clone, Copy, Default, serde::Serialize, Eq, PartialEq)] +#[derive(Clone, Copy, Default, serde::Serialize, serde::Deserialize, Eq, PartialEq)] pub enum Verbosity { /// Use default output #[default] @@ -89,7 +89,15 @@ impl Network { /// Describes which artifacts to generate #[derive( - Copy, Clone, Default, Eq, PartialEq, Debug, clap::ValueEnum, serde::Serialize, + Copy, + Clone, + Default, + Eq, + PartialEq, + Debug, + clap::ValueEnum, + serde::Serialize, + serde::Deserialize, )] #[clap(name = "build-artifacts")] pub enum BuildArtifacts { diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index d462c9fbd..e0d707b32 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -17,6 +17,21 @@ #![doc = include_str!("../README.md")] #![deny(unused_crate_dependencies)] +use bollard::{ + container::{ + Config, + CreateContainerOptions, + WaitContainerOptions, + }, + image::ListImagesOptions, + service::{ + HostConfig, + Mount, + MountTypeEnum, + }, + Docker, +}; +use tokio_stream::StreamExt; use which as _; mod args; @@ -119,7 +134,6 @@ pub struct ExecuteArgs { pub skip_wasm_validation: bool, pub target: Target, pub max_memory_pages: u32, - pub verifiable: bool, } impl Default for ExecuteArgs { @@ -139,13 +153,12 @@ impl Default for ExecuteArgs { skip_wasm_validation: Default::default(), target: Default::default(), max_memory_pages: DEFAULT_MAX_MEMORY_PAGES, - verifiable: Default::default(), } } } /// Result of the build process. -#[derive(serde::Serialize)] +#[derive(serde::Serialize, serde::Deserialize)] pub struct BuildResult { /// Path to the resulting Wasm file. pub dest_wasm: Option, @@ -161,8 +174,10 @@ pub struct BuildResult { pub build_artifact: BuildArtifacts, /// The verbosity flags. pub verbosity: Verbosity, + /// Image used for the verifiable build + pub image: Option, /// The type of formatting to use for the build output. - #[serde(skip_serializing)] + #[serde(skip_serializing, skip_deserializing)] pub output_type: OutputType, } @@ -680,7 +695,6 @@ pub fn execute(args: ExecuteArgs) -> Result { skip_wasm_validation, target, max_memory_pages, - verifiable, } = args; // The CLI flag `optimization-passes` overwrites optimization passes which are @@ -934,11 +948,181 @@ pub fn execute(args: ExecuteArgs) -> Result { build_mode, build_artifact, verbosity, + image: None, output_type, }) } -fn docker_build() {} +pub fn docker_build(args: ExecuteArgs) -> Result { + let ExecuteArgs { + manifest_path, + verbosity, + features, + build_mode, + network, + unstable_flags, + optimization_passes, + keep_debug_symbols, + output_type, + target, + .. + } = args; + tokio::runtime::Runtime::new()?.block_on(async { + let steps = BuildSteps::new(); + let client = Docker::connect_with_socket_defaults()?; + + let optimization_passes = match optimization_passes { + Some(opt_passes) => opt_passes, + None => { + let mut manifest = Manifest::new(manifest_path.clone())?; + match manifest.profile_optimization_passes() { + // if no setting is found, neither on the cli nor in the profile, + // then we use the default + None => OptimizationPasses::default(), + Some(opt_passes) => opt_passes, + } + } + }; + + let images = client + .list_images(Some(ListImagesOptions:: { + all: true, + ..Default::default() + })) + .await + .unwrap(); + + let build_image = images.iter().find(|i| { + i.labels.get("io.parity.image.title") == Some(&"parity/ver-build".to_string()) + }); + if build_image.is_none() { + return Err(anyhow::anyhow!("No image found")) + } + let build_image = build_image.unwrap(); + + let options = Some(CreateContainerOptions { + name: "ink-container", + platform: Some("linux/amd64"), + }); + + let crate_metadata = CrateMetadata::collect(&manifest_path, target)?; + let host_folder = crate_metadata + .manifest_path + .absolute_directory()? + .parent() + .expect("Manifests is located in unreachable directory") + .to_str() + .expect("Cannot convert Path to string.") + .to_owned(); + + let mount = Mount { + target: Some(String::from("/contract")), + source: Some(host_folder.clone()), + typ: Some(MountTypeEnum::BIND), + ..Default::default() + }; + let host_cfg = Some(HostConfig { + mounts: Some(vec![mount]), + ..Default::default() + }); + + let cmds = vec![ + "cargo", + "contract", + "build", + "--release", + "--generate", + "code-only", + "--output-json", + ">", + "result.json", + ] + .iter() + .map(|s| s.to_string()) + .collect(); + + let config = Config { + image: Some(build_image.id.clone()), + entrypoint: Some(cmds), + host_config: host_cfg, + attach_stderr: Some(true), + ..Default::default() + }; + + let container_id = client.create_container(options, config).await?.id; + client + .start_container::(&container_id, None) + .await?; + + let options = Some(WaitContainerOptions { + condition: "not-running", + }); + + let mut wait_stream = client.wait_container(&container_id, options); + while wait_stream.next().await.is_some() {} + + let result_contents = + std::fs::read_to_string(format!("{}/result.json", &host_folder))?; + let build_result: BuildResult = + serde_json::from_str(&result_contents).map_err(|_| { + anyhow::anyhow!( + "Error parsing output from docker build. The build probably failed!" + ) + })?; + + let metadata_result = MetadataArtifacts { + dest_metadata: crate_metadata.metadata_path(), + dest_bundle: crate_metadata.contract_bundle_path(), + }; + + // skip metadata generation if contract unchanged and all metadata artifacts + // exist. + if build_result.optimization_result.is_some() + || !metadata_result.dest_metadata.exists() + || !metadata_result.dest_bundle.exists() + { + let cargo_contract_version = if let Ok(version) = Version::parse(VERSION) { + version + } else { + anyhow::bail!( + "Unable to parse version number for the currently running \ + `cargo-contract` binary." + ); + }; + + let build_info = BuildInfo { + rust_toolchain: util::rust_toolchain()?, + cargo_contract_version, + build_mode, + wasm_opt_settings: WasmOptSettings { + optimization_passes, + keep_debug_symbols, + }, + }; + // if metadata build fails after a code build it might become stale + metadata::execute( + &crate_metadata, + build_result.dest_wasm.as_ref().unwrap().as_path(), + &metadata_result, + &features, + network, + verbosity, + steps, + &unstable_flags, + build_info, + Some(build_image.id.clone()), + )?; + } + + client.remove_container(&container_id, None).await?; + + Ok(BuildResult { + metadata_result: Some(metadata_result), + output_type, + ..build_result + }) + }) +} /// Unique fingerprint for a file to detect whether it has changed. #[derive(Debug, Eq, PartialEq)] @@ -1103,6 +1287,7 @@ mod unit_tests { }), build_mode: Default::default(), build_artifact: Default::default(), + image: None, verbosity: Verbosity::Quiet, output_type: OutputType::Json, }; diff --git a/crates/build/src/metadata.rs b/crates/build/src/metadata.rs index 49a856f72..29b8b8f3f 100644 --- a/crates/build/src/metadata.rs +++ b/crates/build/src/metadata.rs @@ -56,7 +56,7 @@ use std::{ use url::Url; /// Artifacts resulting from metadata generation. -#[derive(serde::Serialize)] +#[derive(serde::Serialize, serde::Deserialize)] pub struct MetadataArtifacts { /// Path to the resulting metadata file. pub dest_metadata: PathBuf, @@ -109,7 +109,7 @@ pub struct WasmOptSettings { /// /// It does so by generating and invoking a temporary workspace member. #[allow(clippy::too_many_arguments)] -pub(crate) fn execute( +pub fn execute( crate_metadata: &CrateMetadata, final_contract_wasm: &Path, metadata_artifacts: &MetadataArtifacts, diff --git a/crates/build/src/wasm_opt.rs b/crates/build/src/wasm_opt.rs index 3f0fcb935..de7009eb5 100644 --- a/crates/build/src/wasm_opt.rs +++ b/crates/build/src/wasm_opt.rs @@ -162,7 +162,7 @@ impl From for OptimizationOptions { } /// Result of the optimization process. -#[derive(serde::Serialize)] +#[derive(serde::Serialize, serde::Deserialize)] pub struct OptimizationResult { /// The path of the optimized Wasm file. pub dest_wasm: PathBuf, diff --git a/crates/cargo-contract/Cargo.toml b/crates/cargo-contract/Cargo.toml index 10f27b9f9..d44c4b567 100644 --- a/crates/cargo-contract/Cargo.toml +++ b/crates/cargo-contract/Cargo.toml @@ -34,8 +34,9 @@ serde_json = "1.0.96" url = { version = "2.3.1", features = ["serde"] } rust_decimal = "1.29" + # dependencies for extrinsics (deploying and calling a contract) -tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } sp-core = "20.0.0" sp-runtime = "23.0.0" sp-weights = "19.0.0" diff --git a/crates/cargo-contract/src/cmd/build.rs b/crates/cargo-contract/src/cmd/build.rs index ee0c2d4ae..abcf4fb2b 100644 --- a/crates/cargo-contract/src/cmd/build.rs +++ b/crates/cargo-contract/src/cmd/build.rs @@ -169,10 +169,13 @@ impl BuildCommand { skip_wasm_validation: self.skip_wasm_validation, target: self.target, max_memory_pages: self.max_memory_pages, - verifiable: self.verifiable, }; - contract_build::execute(args) + if self.verifiable { + contract_build::docker_build(args) + } else { + contract_build::execute(args) + } } } @@ -214,7 +217,6 @@ impl CheckCommand { skip_wasm_validation: false, target: self.target, max_memory_pages: 0, - verifiable: false, }; contract_build::execute(args) diff --git a/crates/cargo-contract/src/cmd/extrinsics/call.rs b/crates/cargo-contract/src/cmd/extrinsics/call.rs index 651edd948..c955a38bb 100644 --- a/crates/cargo-contract/src/cmd/extrinsics/call.rs +++ b/crates/cargo-contract/src/cmd/extrinsics/call.rs @@ -55,9 +55,7 @@ use contract_transcode::Value; use pallet_contracts_primitives::ContractExecResult; use scale::Encode; use sp_weights::Weight; -use tokio::runtime::{ - Runtime, -}; +use tokio::runtime::Runtime; use std::fmt::Debug; use subxt::{ diff --git a/crates/cargo-contract/src/cmd/extrinsics/remove.rs b/crates/cargo-contract/src/cmd/extrinsics/remove.rs index 238ec1f2e..dd8b826e4 100644 --- a/crates/cargo-contract/src/cmd/extrinsics/remove.rs +++ b/crates/cargo-contract/src/cmd/extrinsics/remove.rs @@ -44,9 +44,7 @@ use subxt::{ Config, OnlineClient, }; -use tokio::runtime::{ - Runtime, -}; +use tokio::runtime::Runtime; #[derive(Debug, clap::Args)] #[clap(name = "remove", about = "Remove a contract's code")] diff --git a/crates/cargo-contract/src/cmd/extrinsics/upload.rs b/crates/cargo-contract/src/cmd/extrinsics/upload.rs index 78bc8e420..adf3a3280 100644 --- a/crates/cargo-contract/src/cmd/extrinsics/upload.rs +++ b/crates/cargo-contract/src/cmd/extrinsics/upload.rs @@ -48,9 +48,7 @@ use subxt::{ Config, OnlineClient, }; -use tokio::runtime::{ - Runtime, -}; +use tokio::runtime::Runtime; #[derive(Debug, clap::Args)] #[clap(name = "upload", about = "Upload a contract's code")] diff --git a/crates/cargo-contract/src/cmd/info.rs b/crates/cargo-contract/src/cmd/info.rs index 753960f20..7f6e1df8f 100644 --- a/crates/cargo-contract/src/cmd/info.rs +++ b/crates/cargo-contract/src/cmd/info.rs @@ -38,9 +38,7 @@ use subxt::{ Config, OnlineClient, }; -use tokio::runtime::{ - Runtime, -}; +use tokio::runtime::Runtime; #[derive(Debug, clap::Args)] #[clap(name = "info", about = "Get infos from a contract")] From ade7003918aec5a27c0577cff1e0f3de59dcd19c Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 6 Jun 2023 13:23:29 +0100 Subject: [PATCH 03/64] refactoring execute func --- crates/build/src/lib.rs | 421 +++++++++++++++++++++------------------- 1 file changed, 226 insertions(+), 195 deletions(-) diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index e0d707b32..e4d911137 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -280,11 +280,11 @@ fn exec_cargo_for_onchain_target( crate_metadata: &CrateMetadata, command: &str, features: &Features, - build_mode: BuildMode, - network: Network, - verbosity: Verbosity, + build_mode: &BuildMode, + network: &Network, + verbosity: &Verbosity, unstable_flags: &UnstableFlags, - target: Target, + target: &Target, ) -> Result<()> { let cargo_build = |manifest_path: &ManifestPath| { let target_dir = format!( @@ -302,7 +302,7 @@ fn exec_cargo_for_onchain_target( network.append_to_args(&mut args); let mut features = features.clone(); - if build_mode == BuildMode::Debug { + if build_mode == &BuildMode::Debug { features.push("ink/ink-debug"); } else { args.push("-Zbuild-std-features=panic_immediate_abort".to_owned()); @@ -333,7 +333,7 @@ fn exec_cargo_for_onchain_target( }; let cargo = - util::cargo_cmd(command, &args, manifest_path.directory(), verbosity, env); + util::cargo_cmd(command, &args, manifest_path.directory(), *verbosity, env); invoke_cargo_and_scan_for_error(cargo) }; @@ -689,20 +689,19 @@ pub fn execute(args: ExecuteArgs) -> Result { build_artifact, unstable_flags, optimization_passes, - keep_debug_symbols, lint, output_type, - skip_wasm_validation, target, - max_memory_pages, - } = args; + .. + } = &args; // The CLI flag `optimization-passes` overwrites optimization passes which are // potentially defined in the `Cargo.toml` profile. let optimization_passes = match optimization_passes { - Some(opt_passes) => opt_passes, + Some(opt_passes) => *opt_passes, None => { let mut manifest = Manifest::new(manifest_path.clone())?; + match manifest.profile_optimization_passes() { // if no setting is found, neither on the cli nor in the profile, // then we use the default @@ -712,162 +711,13 @@ pub fn execute(args: ExecuteArgs) -> Result { } }; - let crate_metadata = CrateMetadata::collect(&manifest_path, target)?; + let crate_metadata = CrateMetadata::collect(manifest_path, *target)?; - assert_compatible_ink_dependencies(&manifest_path, verbosity)?; - if build_mode == BuildMode::Debug { + assert_compatible_ink_dependencies(manifest_path, *verbosity)?; + if build_mode == &BuildMode::Debug { assert_debug_mode_supported(&crate_metadata.ink_version)?; } - let maybe_lint = |steps: &mut BuildSteps| -> Result<()> { - let total_steps = build_artifact.steps(); - if lint { - steps.set_total_steps(total_steps + 1); - maybe_println!( - verbosity, - " {} {}", - format!("{steps}").bold(), - "Checking ink! linting rules".bright_green().bold() - ); - steps.increment_current(); - exec_cargo_dylint(&crate_metadata, verbosity)?; - Ok(()) - } else { - steps.set_total_steps(total_steps); - Ok(()) - } - }; - - let local_build = - || -> Result<(Option, BuildInfo, PathBuf, BuildSteps)> { - let mut build_steps = BuildSteps::new(); - let pre_fingerprint = Fingerprint::new(&crate_metadata)?; - - maybe_println!( - verbosity, - " {} {}", - format!("{build_steps}").bold(), - "Building cargo project".bright_green().bold() - ); - build_steps.increment_current(); - exec_cargo_for_onchain_target( - &crate_metadata, - "build", - &features, - build_mode, - network, - verbosity, - &unstable_flags, - target, - )?; - - // we persist the latest target we used so we trigger a rebuild when we switch - fs::write(&crate_metadata.target_file_path, target.llvm_target())?; - - let cargo_contract_version = if let Ok(version) = Version::parse(VERSION) { - version - } else { - anyhow::bail!( - "Unable to parse version number for the currently running \ - `cargo-contract` binary." - ); - }; - - let build_info = BuildInfo { - rust_toolchain: util::rust_toolchain()?, - cargo_contract_version, - build_mode, - wasm_opt_settings: WasmOptSettings { - optimization_passes, - keep_debug_symbols, - }, - }; - - let post_fingerprint = - Fingerprint::new(&crate_metadata)?.ok_or_else(|| { - anyhow::anyhow!( - "Expected '{}' to be generated by build", - crate_metadata.original_code.display() - ) - })?; - - tracing::debug!( - "Fingerprint before build: {:?}, after build: {:?}", - pre_fingerprint, - post_fingerprint - ); - - let dest_code_path = crate_metadata.dest_code.clone(); - - if pre_fingerprint == Some(post_fingerprint) - && crate_metadata.dest_code.exists() - { - tracing::info!( - "No changes in the original wasm at {}, fingerprint {:?}. \ - Skipping Wasm optimization and metadata generation.", - crate_metadata.original_code.display(), - pre_fingerprint - ); - return Ok((None, build_info, dest_code_path, build_steps)) - } - - maybe_lint(&mut build_steps)?; - - maybe_println!( - verbosity, - " {} {}", - format!("{build_steps}").bold(), - "Post processing code".bright_green().bold() - ); - build_steps.increment_current(); - - // remove build artifacts so we don't have anything stale lingering around - for t in Target::iter() { - fs::remove_file( - crate_metadata.dest_code.with_extension(t.dest_extension()), - ) - .ok(); - } - - let original_size = - fs::metadata(&crate_metadata.original_code)?.len() as f64 / 1000.0; - - match target { - Target::Wasm => { - post_process_wasm( - &crate_metadata, - skip_wasm_validation, - &verbosity, - max_memory_pages, - )?; - let handler = - WasmOptHandler::new(optimization_passes, keep_debug_symbols)?; - handler.optimize( - &crate_metadata.dest_code, - &crate_metadata.contract_artifact_name, - )?; - } - Target::RiscV => { - fs::copy(&crate_metadata.original_code, &crate_metadata.dest_code)?; - } - } - - let optimized_size = fs::metadata(&dest_code_path)?.len() as f64 / 1000.0; - - let optimization_result = OptimizationResult { - dest_wasm: crate_metadata.dest_code.clone(), - original_size, - optimized_size, - }; - - Ok(( - Some(optimization_result), - build_info, - crate_metadata.dest_code.clone(), - build_steps, - )) - }; - let clean_metadata = || { fs::remove_file(crate_metadata.metadata_path()).ok(); fs::remove_file(crate_metadata.contract_bundle_path()).ok(); @@ -876,7 +726,13 @@ pub fn execute(args: ExecuteArgs) -> Result { let (opt_result, metadata_result, dest_wasm) = match build_artifact { BuildArtifacts::CheckOnly => { let mut build_steps = BuildSteps::new(); - maybe_lint(&mut build_steps)?; + maybe_lint( + &mut build_steps, + *build_artifact, + *lint, + &crate_metadata, + verbosity, + )?; maybe_println!( verbosity, @@ -887,11 +743,11 @@ pub fn execute(args: ExecuteArgs) -> Result { exec_cargo_for_onchain_target( &crate_metadata, "check", - &features, - BuildMode::Release, + features, + &BuildMode::Release, network, verbosity, - &unstable_flags, + unstable_flags, target, )?; (None, None, None) @@ -899,16 +755,19 @@ pub fn execute(args: ExecuteArgs) -> Result { BuildArtifacts::CodeOnly => { // when building only the code metadata will become stale clean_metadata(); - let (opt_result, _, dest_wasm, _) = local_build()?; + let (opt_result, _, dest_wasm, _) = + local_build(&crate_metadata, &optimization_passes, &args)?; (opt_result, None, Some(dest_wasm)) } BuildArtifacts::All => { - let (opt_result, build_info, dest_wasm, build_steps) = local_build() - .map_err(|e| { - // build error -> bundle is stale - clean_metadata(); - e - })?; + let (opt_result, build_info, dest_wasm, build_steps) = + local_build(&crate_metadata, &optimization_passes, &args).map_err( + |e| { + // build error -> bundle is stale + clean_metadata(); + e + }, + )?; let metadata_result = MetadataArtifacts { dest_metadata: crate_metadata.metadata_path(), @@ -927,11 +786,11 @@ pub fn execute(args: ExecuteArgs) -> Result { &crate_metadata, dest_wasm.as_path(), &metadata_result, - &features, - network, - verbosity, + features, + *network, + *verbosity, build_steps, - &unstable_flags, + unstable_flags, build_info, None, )?; @@ -945,12 +804,184 @@ pub fn execute(args: ExecuteArgs) -> Result { metadata_result, target_directory: crate_metadata.target_directory, optimization_result: opt_result, + build_mode: *build_mode, + build_artifact: *build_artifact, + verbosity: *verbosity, + image: None, + output_type: output_type.clone(), + }) +} + +fn local_build( + crate_metadata: &CrateMetadata, + optimization_passes: &OptimizationPasses, + args: &ExecuteArgs, +) -> Result<(Option, BuildInfo, PathBuf, BuildSteps)> { + let ExecuteArgs { + verbosity, + features, build_mode, + network, build_artifact, + unstable_flags, + keep_debug_symbols, + lint, + skip_wasm_validation, + target, + max_memory_pages, + .. + } = args; + + let mut build_steps = BuildSteps::new(); + let pre_fingerprint = Fingerprint::new(crate_metadata)?; + + maybe_println!( verbosity, - image: None, - output_type, - }) + " {} {}", + format!("{build_steps}").bold(), + "Building cargo project".bright_green().bold() + ); + build_steps.increment_current(); + exec_cargo_for_onchain_target( + crate_metadata, + "build", + features, + build_mode, + network, + verbosity, + unstable_flags, + target, + )?; + + // we persist the latest target we used so we trigger a rebuild when we switch + fs::write(&crate_metadata.target_file_path, target.llvm_target())?; + + let cargo_contract_version = if let Ok(version) = Version::parse(VERSION) { + version + } else { + anyhow::bail!( + "Unable to parse version number for the currently running \ + `cargo-contract` binary." + ); + }; + + let build_info = BuildInfo { + rust_toolchain: util::rust_toolchain()?, + cargo_contract_version, + build_mode: *build_mode, + wasm_opt_settings: WasmOptSettings { + optimization_passes: *optimization_passes, + keep_debug_symbols: *keep_debug_symbols, + }, + }; + + let post_fingerprint = Fingerprint::new(crate_metadata)?.ok_or_else(|| { + anyhow::anyhow!( + "Expected '{}' to be generated by build", + crate_metadata.original_code.display() + ) + })?; + + tracing::debug!( + "Fingerprint before build: {:?}, after build: {:?}", + pre_fingerprint, + post_fingerprint + ); + + let dest_code_path = crate_metadata.dest_code.clone(); + + if pre_fingerprint == Some(post_fingerprint) && crate_metadata.dest_code.exists() { + tracing::info!( + "No changes in the original wasm at {}, fingerprint {:?}. \ + Skipping Wasm optimization and metadata generation.", + crate_metadata.original_code.display(), + pre_fingerprint + ); + return Ok((None, build_info, dest_code_path, build_steps)) + } + + maybe_lint( + &mut build_steps, + *build_artifact, + *lint, + crate_metadata, + verbosity, + )?; + + maybe_println!( + verbosity, + " {} {}", + format!("{build_steps}").bold(), + "Post processing code".bright_green().bold() + ); + build_steps.increment_current(); + + // remove build artifacts so we don't have anything stale lingering around + for t in Target::iter() { + fs::remove_file(crate_metadata.dest_code.with_extension(t.dest_extension())).ok(); + } + + let original_size = + fs::metadata(&crate_metadata.original_code)?.len() as f64 / 1000.0; + + match target { + Target::Wasm => { + post_process_wasm( + crate_metadata, + *skip_wasm_validation, + verbosity, + *max_memory_pages, + )?; + let handler = WasmOptHandler::new(*optimization_passes, *keep_debug_symbols)?; + handler.optimize( + &crate_metadata.dest_code, + &crate_metadata.contract_artifact_name, + )?; + } + Target::RiscV => { + fs::copy(&crate_metadata.original_code, &crate_metadata.dest_code)?; + } + } + + let optimized_size = fs::metadata(&dest_code_path)?.len() as f64 / 1000.0; + + let optimization_result = OptimizationResult { + dest_wasm: crate_metadata.dest_code.clone(), + original_size, + optimized_size, + }; + + Ok(( + Some(optimization_result), + build_info, + crate_metadata.dest_code.clone(), + build_steps, + )) +} + +pub fn maybe_lint( + steps: &mut BuildSteps, + build_artifact: BuildArtifacts, + lint: bool, + crate_metadata: &CrateMetadata, + verbosity: &Verbosity, +) -> Result<()> { + let total_steps = build_artifact.steps(); + if lint { + steps.set_total_steps(total_steps + 1); + maybe_println!( + verbosity, + " {} {}", + format!("{steps}").bold(), + "Checking ink! linting rules".bright_green().bold() + ); + steps.increment_current(); + exec_cargo_dylint(crate_metadata, *verbosity)?; + Ok(()) + } else { + steps.set_total_steps(total_steps); + Ok(()) + } } pub fn docker_build(args: ExecuteArgs) -> Result { @@ -993,7 +1024,7 @@ pub fn docker_build(args: ExecuteArgs) -> Result { .unwrap(); let build_image = images.iter().find(|i| { - i.labels.get("io.parity.image.title") == Some(&"parity/ver-build".to_string()) + i.labels.get("io.parity.image.title") == Some(&"ink-build-image".to_string()) }); if build_image.is_none() { return Err(anyhow::anyhow!("No image found")) @@ -1009,8 +1040,6 @@ pub fn docker_build(args: ExecuteArgs) -> Result { let host_folder = crate_metadata .manifest_path .absolute_directory()? - .parent() - .expect("Manifests is located in unreachable directory") .to_str() .expect("Cannot convert Path to string.") .to_owned(); @@ -1021,21 +1050,19 @@ pub fn docker_build(args: ExecuteArgs) -> Result { typ: Some(MountTypeEnum::BIND), ..Default::default() }; + println!("Mount cfg: {:?}", &mount); let host_cfg = Some(HostConfig { mounts: Some(vec![mount]), ..Default::default() }); + let entrypoint = vec!["/bin/bash", "-c"] + .iter() + .map(|s| s.to_string()) + .collect(); + let cmds = vec![ - "cargo", - "contract", - "build", - "--release", - "--generate", - "code-only", - "--output-json", - ">", - "result.json", + "cargo contract build --release --generate code-only --output-json > result.json", ] .iter() .map(|s| s.to_string()) @@ -1043,7 +1070,8 @@ pub fn docker_build(args: ExecuteArgs) -> Result { let config = Config { image: Some(build_image.id.clone()), - entrypoint: Some(cmds), + entrypoint: Some(entrypoint), + cmd: Some(cmds), host_config: host_cfg, attach_stderr: Some(true), ..Default::default() @@ -1061,8 +1089,10 @@ pub fn docker_build(args: ExecuteArgs) -> Result { let mut wait_stream = client.wait_container(&container_id, options); while wait_stream.next().await.is_some() {} + + let file_path = format!("{}/result.json", &host_folder); let result_contents = - std::fs::read_to_string(format!("{}/result.json", &host_folder))?; + std::fs::read_to_string(&file_path)?; let build_result: BuildResult = serde_json::from_str(&result_contents).map_err(|_| { anyhow::anyhow!( @@ -1114,6 +1144,7 @@ pub fn docker_build(args: ExecuteArgs) -> Result { )?; } + std::fs::remove_file(&file_path)?; client.remove_container(&container_id, None).await?; Ok(BuildResult { From 26328c622bce330679cff62b30ef6b145b2e632b Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 6 Jun 2023 15:08:53 +0100 Subject: [PATCH 04/64] successful docker build --- build-image/Dockerfile | 12 ++- crates/build/src/lib.rs | 184 +++++++++++++++++++---------------- crates/build/src/metadata.rs | 45 +++++---- crates/build/src/wasm_opt.rs | 2 - 4 files changed, 133 insertions(+), 110 deletions(-) diff --git a/build-image/Dockerfile b/build-image/Dockerfile index ae07af6c6..a9f7b47bc 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -37,9 +37,13 @@ RUN set -eux; \ # `RUSTC_BOOSTRAP=1` environment variable. We also need to install the # `wasm32-unknown-unknown` target since that's the platform that ink! smart contracts # run on. - rustup target add wasm32-unknown-unknown --toolchain stable && \ - rustup component add rust-src clippy rustfmt --toolchain stable && \ - rustup default stable && \ + + # The 1.69 toolchain is temporarily required to build ink! contracts because of + # https://github.com/paritytech/cargo-contract/issues/1139 \ + rustup toolchain install 1.69 && \ + rustup target add wasm32-unknown-unknown --toolchain 1.69 && \ + rustup component add rust-src clippy rustfmt --toolchain 1.69 && \ + rustup default 1.69 && \ # `cargo-dylint` and `dylint-link` are dependencies needed to run `cargo-contract`. cargo install cargo-dylint dylint-link && \ @@ -68,5 +72,3 @@ RUN set -eux; \ # A contract workdir which should be bind mounted to the container. See README for detailed documentation. WORKDIR /contract -# Default entry point to the container. Can be overwritten by the child's image -ENTRYPOINT [ "cargo", "contract", "build", "--release", "--generate", "code-only" ] diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index e4d911137..fa4032991 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -31,6 +31,7 @@ use bollard::{ }, Docker, }; +use contract_metadata::ContractMetadata; use tokio_stream::StreamExt; use which as _; @@ -106,7 +107,10 @@ use std::{ PathBuf, }, process::Command, - str, + str::{ + self, + FromStr, + }, }; use strum::IntoEnumIterator; @@ -792,7 +796,6 @@ pub fn execute(args: ExecuteArgs) -> Result { build_steps, unstable_flags, build_info, - None, )?; } (opt_result, Some(metadata_result), Some(dest_wasm)) @@ -946,7 +949,6 @@ fn local_build( let optimized_size = fs::metadata(&dest_code_path)?.len() as f64 / 1000.0; let optimization_result = OptimizationResult { - dest_wasm: crate_metadata.dest_code.clone(), original_size, optimized_size, }; @@ -999,22 +1001,9 @@ pub fn docker_build(args: ExecuteArgs) -> Result { .. } = args; tokio::runtime::Runtime::new()?.block_on(async { - let steps = BuildSteps::new(); + let mut build_steps = BuildSteps::new(); let client = Docker::connect_with_socket_defaults()?; - let optimization_passes = match optimization_passes { - Some(opt_passes) => opt_passes, - None => { - let mut manifest = Manifest::new(manifest_path.clone())?; - match manifest.profile_optimization_passes() { - // if no setting is found, neither on the cli nor in the profile, - // then we use the default - None => OptimizationPasses::default(), - Some(opt_passes) => opt_passes, - } - } - }; - let images = client .list_images(Some(ListImagesOptions:: { all: true, @@ -1040,13 +1029,19 @@ pub fn docker_build(args: ExecuteArgs) -> Result { let host_folder = crate_metadata .manifest_path .absolute_directory()? - .to_str() - .expect("Cannot convert Path to string.") + .as_path() .to_owned(); + let file_path = host_folder.join(Path::new("build_result.json")); + let mount = Mount { target: Some(String::from("/contract")), - source: Some(host_folder.clone()), + source: Some( + host_folder + .to_str() + .context("Cannot convert path to string.")? + .to_string(), + ), typ: Some(MountTypeEnum::BIND), ..Default::default() }; @@ -1057,16 +1052,15 @@ pub fn docker_build(args: ExecuteArgs) -> Result { }); let entrypoint = vec!["/bin/bash", "-c"] - .iter() - .map(|s| s.to_string()) - .collect(); + .iter() + .map(|s| s.to_string()) + .collect(); - let cmds = vec![ - "cargo contract build --release --generate code-only --output-json > result.json", - ] - .iter() - .map(|s| s.to_string()) - .collect(); + let cmds = + vec!["cargo contract build --release --output-json > build_result.json"] + .iter() + .map(|s| s.to_string()) + .collect(); let config = Config { image: Some(build_image.id.clone()), @@ -1077,10 +1071,27 @@ pub fn docker_build(args: ExecuteArgs) -> Result { ..Default::default() }; - let container_id = client.create_container(options, config).await?.id; - client - .start_container::(&container_id, None) - .await?; + let container_id = match client + .create_container(options.clone(), config.clone()) + .await + { + Ok(response) => response.id, + Err(_) => { + // container might exist, so we delete the previous one + let _ = client.remove_container("ink-container", None).await; + client.create_container(options, config).await?.id + } + }; + match client.start_container::(&container_id, None).await { + Ok(_) => { + // TODO: message + } + Err(_) => { + // TODO: message + let _ = client.remove_container(&container_id, None).await; + let _ = fs::remove_file(&file_path); + } + } let options = Some(WaitContainerOptions { condition: "not-running", @@ -1088,70 +1099,72 @@ pub fn docker_build(args: ExecuteArgs) -> Result { let mut wait_stream = client.wait_container(&container_id, options); while wait_stream.next().await.is_some() {} + client.remove_container(&container_id, None).await?; + let result_contents = match std::fs::read_to_string(&file_path) { + Ok(content) => { + std::fs::remove_file(&file_path)?; + content + } + Err(e) => { + std::fs::remove_file(&file_path)?; + anyhow::bail!(e); + } + }; - let file_path = format!("{}/result.json", &host_folder); - let result_contents = - std::fs::read_to_string(&file_path)?; - let build_result: BuildResult = - serde_json::from_str(&result_contents).map_err(|_| { + let mut build_result: BuildResult = serde_json::from_str(&result_contents) + .map_err(|_| { anyhow::anyhow!( "Error parsing output from docker build. The build probably failed!" ) })?; - let metadata_result = MetadataArtifacts { - dest_metadata: crate_metadata.metadata_path(), - dest_bundle: crate_metadata.contract_bundle_path(), - }; + let new_path = host_folder.join( + build_result + .target_directory + .as_path() + .strip_prefix("/contract")?, + ); + build_result.target_directory = new_path; + + let new_path = build_result.dest_wasm.as_ref().map(|p| { + host_folder.join( + p.as_path() + .strip_prefix("/contract") + .expect("cannot strip prefix"), + ) + }); + build_result.dest_wasm = new_path; + + build_result.metadata_result.as_mut().map(|mut m| { + m.dest_bundle = host_folder.join( + m.dest_bundle + .as_path() + .strip_prefix("/contract") + .expect("cannot strip prefix"), + ); + m.dest_metadata = host_folder.join( + m.dest_metadata + .as_path() + .strip_prefix("/contract") + .expect("cannot strip prefix"), + ); + m + }); - // skip metadata generation if contract unchanged and all metadata artifacts - // exist. - if build_result.optimization_result.is_some() - || !metadata_result.dest_metadata.exists() - || !metadata_result.dest_bundle.exists() - { - let cargo_contract_version = if let Ok(version) = Version::parse(VERSION) { - version - } else { - anyhow::bail!( - "Unable to parse version number for the currently running \ - `cargo-contract` binary." - ); - }; + if let Some(metadata_artifacts) = &build_result.metadata_result { + let mut metadata = ContractMetadata::load(&metadata_artifacts.dest_bundle)?; + metadata.image = Some(build_image.id.to_string()); - let build_info = BuildInfo { - rust_toolchain: util::rust_toolchain()?, - cargo_contract_version, - build_mode, - wasm_opt_settings: WasmOptSettings { - optimization_passes, - keep_debug_symbols, - }, - }; - // if metadata build fails after a code build it might become stale - metadata::execute( - &crate_metadata, - build_result.dest_wasm.as_ref().unwrap().as_path(), - &metadata_result, - &features, - network, - verbosity, - steps, - &unstable_flags, - build_info, - Some(build_image.id.clone()), + metadata::write_metadata( + metadata_artifacts, + metadata, + &mut build_steps, + &verbosity, )?; } - std::fs::remove_file(&file_path)?; - client.remove_container(&container_id, None).await?; - - Ok(BuildResult { - metadata_result: Some(metadata_result), - output_type, - ..build_result - }) + Ok(build_result) }) } @@ -1312,7 +1325,6 @@ mod unit_tests { }), target_directory: PathBuf::from("/path/to/target"), optimization_result: Some(OptimizationResult { - dest_wasm: PathBuf::from("/path/to/contract.wasm"), original_size: 64.0, optimized_size: 32.0, }), diff --git a/crates/build/src/metadata.rs b/crates/build/src/metadata.rs index 29b8b8f3f..d96940fbd 100644 --- a/crates/build/src/metadata.rs +++ b/crates/build/src/metadata.rs @@ -119,7 +119,6 @@ pub fn execute( mut build_steps: BuildSteps, unstable_options: &UnstableFlags, build_info: BuildInfo, - image: Option, ) -> Result<()> { // build the extended contract project metadata let ExtendedMetadataResult { @@ -161,23 +160,9 @@ pub fn execute( let ink_meta: serde_json::Map = serde_json::from_slice(&output.stdout)?; - let metadata = ContractMetadata::new(source, contract, image, user, ink_meta); - { - let mut metadata = metadata.clone(); - metadata.remove_source_wasm_attribute(); - let contents = serde_json::to_string_pretty(&metadata)?; - fs::write(&metadata_artifacts.dest_metadata, contents)?; - build_steps.increment_current(); - } + let metadata = ContractMetadata::new(source, contract, None, user, ink_meta); - maybe_println!( - verbosity, - " {} {}", - format!("{build_steps}").bold(), - "Generating bundle".bright_green().bold() - ); - let contents = serde_json::to_string(&metadata)?; - fs::write(&metadata_artifacts.dest_bundle, contents)?; + write_metadata(metadata_artifacts, metadata, &mut build_steps, &verbosity)?; Ok(()) }; @@ -200,6 +185,32 @@ pub fn execute( Ok(()) } +pub fn write_metadata( + metadata_artifacts: &MetadataArtifacts, + metadata: ContractMetadata, + build_steps: &mut BuildSteps, + verbosity: &Verbosity, +) -> Result<()> { + { + let mut metadata = metadata.clone(); + metadata.remove_source_wasm_attribute(); + let contents = serde_json::to_string_pretty(&metadata)?; + fs::write(&metadata_artifacts.dest_metadata, contents)?; + build_steps.increment_current(); + } + + maybe_println!( + verbosity, + " {} {}", + format!("{build_steps}").bold(), + "Generating bundle".bright_green().bold() + ); + let contents = serde_json::to_string(&metadata)?; + fs::write(&metadata_artifacts.dest_bundle, contents)?; + + Ok(()) +} + /// Generate the extended contract project metadata fn extended_metadata( crate_metadata: &CrateMetadata, diff --git a/crates/build/src/wasm_opt.rs b/crates/build/src/wasm_opt.rs index de7009eb5..23f738901 100644 --- a/crates/build/src/wasm_opt.rs +++ b/crates/build/src/wasm_opt.rs @@ -164,8 +164,6 @@ impl From for OptimizationOptions { /// Result of the optimization process. #[derive(serde::Serialize, serde::Deserialize)] pub struct OptimizationResult { - /// The path of the optimized Wasm file. - pub dest_wasm: PathBuf, /// The original Wasm size. pub original_size: f64, /// The Wasm size after optimizations have been applied. From 773a2ff11bcf4911cc69de716845b01f6004ec46 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 6 Jun 2023 16:55:15 +0100 Subject: [PATCH 05/64] persist flags in the main execution context --- crates/build/src/lib.rs | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index fa4032991..802f3a839 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -1032,7 +1032,7 @@ pub fn docker_build(args: ExecuteArgs) -> Result { .as_path() .to_owned(); - let file_path = host_folder.join(Path::new("build_result.json")); + let file_path = host_folder.join(Path::new("target/build_result.json")); let mount = Mount { target: Some(String::from("/contract")), @@ -1056,11 +1056,12 @@ pub fn docker_build(args: ExecuteArgs) -> Result { .map(|s| s.to_string()) .collect(); - let cmds = - vec!["cargo contract build --release --output-json > build_result.json"] - .iter() - .map(|s| s.to_string()) - .collect(); + let cmds = vec![ + "cargo contract build --release --output-json > target/build_result.json", + ] + .iter() + .map(|s| s.to_string()) + .collect(); let config = Config { image: Some(build_image.id.clone()), @@ -1086,10 +1087,11 @@ pub fn docker_build(args: ExecuteArgs) -> Result { Ok(_) => { // TODO: message } - Err(_) => { + Err(e) => { // TODO: message let _ = client.remove_container(&container_id, None).await; let _ = fs::remove_file(&file_path); + anyhow::bail!(e) } } @@ -1099,7 +1101,8 @@ pub fn docker_build(args: ExecuteArgs) -> Result { let mut wait_stream = client.wait_container(&container_id, options); while wait_stream.next().await.is_some() {} - client.remove_container(&container_id, None).await?; + + let _ = client.remove_container(&container_id, None).await; let result_contents = match std::fs::read_to_string(&file_path) { Ok(content) => { @@ -1164,7 +1167,11 @@ pub fn docker_build(args: ExecuteArgs) -> Result { )?; } - Ok(build_result) + Ok(BuildResult { + output_type, + verbosity, + ..build_result + }) }) } From 763cc77cefef867a45a6a0412426b0b1d560f6fe Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Wed, 7 Jun 2023 12:00:57 +0100 Subject: [PATCH 06/64] docker readme + respect host flags --- build-image/Dockerfile | 6 +- build-image/README.md | 56 ++++++++ build-image/build.sh | 2 +- crates/build/src/docker.rs | 264 +++++++++++++++++++++++++++++++++++++ crates/build/src/lib.rs | 212 +---------------------------- 5 files changed, 329 insertions(+), 211 deletions(-) create mode 100644 crates/build/src/docker.rs diff --git a/build-image/Dockerfile b/build-image/Dockerfile index a9f7b47bc..89cdfb653 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -6,7 +6,7 @@ FROM ${REGISTRY_PATH}/base-ci-linux:latest # metadata LABEL io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${REGISTRY_PATH}/ink-build-image" \ + io.parity.image.title="${REGISTRY_PATH}/contracts-verified" \ io.parity.image.documentation="https://github.com/paritytech/scripts/blob/${VCS_REF}/\ dockerfiles/ink-ci-linux/README.md" \ io.parity.image.description="Inherits from docker.io/paritytech/base-ci-linux. \ @@ -21,6 +21,8 @@ WORKDIR /builds RUN set -eux; \ apt-get -y update && \ + apt-get install -y --no-install-recommends zlib1g-dev npm wabt && \ + npm install --ignore-scripts -g yarn && \ # `binaryen` is needed by `cargo-contract` for optimizing Wasm files. # We fetch the latest release which contains a Linux binary. @@ -72,3 +74,5 @@ RUN set -eux; \ # A contract workdir which should be bind mounted to the container. See README for detailed documentation. WORKDIR /contract +# A default entry point for manual execution +ENTRYPOINT [ "cargo", "contract", "build", "--release" ] diff --git a/build-image/README.md b/build-image/README.md index e74e5db25..2e976a301 100644 --- a/build-image/README.md +++ b/build-image/README.md @@ -1,2 +1,58 @@ # Verifiable build using Docker +Docker image based on our base CI image ``. + +Used for reproducible builds in `cargo contract --verifiable` + +## Dependencies and Tools + +- `llvm-dev` +- `zlib1g-dev` +- `npm` +- `yarn` +- `wabt` +- `binaryen` + +**Inherited from ``** + +- `libssl-dev` +- `clang-10` +- `lld` +- `libclang-dev` +- `make` +- `cmake` +- `git` +- `pkg-config` +- `curl` +- `time` +- `rhash` +- `ca-certificates` +- `jq` + +**Rust versions:** + +Currently, the 1.69 toolchain is temporarily required to build ink! contracts because of https://github.com/paritytech/cargo-contract/issues/1139 + +**Rust tools & toolchains:** + +We use stable releases from crates.io + +- `cargo-contract` +- `cargo-dylint` and `dylint-link` +- `pwasm-utils-cli` +- `solang` +- `wasm32-unknown-unknown`: The toolchain to compile Rust codebases for Wasm. + +[Click here](https://hub.docker.com/repository/docker/paritytech/contracts-ci-linux) for the registry. + +## Usage + +The default entry point is `ENTRYPOINT [ "cargo", "contract", "build", "--release" ]` +and work directory is set to `/contract`. You need to mount the folder with the contract to the container. + +```bash +docker run -d \ + --name ink-container \ + --mount type=bind,source="$(pwd)",target="/contract" \ + paritytech/contracts-verified +``` diff --git a/build-image/build.sh b/build-image/build.sh index 4d3c241c8..80aecab82 100644 --- a/build-image/build.sh +++ b/build-image/build.sh @@ -1,4 +1,4 @@ docker run -d \ - --name flipper \ + --name ink-container \ --mount type=bind,source="$(pwd)",target="/contract" \ parity/ver-build diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs new file mode 100644 index 000000000..5b54f9af9 --- /dev/null +++ b/crates/build/src/docker.rs @@ -0,0 +1,264 @@ +use std::path::Path; + +use anyhow::{ + Context, + Result, +}; +use bollard::{ + container::{ + Config, + CreateContainerOptions, + WaitContainerOptions, + }, + image::ListImagesOptions, + service::{ + HostConfig, + Mount, + MountTypeEnum, + }, + Docker, +}; +use contract_metadata::ContractMetadata; +use tokio_stream::StreamExt; + +use crate::{ + BuildArtifacts, + BuildMode, + BuildResult, + BuildSteps, + CrateMetadata, + ExecuteArgs, + Network, + OptimizationPasses, + DEFAULT_MAX_MEMORY_PAGES, +}; + +pub fn docker_build(args: ExecuteArgs) -> Result { + let ExecuteArgs { + manifest_path, + verbosity, + features, + build_mode, + network, + unstable_flags, + optimization_passes, + keep_debug_symbols, + output_type, + target, + max_memory_pages, + build_artifact, + .. + } = args; + tokio::runtime::Runtime::new()?.block_on(async { + let mut build_steps = BuildSteps::new(); + let client = Docker::connect_with_socket_defaults()?; + + let images = client + .list_images(Some(ListImagesOptions:: { + all: true, + ..Default::default() + })) + .await + .unwrap(); + + let build_image = images.iter().find(|i| { + i.labels.get("io.parity.image.title") == Some(&"/ink-build-image".to_string()) + }); + if build_image.is_none() { + return Err(anyhow::anyhow!("No image found")) + } + let build_image = build_image.unwrap(); + + let options = Some(CreateContainerOptions { + name: "ink-container", + platform: Some("linux/amd64"), + }); + + let crate_metadata = CrateMetadata::collect(&manifest_path, target)?; + let host_folder = crate_metadata + .manifest_path + .absolute_directory()? + .as_path() + .to_owned(); + + let file_path = host_folder.join(Path::new("target/build_result.json")); + + let mount = Mount { + target: Some(String::from("/contract")), + source: Some( + host_folder + .to_str() + .context("Cannot convert path to string.")? + .to_string(), + ), + typ: Some(MountTypeEnum::BIND), + ..Default::default() + }; + println!("Mount cfg: {:?}", &mount); + let host_cfg = Some(HostConfig { + mounts: Some(vec![mount]), + ..Default::default() + }); + + let entrypoint = vec!["/bin/bash", "-c"] + .iter() + .map(|s| s.to_string()) + .collect(); + + let mut args: Vec = Vec::new(); + + features.append_to_args(&mut args); + if keep_debug_symbols { + args.push("--keep-debug-symbols".to_owned()); + } + + if let Some(passes) = optimization_passes { + if passes != OptimizationPasses::default() { + args.push(format!("--optimization-passes {}", passes)); + } + } + + if network == Network::Offline { + args.push("--offline".to_owned()); + } + + if build_mode == BuildMode::Release { + args.push("--release".to_owned()); + } + + if unstable_flags.original_manifest { + args.push("-Z original-manifest".to_owned()); + } + + let s = match build_artifact { + BuildArtifacts::CodeOnly => "--generate code-only".to_owned(), + BuildArtifacts::All => String::new(), + BuildArtifacts::CheckOnly => { + anyhow::bail!("--generate check-only is invalid flag for this command!"); + } + }; + + args.push(s); + + if max_memory_pages != DEFAULT_MAX_MEMORY_PAGES { + args.push(format!("--max-memory-pages {}", max_memory_pages)); + } + + let joined_args = args.join(" "); + + let cmds = vec![format!( + "cargo contract build {} --output-json > target/build_result.json", + joined_args + )]; + + let config = Config { + image: Some(build_image.id.clone()), + entrypoint: Some(entrypoint), + cmd: Some(cmds), + host_config: host_cfg, + attach_stderr: Some(true), + ..Default::default() + }; + + let container_id = match client + .create_container(options.clone(), config.clone()) + .await + { + Ok(response) => response.id, + Err(_) => { + // container might exist, so we delete the previous one + let _ = client.remove_container("ink-container", None).await; + client.create_container(options, config).await?.id + } + }; + match client.start_container::(&container_id, None).await { + Ok(_) => { + // TODO: message + } + Err(e) => { + // TODO: message + let _ = client.remove_container(&container_id, None).await; + let _ = std::fs::remove_file(&file_path); + anyhow::bail!(e) + } + } + + let options = Some(WaitContainerOptions { + condition: "not-running", + }); + + let mut wait_stream = client.wait_container(&container_id, options); + while wait_stream.next().await.is_some() {} + + let _ = client.remove_container(&container_id, None).await; + + let result_contents = match std::fs::read_to_string(&file_path) { + Ok(content) => { + std::fs::remove_file(&file_path)?; + content + } + Err(e) => { + std::fs::remove_file(&file_path)?; + anyhow::bail!(e); + } + }; + + let mut build_result: BuildResult = serde_json::from_str(&result_contents) + .map_err(|_| { + anyhow::anyhow!( + "Error parsing output from docker build. The build probably failed!" + ) + })?; + + let new_path = host_folder.join( + build_result + .target_directory + .as_path() + .strip_prefix("/contract")?, + ); + build_result.target_directory = new_path; + + let new_path = build_result.dest_wasm.as_ref().map(|p| { + host_folder.join( + p.as_path() + .strip_prefix("/contract") + .expect("cannot strip prefix"), + ) + }); + build_result.dest_wasm = new_path; + + build_result.metadata_result.as_mut().map(|mut m| { + m.dest_bundle = host_folder.join( + m.dest_bundle + .as_path() + .strip_prefix("/contract") + .expect("cannot strip prefix"), + ); + m.dest_metadata = host_folder.join( + m.dest_metadata + .as_path() + .strip_prefix("/contract") + .expect("cannot strip prefix"), + ); + m + }); + + if let Some(metadata_artifacts) = &build_result.metadata_result { + let mut metadata = ContractMetadata::load(&metadata_artifacts.dest_bundle)?; + metadata.image = Some(build_image.id.to_string()); + + crate::metadata::write_metadata( + metadata_artifacts, + metadata, + &mut build_steps, + &verbosity, + )?; + } + + Ok(BuildResult { + output_type, + verbosity, + ..build_result + }) + }) +} diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index 802f3a839..d6e3f7497 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -17,26 +17,11 @@ #![doc = include_str!("../README.md")] #![deny(unused_crate_dependencies)] -use bollard::{ - container::{ - Config, - CreateContainerOptions, - WaitContainerOptions, - }, - image::ListImagesOptions, - service::{ - HostConfig, - Mount, - MountTypeEnum, - }, - Docker, -}; -use contract_metadata::ContractMetadata; -use tokio_stream::StreamExt; use which as _; mod args; mod crate_metadata; +mod docker; pub mod metadata; mod new; #[cfg(test)] @@ -84,6 +69,7 @@ pub use self::{ }; use crate::wasm_opt::WasmOptHandler; +pub use docker::docker_build; use anyhow::{ Context, @@ -107,10 +93,7 @@ use std::{ PathBuf, }, process::Command, - str::{ - self, - FromStr, - }, + str, }; use strum::IntoEnumIterator; @@ -986,195 +969,6 @@ pub fn maybe_lint( } } -pub fn docker_build(args: ExecuteArgs) -> Result { - let ExecuteArgs { - manifest_path, - verbosity, - features, - build_mode, - network, - unstable_flags, - optimization_passes, - keep_debug_symbols, - output_type, - target, - .. - } = args; - tokio::runtime::Runtime::new()?.block_on(async { - let mut build_steps = BuildSteps::new(); - let client = Docker::connect_with_socket_defaults()?; - - let images = client - .list_images(Some(ListImagesOptions:: { - all: true, - ..Default::default() - })) - .await - .unwrap(); - - let build_image = images.iter().find(|i| { - i.labels.get("io.parity.image.title") == Some(&"ink-build-image".to_string()) - }); - if build_image.is_none() { - return Err(anyhow::anyhow!("No image found")) - } - let build_image = build_image.unwrap(); - - let options = Some(CreateContainerOptions { - name: "ink-container", - platform: Some("linux/amd64"), - }); - - let crate_metadata = CrateMetadata::collect(&manifest_path, target)?; - let host_folder = crate_metadata - .manifest_path - .absolute_directory()? - .as_path() - .to_owned(); - - let file_path = host_folder.join(Path::new("target/build_result.json")); - - let mount = Mount { - target: Some(String::from("/contract")), - source: Some( - host_folder - .to_str() - .context("Cannot convert path to string.")? - .to_string(), - ), - typ: Some(MountTypeEnum::BIND), - ..Default::default() - }; - println!("Mount cfg: {:?}", &mount); - let host_cfg = Some(HostConfig { - mounts: Some(vec![mount]), - ..Default::default() - }); - - let entrypoint = vec!["/bin/bash", "-c"] - .iter() - .map(|s| s.to_string()) - .collect(); - - let cmds = vec![ - "cargo contract build --release --output-json > target/build_result.json", - ] - .iter() - .map(|s| s.to_string()) - .collect(); - - let config = Config { - image: Some(build_image.id.clone()), - entrypoint: Some(entrypoint), - cmd: Some(cmds), - host_config: host_cfg, - attach_stderr: Some(true), - ..Default::default() - }; - - let container_id = match client - .create_container(options.clone(), config.clone()) - .await - { - Ok(response) => response.id, - Err(_) => { - // container might exist, so we delete the previous one - let _ = client.remove_container("ink-container", None).await; - client.create_container(options, config).await?.id - } - }; - match client.start_container::(&container_id, None).await { - Ok(_) => { - // TODO: message - } - Err(e) => { - // TODO: message - let _ = client.remove_container(&container_id, None).await; - let _ = fs::remove_file(&file_path); - anyhow::bail!(e) - } - } - - let options = Some(WaitContainerOptions { - condition: "not-running", - }); - - let mut wait_stream = client.wait_container(&container_id, options); - while wait_stream.next().await.is_some() {} - - let _ = client.remove_container(&container_id, None).await; - - let result_contents = match std::fs::read_to_string(&file_path) { - Ok(content) => { - std::fs::remove_file(&file_path)?; - content - } - Err(e) => { - std::fs::remove_file(&file_path)?; - anyhow::bail!(e); - } - }; - - let mut build_result: BuildResult = serde_json::from_str(&result_contents) - .map_err(|_| { - anyhow::anyhow!( - "Error parsing output from docker build. The build probably failed!" - ) - })?; - - let new_path = host_folder.join( - build_result - .target_directory - .as_path() - .strip_prefix("/contract")?, - ); - build_result.target_directory = new_path; - - let new_path = build_result.dest_wasm.as_ref().map(|p| { - host_folder.join( - p.as_path() - .strip_prefix("/contract") - .expect("cannot strip prefix"), - ) - }); - build_result.dest_wasm = new_path; - - build_result.metadata_result.as_mut().map(|mut m| { - m.dest_bundle = host_folder.join( - m.dest_bundle - .as_path() - .strip_prefix("/contract") - .expect("cannot strip prefix"), - ); - m.dest_metadata = host_folder.join( - m.dest_metadata - .as_path() - .strip_prefix("/contract") - .expect("cannot strip prefix"), - ); - m - }); - - if let Some(metadata_artifacts) = &build_result.metadata_result { - let mut metadata = ContractMetadata::load(&metadata_artifacts.dest_bundle)?; - metadata.image = Some(build_image.id.to_string()); - - metadata::write_metadata( - metadata_artifacts, - metadata, - &mut build_steps, - &verbosity, - )?; - } - - Ok(BuildResult { - output_type, - verbosity, - ..build_result - }) - }) -} - /// Unique fingerprint for a file to detect whether it has changed. #[derive(Debug, Eq, PartialEq)] struct Fingerprint { From 300b725eea3de311293f563c991d4351750b0caf Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Wed, 7 Jun 2023 12:15:13 +0100 Subject: [PATCH 07/64] add build steps --- crates/build/src/docker.rs | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index 5b54f9af9..1c247a69a 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -22,6 +22,7 @@ use contract_metadata::ContractMetadata; use tokio_stream::StreamExt; use crate::{ + maybe_println, BuildArtifacts, BuildMode, BuildResult, @@ -33,6 +34,8 @@ use crate::{ DEFAULT_MAX_MEMORY_PAGES, }; +use colored::Colorize; + pub fn docker_build(args: ExecuteArgs) -> Result { let ExecuteArgs { manifest_path, @@ -51,6 +54,19 @@ pub fn docker_build(args: ExecuteArgs) -> Result { } = args; tokio::runtime::Runtime::new()?.block_on(async { let mut build_steps = BuildSteps::new(); + + build_steps.set_total_steps(4); + if build_artifact == BuildArtifacts::CodeOnly { + build_steps.set_total_steps(3); + } + + maybe_println!( + verbosity, + " {} {}", + format!("{build_steps}").bold(), + "Executing verifiable build".bright_green().bold() + ); + let client = Docker::connect_with_socket_defaults()?; let images = client @@ -183,6 +199,18 @@ pub fn docker_build(args: ExecuteArgs) -> Result { } } + build_steps.increment_current(); + + maybe_println!( + verbosity, + " {} {}\n{}", + format!("{build_steps}").bold(), + "Started the build inside the container" + .bright_green() + .bold(), + "This might take a while. Check the docker engine for the logs." + ); + let options = Some(WaitContainerOptions { condition: "not-running", }); @@ -192,6 +220,16 @@ pub fn docker_build(args: ExecuteArgs) -> Result { let _ = client.remove_container(&container_id, None).await; + build_steps.increment_current(); + maybe_println!( + verbosity, + " {} {}", + format!("{build_steps}").bold(), + "Docker container has finished the build." + .bright_green() + .bold(), + ); + let result_contents = match std::fs::read_to_string(&file_path) { Ok(content) => { std::fs::remove_file(&file_path)?; From 7dca32b00806c066a8f5a16d5795f61942e30af9 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Thu, 8 Jun 2023 17:21:20 +0100 Subject: [PATCH 08/64] better image, custom image arg, container reuse --- build-image/Dockerfile | 113 ++++++-------- build-image/README.md | 48 +++--- build-image/build.sh | 4 - crates/build/src/docker.rs | 144 ++++++++++++------ crates/build/src/lib.rs | 17 ++- crates/cargo-contract/src/cmd/build.rs | 22 ++- .../cargo-contract/src/cmd/extrinsics/call.rs | 107 +++++++------ .../src/cmd/extrinsics/instantiate.rs | 75 ++++----- .../cargo-contract/src/cmd/extrinsics/mod.rs | 5 +- .../src/cmd/extrinsics/remove.rs | 57 +++---- .../src/cmd/extrinsics/upload.rs | 91 +++++------ crates/cargo-contract/src/cmd/info.rs | 63 ++++---- 12 files changed, 405 insertions(+), 341 deletions(-) delete mode 100644 build-image/build.sh diff --git a/build-image/Dockerfile b/build-image/Dockerfile index 89cdfb653..5dfb708c1 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -1,78 +1,63 @@ -ARG VCS_REF=master -ARG REGISTRY_PATH=docker.io/paritytech -ARG BUILD_DATE - -FROM ${REGISTRY_PATH}/base-ci-linux:latest +FROM docker.io/bitnami/minideb:bullseye-amd64 as slimmed-rust # metadata LABEL io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${REGISTRY_PATH}/contracts-verified" \ - io.parity.image.documentation="https://github.com/paritytech/scripts/blob/${VCS_REF}/\ + io.parity.image.title="contracts-verifiable" \ + io.parity.image.documentation="https://github.com/paritytech/scripts/blob/master/\ dockerfiles/ink-ci-linux/README.md" \ - io.parity.image.description="Inherits from docker.io/paritytech/base-ci-linux. \ + io.parity.image.description="Inherits from docker.io/bitnami/minideb:bullseye. \ rust nightly, clippy, rustfmt, miri, rust-src, rustc-dev, grcov, rust-covfix, \ llvm-tools-preview, cargo-contract, xargo, binaryen, parallel, codecov, ink, solang" \ - io.parity.image.revision="${VCS_REF}" \ - io.parity.image.documentation="https://github.com/paritytech/cargo-contract/blob/${VCS_REF}/\ - build-image/README.md" \ - io.parity.image.created="${BUILD_DATE}" - -WORKDIR /builds - -RUN set -eux; \ - apt-get -y update && \ - apt-get install -y --no-install-recommends zlib1g-dev npm wabt && \ - npm install --ignore-scripts -g yarn && \ - - # `binaryen` is needed by `cargo-contract` for optimizing Wasm files. - # We fetch the latest release which contains a Linux binary. - curl -L $(curl --silent https://api.github.com/repos/WebAssembly/binaryen/releases \ - | jq -r '.[0].assets | [.[] | .browser_download_url] | map(select(match("x86_64-linux\\.tar\\.gz$"))) | .[0]' \ - ) | tar -xz -C /usr/local/bin/ --wildcards --strip-components=2 'binaryen-*/bin/wasm-opt' && \ - - # Install LLVM for solang - curl -L https://github.com/hyperledger/solang-llvm/releases/download/llvm15-1/llvm15.0-linux-x86-64.tar.xz > llvm15.0-linux-x86-64.tar.xz && \ - tar Jxf llvm15.0-linux-x86-64.tar.xz && \ - export PATH=$(pwd)/llvm15.0/bin:$PATH && \ - - # The stable toolchain is used to build ink! itself through the use of the - # `RUSTC_BOOSTRAP=1` environment variable. We also need to install the - # `wasm32-unknown-unknown` target since that's the platform that ink! smart contracts - # run on. + io.parity.image.documentation="https://github.com/paritytech/cargo-contract/blob/master/\ + build-image/README.md" +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH \ # The 1.69 toolchain is temporarily required to build ink! contracts because of - # https://github.com/paritytech/cargo-contract/issues/1139 \ - rustup toolchain install 1.69 && \ - rustup target add wasm32-unknown-unknown --toolchain 1.69 && \ - rustup component add rust-src clippy rustfmt --toolchain 1.69 && \ - rustup default 1.69 && \ - - # `cargo-dylint` and `dylint-link` are dependencies needed to run `cargo-contract`. - cargo install cargo-dylint dylint-link && \ - - # Install the latest stable release of `cargo-contract` - cargo install cargo-contract && \ - - # Install the latest `solang` - cargo install solang && \ - - # Versions - rustup show && \ - cargo --version && \ - cargo-contract --version && \ - wasm-opt --version && \ - - # Clean up and remove compilation artifacts that a cargo install creates (>250M). - rm -rf "${CARGO_HOME}/registry" "${CARGO_HOME}/git" /root/.cache/sccache && \ - - # apt clean up - apt-get remove -y gnupg && \ + # https://github.com/paritytech/cargo-contract/issues/1139 + RUST_VERSION=1.69 + +# Minimal Rust dependencies. +RUN set -eux \ + && apt-get update && apt-get -y install wget \ + && url="https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init" \ + && wget "$url" \ + && chmod +x rustup-init \ + && ./rustup-init -y --no-modify-path --profile minimal --component rust-src rustfmt --default-toolchain $RUST_VERSION \ + && rm rustup-init \ + && chmod -R a+w $RUSTUP_HOME $CARGO_HOME \ + && rustup --version \ + && cargo --version \ + && rustc --version \ + && apt-get remove -y --auto-remove wget \ + && apt-get -y install gcc \ + && rm -rf /var/lib/apt/lists/* + +FROM slimmed-rust as cc-builder + +# This is important, see https://github.com/rust-lang/docker-rust/issues/85 +ENV RUSTFLAGS="-C target-feature=-crt-static" + +RUN apt-get -y update && apt-get -y install gcc g++ +# Install cargo contract +RUN cargo install cargo-contract + +# Cleanup after `cargo install` +RUN rm -rf ${CARGO_HOME}/"registry" ${CARGO_HOME}/"git" /root/.cache/sccache + +# apt clean up +RUN apt-get remove -y gnupg && \ apt-get autoremove -y && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -# A contract workdir which should be bind mounted to the container. See README for detailed documentation. + +FROM slimmed-rust as ink-dev + +COPY --from=cc-builder /usr/local/cargo/bin/cargo-contract /usr/local/bin/cargo-contract + WORKDIR /contract -# A default entry point for manual execution -ENTRYPOINT [ "cargo", "contract", "build", "--release" ] +# default entry point +ENTRYPOINT ["cargo", "contract", "build", "--release"] diff --git a/build-image/README.md b/build-image/README.md index 2e976a301..2e7332796 100644 --- a/build-image/README.md +++ b/build-image/README.md @@ -1,33 +1,8 @@ -# Verifiable build using Docker +# Verifiable builds using Docker Docker image based on our base CI image ``. -Used for reproducible builds in `cargo contract --verifiable` - -## Dependencies and Tools - -- `llvm-dev` -- `zlib1g-dev` -- `npm` -- `yarn` -- `wabt` -- `binaryen` - -**Inherited from ``** - -- `libssl-dev` -- `clang-10` -- `lld` -- `libclang-dev` -- `make` -- `cmake` -- `git` -- `pkg-config` -- `curl` -- `time` -- `rhash` -- `ca-certificates` -- `jq` +Used for reproducible builds in `cargo contract build --verifiable` **Rust versions:** @@ -38,12 +13,9 @@ Currently, the 1.69 toolchain is temporarily required to build ink! contracts be We use stable releases from crates.io - `cargo-contract` -- `cargo-dylint` and `dylint-link` -- `pwasm-utils-cli` -- `solang` - `wasm32-unknown-unknown`: The toolchain to compile Rust codebases for Wasm. -[Click here](https://hub.docker.com/repository/docker/paritytech/contracts-ci-linux) for the registry. +[Click here](https://hub.docker.com/repository/docker/paritytech/contracts-verifiable) for the registry. ## Usage @@ -56,3 +28,17 @@ docker run -d \ --mount type=bind,source="$(pwd)",target="/contract" \ paritytech/contracts-verified ``` + +If you have multi-contract project: +``` +my-app/ +├─ ink-project-a/ +│ ├─ Cargo.toml +│ ├─ lib.rs +├─ ink-project-b/ +│ ├─ Cargo.toml +│ ├─ lib.rs +├─ rust-toolchain +``` +Make sure you mount `my-app` directory and then call +`cargo contract build --verifiable --release --manifest-path ink-project-a/Cargo.toml` diff --git a/build-image/build.sh b/build-image/build.sh deleted file mode 100644 index 80aecab82..000000000 --- a/build-image/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -docker run -d \ - --name ink-container \ - --mount type=bind,source="$(pwd)",target="/contract" \ - parity/ver-build diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index 1c247a69a..9c33b26ce 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -1,4 +1,14 @@ -use std::path::Path; +use std::{ + collections::{ + hash_map::DefaultHasher, + HashMap, + }, + hash::{ + Hash, + Hasher, + }, + path::Path, +}; use anyhow::{ Context, @@ -8,6 +18,8 @@ use bollard::{ container::{ Config, CreateContainerOptions, + ListContainersOptions, + StopContainerOptions, WaitContainerOptions, }, image::ListImagesOptions, @@ -24,7 +36,6 @@ use tokio_stream::StreamExt; use crate::{ maybe_println, BuildArtifacts, - BuildMode, BuildResult, BuildSteps, CrateMetadata, @@ -36,12 +47,19 @@ use crate::{ use colored::Colorize; +#[derive(Clone, Debug, Default)] +pub enum ImageVariant { + #[default] + Default, + Custom(String), +} + +/// Launched the docker container to execute verifiable build pub fn docker_build(args: ExecuteArgs) -> Result { let ExecuteArgs { manifest_path, verbosity, features, - build_mode, network, unstable_flags, optimization_passes, @@ -55,9 +73,9 @@ pub fn docker_build(args: ExecuteArgs) -> Result { tokio::runtime::Runtime::new()?.block_on(async { let mut build_steps = BuildSteps::new(); - build_steps.set_total_steps(4); + build_steps.set_total_steps(5); if build_artifact == BuildArtifacts::CodeOnly { - build_steps.set_total_steps(3); + build_steps.set_total_steps(4); } maybe_println!( @@ -69,27 +87,22 @@ pub fn docker_build(args: ExecuteArgs) -> Result { let client = Docker::connect_with_socket_defaults()?; + // TODO: replace with image pulling, once the image is pushed to registry let images = client .list_images(Some(ListImagesOptions:: { all: true, ..Default::default() })) - .await - .unwrap(); - + .await?; let build_image = images.iter().find(|i| { - i.labels.get("io.parity.image.title") == Some(&"/ink-build-image".to_string()) + i.labels.get("io.parity.image.title") + == Some(&"contracts-verifiable".to_string()) }); if build_image.is_none() { return Err(anyhow::anyhow!("No image found")) } let build_image = build_image.unwrap(); - let options = Some(CreateContainerOptions { - name: "ink-container", - platform: Some("linux/amd64"), - }); - let crate_metadata = CrateMetadata::collect(&manifest_path, target)?; let host_folder = crate_metadata .manifest_path @@ -110,18 +123,14 @@ pub fn docker_build(args: ExecuteArgs) -> Result { typ: Some(MountTypeEnum::BIND), ..Default::default() }; - println!("Mount cfg: {:?}", &mount); let host_cfg = Some(HostConfig { mounts: Some(vec![mount]), ..Default::default() }); - let entrypoint = vec!["/bin/bash", "-c"] - .iter() - .map(|s| s.to_string()) - .collect(); + let mut entrypoint = vec!["/bin/bash".to_string(), "-c".to_string()]; - let mut args: Vec = Vec::new(); + let mut args: Vec = vec!["--release".to_string()]; features.append_to_args(&mut args); if keep_debug_symbols { @@ -138,10 +147,6 @@ pub fn docker_build(args: ExecuteArgs) -> Result { args.push("--offline".to_owned()); } - if build_mode == BuildMode::Release { - args.push("--release".to_owned()); - } - if unstable_flags.original_manifest { args.push("-Z original-manifest".to_owned()); } @@ -162,45 +167,82 @@ pub fn docker_build(args: ExecuteArgs) -> Result { let joined_args = args.join(" "); - let cmds = vec![format!( - "cargo contract build {} --output-json > target/build_result.json", + let mut cmds = vec![format!( + "mkdir -p target && cargo contract build {} --output-json > target/build_result.json", joined_args )]; + entrypoint.append(&mut cmds); + + // in order to optimise the container usage + // we are hashing the inputted command + // in order to reuse containers for different permutations of arguments + let mut s = DefaultHasher::new(); + entrypoint.hash(&mut s); + let digest = s.finish(); + //taking the first 5 digits to be a unique identifier + let digest_code: String = digest.to_string().chars().take(5).collect(); + + let container_name = + format!("ink-verified-{}-{}", crate_metadata.contract_artifact_name, digest_code); + + let mut filters = HashMap::new(); + filters.insert("name".to_string(), vec![container_name.clone()]); + + let containers = client + .list_containers(Some(ListContainersOptions:: { + all: true, + filters, + ..Default::default() + })) + .await?; + + let mut labels = HashMap::new(); + labels.insert("cmd_digest".to_string(), digest.to_string()); let config = Config { image: Some(build_image.id.clone()), entrypoint: Some(entrypoint), - cmd: Some(cmds), + cmd: None, + labels: Some(labels), host_config: host_cfg, attach_stderr: Some(true), + tty: Some(true), ..Default::default() }; + let options = Some(CreateContainerOptions { + name: container_name.as_str(), + platform: Some("linux/amd64"), + }); - let container_id = match client - .create_container(options.clone(), config.clone()) - .await - { - Ok(response) => response.id, - Err(_) => { - // container might exist, so we delete the previous one - let _ = client.remove_container("ink-container", None).await; - client.create_container(options, config).await?.id + let container_id: String = if containers.is_empty() { + client + .create_container(options.clone(), config.clone()) + .await? + .id + } else { + let c = containers + .first() + .context("Error finding existing container")? + .clone(); + if c.labels + .context("image does not have labels")? + .get("cmd_digest") + == Some(&digest.to_string()) + { + c.id.context("Container does not have an id")? + } else { + client + .create_container(options.clone(), config.clone()) + .await? + .id } }; - match client.start_container::(&container_id, None).await { - Ok(_) => { - // TODO: message - } - Err(e) => { - // TODO: message - let _ = client.remove_container(&container_id, None).await; - let _ = std::fs::remove_file(&file_path); - anyhow::bail!(e) - } - } - build_steps.increment_current(); + client + .start_container::(&container_id, None) + .await?; + build_steps.increment_current(); maybe_println!( verbosity, " {} {}\n{}", @@ -208,7 +250,7 @@ pub fn docker_build(args: ExecuteArgs) -> Result { "Started the build inside the container" .bright_green() .bold(), - "This might take a while. Check the docker engine for the logs." + "This might take a while. Check container logs for more details." ); let options = Some(WaitContainerOptions { @@ -218,7 +260,9 @@ pub fn docker_build(args: ExecuteArgs) -> Result { let mut wait_stream = client.wait_container(&container_id, options); while wait_stream.next().await.is_some() {} - let _ = client.remove_container(&container_id, None).await; + client + .stop_container(&container_id, Some(StopContainerOptions { t: 20 })) + .await?; build_steps.increment_current(); maybe_println!( diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index d6e3f7497..3ce69799e 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -69,7 +69,10 @@ pub use self::{ }; use crate::wasm_opt::WasmOptHandler; -pub use docker::docker_build; +pub use docker::{ + docker_build, + ImageVariant, +}; use anyhow::{ Context, @@ -121,6 +124,7 @@ pub struct ExecuteArgs { pub skip_wasm_validation: bool, pub target: Target, pub max_memory_pages: u32, + pub image: Option, } impl Default for ExecuteArgs { @@ -140,6 +144,7 @@ impl Default for ExecuteArgs { skip_wasm_validation: Default::default(), target: Default::default(), max_memory_pages: DEFAULT_MAX_MEMORY_PAGES, + image: Default::default(), } } } @@ -679,9 +684,15 @@ pub fn execute(args: ExecuteArgs) -> Result { lint, output_type, target, + image, .. } = &args; + // if image exists, then --verifiable was called and we need to build inside docker. + if image.is_some() { + return docker_build(args) + } + // The CLI flag `optimization-passes` overwrites optimization passes which are // potentially defined in the `Cargo.toml` profile. let optimization_passes = match optimization_passes { @@ -1109,13 +1120,13 @@ mod unit_tests { }, "target_directory": "/path/to/target", "optimization_result": { - "dest_wasm": "/path/to/contract.wasm", "original_size": 64.0, "optimized_size": 32.0 }, "build_mode": "Debug", "build_artifact": "All", - "verbosity": "Quiet" + "verbosity": "Quiet", + "image": null }"#; let build_result = BuildResult { diff --git a/crates/cargo-contract/src/cmd/build.rs b/crates/cargo-contract/src/cmd/build.rs index abcf4fb2b..de90cc83c 100644 --- a/crates/cargo-contract/src/cmd/build.rs +++ b/crates/cargo-contract/src/cmd/build.rs @@ -21,6 +21,7 @@ use contract_build::{ BuildResult, ExecuteArgs, Features, + ImageVariant, ManifestPath, Network, OptimizationPasses, @@ -125,6 +126,9 @@ pub struct BuildCommand { /// Requires docker daemon running. #[clap(long, short = 'V', default_value_t = false)] verifiable: bool, + /// Specify a custom image for the verifiable build + #[clap(long, default_value = None)] + image: Option, } impl BuildCommand { @@ -149,6 +153,15 @@ impl BuildCommand { false => OutputType::HumanReadable, }; + let image = if self.verifiable { + match &self.image { + Some(i) => Some(ImageVariant::Custom(i.clone())), + None => Some(ImageVariant::Default), + } + } else { + None + }; + // We want to ensure that the only thing in `STDOUT` is our JSON formatted string. if matches!(output_type, OutputType::Json) { verbosity = Verbosity::Quiet; @@ -169,13 +182,9 @@ impl BuildCommand { skip_wasm_validation: self.skip_wasm_validation, target: self.target, max_memory_pages: self.max_memory_pages, + image, }; - - if self.verifiable { - contract_build::docker_build(args) - } else { - contract_build::execute(args) - } + contract_build::execute(args) } } @@ -217,6 +226,7 @@ impl CheckCommand { skip_wasm_validation: false, target: self.target, max_memory_pages: 0, + image: None, }; contract_build::execute(args) diff --git a/crates/cargo-contract/src/cmd/extrinsics/call.rs b/crates/cargo-contract/src/cmd/extrinsics/call.rs index c955a38bb..51a336fc9 100644 --- a/crates/cargo-contract/src/cmd/extrinsics/call.rs +++ b/crates/cargo-contract/src/cmd/extrinsics/call.rs @@ -35,6 +35,7 @@ use crate::{ extrinsics::{ display_contract_exec_result_debug, display_dry_run_result_warning, + error::GenericError, events::DisplayEvents, ErrorVariant, }, @@ -109,60 +110,66 @@ impl CallCommand { let signer = super::pair_signer(self.extrinsic_opts.signer()?); - Runtime::new().unwrap().block_on(async { - let url = self.extrinsic_opts.url_to_string(); - let client = OnlineClient::from_url(url.clone()).await?; + Runtime::new() + .map_err(|e| { + ErrorVariant::Generic(GenericError::from_message(e.to_string())) + })? + .block_on(async { + let url = self.extrinsic_opts.url_to_string(); + let client = OnlineClient::from_url(url.clone()).await?; - if !self.extrinsic_opts.execute { - let result = self - .call_dry_run(call_data.clone(), &client, &signer) - .await?; - match result.result { - Ok(ref ret_val) => { - let value = transcoder - .decode_return(&self.message, &mut &ret_val.data[..]) - .context(format!( - "Failed to decode return value {:?}", - &ret_val - ))?; - let dry_run_result = CallDryRunResult { - result: String::from("Success!"), - reverted: ret_val.did_revert(), - data: value, - gas_consumed: result.gas_consumed, - gas_required: result.gas_required, - storage_deposit: StorageDeposit::from( - &result.storage_deposit, - ), - }; - if self.output_json { - println!("{}", dry_run_result.to_json()?); - } else { - dry_run_result.print(); - display_contract_exec_result_debug::<_, DEFAULT_KEY_COL_WIDTH>( - &result, - )?; - display_dry_run_result_warning("message"); - }; - } - Err(ref err) => { - let metadata = client.metadata(); - let object = ErrorVariant::from_dispatch_error(err, &metadata)?; - if self.output_json { - return Err(object) - } else { - name_value_println!("Result", object, MAX_KEY_COL_WIDTH); - display_contract_exec_result::<_, MAX_KEY_COL_WIDTH>( - &result, - )?; + if !self.extrinsic_opts.execute { + let result = self + .call_dry_run(call_data.clone(), &client, &signer) + .await?; + match result.result { + Ok(ref ret_val) => { + let value = transcoder + .decode_return(&self.message, &mut &ret_val.data[..]) + .context(format!( + "Failed to decode return value {:?}", + &ret_val + ))?; + let dry_run_result = CallDryRunResult { + result: String::from("Success!"), + reverted: ret_val.did_revert(), + data: value, + gas_consumed: result.gas_consumed, + gas_required: result.gas_required, + storage_deposit: StorageDeposit::from( + &result.storage_deposit, + ), + }; + if self.output_json { + println!("{}", dry_run_result.to_json()?); + } else { + dry_run_result.print(); + display_contract_exec_result_debug::< + _, + DEFAULT_KEY_COL_WIDTH, + >(&result)?; + display_dry_run_result_warning("message"); + }; + } + Err(ref err) => { + let metadata = client.metadata(); + let object = + ErrorVariant::from_dispatch_error(err, &metadata)?; + if self.output_json { + return Err(object) + } else { + name_value_println!("Result", object, MAX_KEY_COL_WIDTH); + display_contract_exec_result::<_, MAX_KEY_COL_WIDTH>( + &result, + )?; + } } } + } else { + self.call(&client, call_data, &signer, &transcoder).await?; } - } else { - self.call(&client, call_data, &signer, &transcoder).await?; - } - Ok(()) - }) + Ok(()) + }) } async fn call_dry_run( diff --git a/crates/cargo-contract/src/cmd/extrinsics/instantiate.rs b/crates/cargo-contract/src/cmd/extrinsics/instantiate.rs index 9c3e5d517..0f1ff49c7 100644 --- a/crates/cargo-contract/src/cmd/extrinsics/instantiate.rs +++ b/crates/cargo-contract/src/cmd/extrinsics/instantiate.rs @@ -16,6 +16,7 @@ use super::{ display_contract_exec_result, + error::GenericError, prompt_confirm_tx, state_call, submit_extrinsic, @@ -126,41 +127,45 @@ impl InstantiateCommand { }; let salt = self.salt.clone().map(|s| s.0).unwrap_or_default(); - Runtime::new().unwrap().block_on(async { - let client = OnlineClient::from_url(url.clone()).await?; - - let token_metadata = TokenMetadata::query(&client).await?; - - let args = InstantiateArgs { - constructor: self.constructor.clone(), - raw_args: self.args.clone(), - value: self.value.denominate_balance(&token_metadata)?, - gas_limit: self.gas_limit, - proof_size: self.proof_size, - storage_deposit_limit: self - .extrinsic_opts - .storage_deposit_limit - .as_ref() - .map(|bv| bv.denominate_balance(&token_metadata)) - .transpose()?, - code, - data, - salt, - }; - - let exec = Exec { - args, - opts: self.extrinsic_opts.clone(), - url, - client, - verbosity, - signer, - transcoder, - output_json: self.output_json, - }; - - exec.exec(self.extrinsic_opts.execute).await - }) + Runtime::new() + .map_err(|e| { + ErrorVariant::Generic(GenericError::from_message(e.to_string())) + })? + .block_on(async { + let client = OnlineClient::from_url(url.clone()).await?; + + let token_metadata = TokenMetadata::query(&client).await?; + + let args = InstantiateArgs { + constructor: self.constructor.clone(), + raw_args: self.args.clone(), + value: self.value.denominate_balance(&token_metadata)?, + gas_limit: self.gas_limit, + proof_size: self.proof_size, + storage_deposit_limit: self + .extrinsic_opts + .storage_deposit_limit + .as_ref() + .map(|bv| bv.denominate_balance(&token_metadata)) + .transpose()?, + code, + data, + salt, + }; + + let exec = Exec { + args, + opts: self.extrinsic_opts.clone(), + url, + client, + verbosity, + signer, + transcoder, + output_json: self.output_json, + }; + + exec.exec(self.extrinsic_opts.execute).await + }) } } diff --git a/crates/cargo-contract/src/cmd/extrinsics/mod.rs b/crates/cargo-contract/src/cmd/extrinsics/mod.rs index 31925eaf8..0374076fc 100644 --- a/crates/cargo-contract/src/cmd/extrinsics/mod.rs +++ b/crates/cargo-contract/src/cmd/extrinsics/mod.rs @@ -91,7 +91,10 @@ pub use balance::{ pub use call::CallCommand; use contract_metadata::ContractMetadata; pub use contract_transcode::ContractMessageTranscoder; -pub use error::ErrorVariant; +pub use error::{ + ErrorVariant, + GenericError, +}; pub use instantiate::InstantiateCommand; pub use remove::RemoveCommand; pub use subxt::PolkadotConfig as DefaultConfig; diff --git a/crates/cargo-contract/src/cmd/extrinsics/remove.rs b/crates/cargo-contract/src/cmd/extrinsics/remove.rs index dd8b826e4..93cd3c1df 100644 --- a/crates/cargo-contract/src/cmd/extrinsics/remove.rs +++ b/crates/cargo-contract/src/cmd/extrinsics/remove.rs @@ -15,6 +15,7 @@ // along with cargo-contract. If not, see . use super::{ + error::GenericError, submit_extrinsic, Client, ContractMessageTranscoder, @@ -88,35 +89,39 @@ impl RemoveCommand { } }?; - Runtime::new().unwrap().block_on(async { - let url = self.extrinsic_opts.url_to_string(); - let client = OnlineClient::from_url(url.clone()).await?; - if let Some(code_removed) = self - .remove_code( - &client, - sp_core::H256(final_code_hash), - &signer, - &transcoder, - ) - .await? - { - let remove_result = code_removed.code_hash; + Runtime::new() + .map_err(|e| { + ErrorVariant::Generic(GenericError::from_message(e.to_string())) + })? + .block_on(async { + let url = self.extrinsic_opts.url_to_string(); + let client = OnlineClient::from_url(url.clone()).await?; + if let Some(code_removed) = self + .remove_code( + &client, + sp_core::H256(final_code_hash), + &signer, + &transcoder, + ) + .await? + { + let remove_result = code_removed.code_hash; - if self.output_json { - println!("{}", &remove_result); + if self.output_json { + println!("{}", &remove_result); + } else { + name_value_println!("Code hash", format!("{remove_result:?}")); + } + Result::<(), ErrorVariant>::Ok(()) } else { - name_value_println!("Code hash", format!("{remove_result:?}")); + let error_code_hash = hex::encode(final_code_hash); + Err(anyhow::anyhow!( + "Error removing the code for the supplied code hash: {}", + error_code_hash + ) + .into()) } - Result::<(), ErrorVariant>::Ok(()) - } else { - let error_code_hash = hex::encode(final_code_hash); - Err(anyhow::anyhow!( - "Error removing the code for the supplied code hash: {}", - error_code_hash - ) - .into()) - } - }) + }) } async fn remove_code( diff --git a/crates/cargo-contract/src/cmd/extrinsics/upload.rs b/crates/cargo-contract/src/cmd/extrinsics/upload.rs index adf3a3280..6ef8436d2 100644 --- a/crates/cargo-contract/src/cmd/extrinsics/upload.rs +++ b/crates/cargo-contract/src/cmd/extrinsics/upload.rs @@ -16,6 +16,7 @@ use super::{ display_dry_run_result_warning, + error::GenericError, state_call, submit_extrinsic, Client, @@ -78,54 +79,58 @@ impl UploadCommand { })?; let code_hash = code.code_hash(); - Runtime::new().unwrap().block_on(async { - let url = self.extrinsic_opts.url_to_string(); - let client = OnlineClient::from_url(url.clone()).await?; - - if !self.extrinsic_opts.execute { - match self.upload_code_rpc(code, &client, &signer).await? { - Ok(result) => { - let upload_result = UploadDryRunResult { - result: String::from("Success!"), - code_hash: format!("{:?}", result.code_hash), - deposit: result.deposit, - }; - if self.output_json { - println!("{}", upload_result.to_json()?); - } else { - upload_result.print(); - display_dry_run_result_warning("upload"); + Runtime::new() + .map_err(|e| + ErrorVariant::Generic(GenericError::from_message(e.to_string())) + )? + .block_on(async { + let url = self.extrinsic_opts.url_to_string(); + let client = OnlineClient::from_url(url.clone()).await?; + + if !self.extrinsic_opts.execute { + match self.upload_code_rpc(code, &client, &signer).await? { + Ok(result) => { + let upload_result = UploadDryRunResult { + result: String::from("Success!"), + code_hash: format!("{:?}", result.code_hash), + deposit: result.deposit, + }; + if self.output_json { + println!("{}", upload_result.to_json()?); + } else { + upload_result.print(); + display_dry_run_result_warning("upload"); + } } - } - Err(err) => { - let metadata = client.metadata(); - let err = ErrorVariant::from_dispatch_error(&err, &metadata)?; - if self.output_json { - return Err(err) - } else { - name_value_println!("Result", err); + Err(err) => { + let metadata = client.metadata(); + let err = ErrorVariant::from_dispatch_error(&err, &metadata)?; + if self.output_json { + return Err(err) + } else { + name_value_println!("Result", err); + } } } - } - } else if let Some(code_stored) = - self.upload_code(&client, code, &signer).await? - { - let upload_result = UploadResult { - code_hash: format!("{:?}", code_stored.code_hash), - }; - if self.output_json { - println!("{}", upload_result.to_json()?); + } else if let Some(code_stored) = + self.upload_code(&client, code, &signer).await? + { + let upload_result = UploadResult { + code_hash: format!("{:?}", code_stored.code_hash), + }; + if self.output_json { + println!("{}", upload_result.to_json()?); + } else { + upload_result.print(); + } } else { - upload_result.print(); + let code_hash = hex::encode(code_hash); + return Err(anyhow::anyhow!( + "This contract has already been uploaded with code hash: 0x{code_hash}" + ) + .into()) } - } else { - let code_hash = hex::encode(code_hash); - return Err(anyhow::anyhow!( - "This contract has already been uploaded with code hash: 0x{code_hash}" - ) - .into()) - } - Ok(()) + Ok(()) }) } diff --git a/crates/cargo-contract/src/cmd/info.rs b/crates/cargo-contract/src/cmd/info.rs index 7f6e1df8f..60aab54e0 100644 --- a/crates/cargo-contract/src/cmd/info.rs +++ b/crates/cargo-contract/src/cmd/info.rs @@ -21,7 +21,10 @@ use super::{ }; use crate::{ cmd::{ - extrinsics::MAX_KEY_COL_WIDTH, + extrinsics::{ + GenericError, + MAX_KEY_COL_WIDTH, + }, runtime_api::api::runtime_types::pallet_contracts::storage::ContractInfo, Balance, CodeHash, @@ -66,37 +69,41 @@ impl InfoCommand { self.contract ); - Runtime::new().unwrap().block_on(async { - let url = self.url.clone(); - let client = OnlineClient::::from_url(url).await?; + Runtime::new() + .map_err(|e| { + ErrorVariant::Generic(GenericError::from_message(e.to_string())) + })? + .block_on(async { + let url = self.url.clone(); + let client = OnlineClient::::from_url(url).await?; - let info_result = self.fetch_contract_info(&client).await?; + let info_result = self.fetch_contract_info(&client).await?; - match info_result { - Some(info_result) => { - let convert_trie_id = hex::encode(info_result.trie_id.0); - let info_to_json = InfoToJson { - trie_id: convert_trie_id, - code_hash: info_result.code_hash, - storage_items: info_result.storage_items, - storage_item_deposit: info_result.storage_item_deposit, - }; - if self.output_json { - println!("{}", info_to_json.to_json()?); - } else { - info_to_json.basic_display_format_contract_info(); + match info_result { + Some(info_result) => { + let convert_trie_id = hex::encode(info_result.trie_id.0); + let info_to_json = InfoToJson { + trie_id: convert_trie_id, + code_hash: info_result.code_hash, + storage_items: info_result.storage_items, + storage_item_deposit: info_result.storage_item_deposit, + }; + if self.output_json { + println!("{}", info_to_json.to_json()?); + } else { + info_to_json.basic_display_format_contract_info(); + } + Ok(()) + } + None => { + Err(anyhow!( + "No contract information was found for account id {}", + self.contract + ) + .into()) } - Ok(()) - } - None => { - Err(anyhow!( - "No contract information was found for account id {}", - self.contract - ) - .into()) } - } - }) + }) } async fn fetch_contract_info(&self, client: &Client) -> Result> { From 55b72f11bfd62c39453af33074eeca57d356f0db Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Thu, 8 Jun 2023 18:44:32 +0100 Subject: [PATCH 09/64] fix readme example --- crates/build/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/build/README.md b/crates/build/README.md index ad051ff54..490f479ce 100644 --- a/crates/build/README.md +++ b/crates/build/README.md @@ -1,6 +1,6 @@ # contract-build -A crate for building [`ink!`](https://github.com/paritytech/ink) smart contracts. Used by +A crate for building [`ink!`](https://github.com/paritytech/ink) smart contracts. Used by [`cargo-contract`](https://github.com/paritytech/cargo-contract). ## Usage @@ -36,7 +36,8 @@ let args = contract_build::ExecuteArgs { skip_wasm_validation: false, target: Target::Wasm, max_memory_pages: 16, + image: None, }; contract_build::execute(args); -``` \ No newline at end of file +``` From 9955f258744cbc04634615f5e12e60eec2d4761f Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Thu, 8 Jun 2023 19:22:58 +0100 Subject: [PATCH 10/64] fix docs --- crates/metadata/src/lib.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/metadata/src/lib.rs b/crates/metadata/src/lib.rs index d3e7c88ba..1cb5b6a07 100644 --- a/crates/metadata/src/lib.rs +++ b/crates/metadata/src/lib.rs @@ -54,8 +54,13 @@ //! let user = User::new(user_json); //! // contract abi raw json generated by contract compilation //! let abi_json: Map = Map::new(); +//! // image digest used for the verifiable build. +//! let image = String::from( +//! "sha256:824b6fde00873a146f52b41e7154e98c9d371609f9fd77e1b22ccd530f052d66", +//! ); //! -//! let metadata = ContractMetadata::new(source, contract, Some(user), abi_json); +//! let metadata = +//! ContractMetadata::new(source, contract, Some(image), Some(user), abi_json); //! //! // serialize to json //! let json = serde_json::to_value(&metadata).unwrap(); From 5fe03fb73fa51e4a2052f226beb7430bb1a43c42 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Thu, 8 Jun 2023 19:32:28 +0100 Subject: [PATCH 11/64] update README --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 4d904b8a9..867e28d01 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,9 @@ Modern releases of gcc and clang, as well as Visual Studio 2019+ should work. * (MacOS) `brew install openssl` * `cargo install cargo-dylint dylint-link`. +* Step 4: (**Optional**) Install [Docker Engine](https://docs.docker.com/engine/install) +to be able to produce verifiable builds. + You can always update the `cargo-contract` binary to the latest version by running the Step 2. ### Installation using Docker Image @@ -69,6 +72,16 @@ docker run --rm -it -v $(pwd):/sources paritytech/contracts-ci-linux:production If you want to reproduce other steps of CI process you can use the following [guide](https://github.com/paritytech/scripts#reproduce-ci-locally). +### Verifiable builds + +Some block explorers require the Wasm binary to be compiled in the deterministic environment. +To achieve it, you should build your contract using Docker image we provide: +```bash +cargo contract build --verifiable +``` + +You can find more detailed documentation how to use the image [here](/build-image/README.md) + ## Usage You can always use `cargo contract help` to print information on available From 47d63d04d1d2128fd681bef184f90b7b97024c20 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Mon, 12 Jun 2023 11:07:35 +0100 Subject: [PATCH 12/64] docs fixes --- build-image/Dockerfile | 4 +--- build-image/README.md | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/build-image/Dockerfile b/build-image/Dockerfile index 5dfb708c1..bbe381a56 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -3,8 +3,6 @@ FROM docker.io/bitnami/minideb:bullseye-amd64 as slimmed-rust # metadata LABEL io.parity.image.vendor="Parity Technologies" \ io.parity.image.title="contracts-verifiable" \ - io.parity.image.documentation="https://github.com/paritytech/scripts/blob/master/\ - dockerfiles/ink-ci-linux/README.md" \ io.parity.image.description="Inherits from docker.io/bitnami/minideb:bullseye. \ rust nightly, clippy, rustfmt, miri, rust-src, rustc-dev, grcov, rust-covfix, \ llvm-tools-preview, cargo-contract, xargo, binaryen, parallel, codecov, ink, solang" \ @@ -41,7 +39,7 @@ ENV RUSTFLAGS="-C target-feature=-crt-static" RUN apt-get -y update && apt-get -y install gcc g++ # Install cargo contract -RUN cargo install cargo-contract +RUN cargo install cargo-contract --version 3.0.1 # Cleanup after `cargo install` RUN rm -rf ${CARGO_HOME}/"registry" ${CARGO_HOME}/"git" /root/.cache/sccache diff --git a/build-image/README.md b/build-image/README.md index 2e7332796..82d1b480c 100644 --- a/build-image/README.md +++ b/build-image/README.md @@ -29,7 +29,7 @@ docker run -d \ paritytech/contracts-verified ``` -If you have multi-contract project: +For multi-contract projects, like in the example below: ``` my-app/ ├─ ink-project-a/ @@ -40,5 +40,5 @@ my-app/ │ ├─ lib.rs ├─ rust-toolchain ``` -Make sure you mount `my-app` directory and then call +Make sure to run the command inside `my-app` directory and specify relative manifest paths: `cargo contract build --verifiable --release --manifest-path ink-project-a/Cargo.toml` From 6d3d7fd2eadc26ffe3af85ab0f6a4b161f8c36bd Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Mon, 12 Jun 2023 11:08:49 +0100 Subject: [PATCH 13/64] clarification in the docs --- build-image/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-image/README.md b/build-image/README.md index 82d1b480c..50a78cdca 100644 --- a/build-image/README.md +++ b/build-image/README.md @@ -40,5 +40,6 @@ my-app/ │ ├─ lib.rs ├─ rust-toolchain ``` -Make sure to run the command inside `my-app` directory and specify relative manifest paths: +Make sure to run the command inside `my-app` directory and specify a relative manifest path +to the root contract: `cargo contract build --verifiable --release --manifest-path ink-project-a/Cargo.toml` From 334ed7ca6dbc49493eacaa82eaeea0bcf1a4f364 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Mon, 12 Jun 2023 11:29:43 +0100 Subject: [PATCH 14/64] use locked for cargo install --- build-image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-image/Dockerfile b/build-image/Dockerfile index bbe381a56..993dd9093 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -39,7 +39,7 @@ ENV RUSTFLAGS="-C target-feature=-crt-static" RUN apt-get -y update && apt-get -y install gcc g++ # Install cargo contract -RUN cargo install cargo-contract --version 3.0.1 +RUN cargo install cargo-contract --locked --version 3.0.1 # Cleanup after `cargo install` RUN rm -rf ${CARGO_HOME}/"registry" ${CARGO_HOME}/"git" /root/.cache/sccache From 55ce730bb22178a356cb91a1ea43348ac09eeb1d Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Mon, 12 Jun 2023 12:21:18 +0100 Subject: [PATCH 15/64] concat install and cleanup commands --- build-image/Dockerfile | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/build-image/Dockerfile b/build-image/Dockerfile index 993dd9093..06cd47087 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -37,18 +37,17 @@ FROM slimmed-rust as cc-builder # This is important, see https://github.com/rust-lang/docker-rust/issues/85 ENV RUSTFLAGS="-C target-feature=-crt-static" -RUN apt-get -y update && apt-get -y install gcc g++ -# Install cargo contract -RUN cargo install cargo-contract --locked --version 3.0.1 - -# Cleanup after `cargo install` -RUN rm -rf ${CARGO_HOME}/"registry" ${CARGO_HOME}/"git" /root/.cache/sccache - -# apt clean up -RUN apt-get remove -y gnupg && \ - apt-get autoremove -y && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* + # Install required packages for `cargo-contract` +RUN apt-get -y update && apt-get -y install gcc g++ \ + # Install cargo contract + && cargo install cargo-contract --locked --version 3.0.1 \ + # Cleanup after `cargo install` + && rm -rf ${CARGO_HOME}/"registry" ${CARGO_HOME}/"git" /root/.cache/sccache \ + # apt clean up + && apt-get remove -y gnupg \ + && apt-get autoremove -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* FROM slimmed-rust as ink-dev From 54e029134af968ddb46ba69bf4f1fde9e316c9c8 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 13 Jun 2023 12:41:41 +0100 Subject: [PATCH 16/64] use arg instead of env in dockerfile --- build-image/Dockerfile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/build-image/Dockerfile b/build-image/Dockerfile index 06cd47087..096b5cac2 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -1,3 +1,10 @@ +# The rust version to use +# The 1.69 toolchain is temporarily required to build ink! contracts because of +# https://github.com/paritytech/cargo-contract/issues/1139 +ARG RUST_VERSION=1.69 +# The cargo contract version to use +ARG CARGO_CONTRACT_VERSION=3.0.1 + FROM docker.io/bitnami/minideb:bullseye-amd64 as slimmed-rust # metadata @@ -11,10 +18,7 @@ LABEL io.parity.image.vendor="Parity Technologies" \ ENV RUSTUP_HOME=/usr/local/rustup \ CARGO_HOME=/usr/local/cargo \ - PATH=/usr/local/cargo/bin:$PATH \ - # The 1.69 toolchain is temporarily required to build ink! contracts because of - # https://github.com/paritytech/cargo-contract/issues/1139 - RUST_VERSION=1.69 + PATH=/usr/local/cargo/bin:$PATH # Minimal Rust dependencies. RUN set -eux \ @@ -40,7 +44,7 @@ ENV RUSTFLAGS="-C target-feature=-crt-static" # Install required packages for `cargo-contract` RUN apt-get -y update && apt-get -y install gcc g++ \ # Install cargo contract - && cargo install cargo-contract --locked --version 3.0.1 \ + && cargo install cargo-contract --locked --version $CARGO_CONTRACT_VERSION \ # Cleanup after `cargo install` && rm -rf ${CARGO_HOME}/"registry" ${CARGO_HOME}/"git" /root/.cache/sccache \ # apt clean up From e28ddcbf759b97ca076daef57089bdcf79e32fe6 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 13 Jun 2023 12:44:13 +0100 Subject: [PATCH 17/64] add version labels --- build-image/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build-image/Dockerfile b/build-image/Dockerfile index 096b5cac2..50243b135 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -14,7 +14,9 @@ LABEL io.parity.image.vendor="Parity Technologies" \ rust nightly, clippy, rustfmt, miri, rust-src, rustc-dev, grcov, rust-covfix, \ llvm-tools-preview, cargo-contract, xargo, binaryen, parallel, codecov, ink, solang" \ io.parity.image.documentation="https://github.com/paritytech/cargo-contract/blob/master/\ - build-image/README.md" + build-image/README.md" \ + io.parity.version.rust=${RUST_VERSION} \ + io.parity.version.cargo-contract=${CARGO_CONTRACT_VERSION} ENV RUSTUP_HOME=/usr/local/rustup \ CARGO_HOME=/usr/local/cargo \ From a51ea03ad606263753922f620953f6f41e66fef3 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 13 Jun 2023 12:48:48 +0100 Subject: [PATCH 18/64] update comments --- build-image/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-image/README.md b/build-image/README.md index 50a78cdca..b788e2e4e 100644 --- a/build-image/README.md +++ b/build-image/README.md @@ -1,6 +1,6 @@ # Verifiable builds using Docker -Docker image based on our base CI image ``. +Docker image based on the minimalistic Debian image `bitnami/minideb:bullseye-amd64`. Used for reproducible builds in `cargo contract build --verifiable` From 80c92529859e085dcf45be87d8af3278a8198c76 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 13 Jun 2023 14:36:45 +0100 Subject: [PATCH 19/64] decompose the function --- crates/build/src/docker.rs | 492 ++++++++++++++++++++----------------- 1 file changed, 266 insertions(+), 226 deletions(-) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index 9c33b26ce..e3ff4c7ac 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -7,7 +7,11 @@ use std::{ Hash, Hasher, }, - path::Path, + io::BufReader, + path::{ + Path, + PathBuf, + }, }; use anyhow::{ @@ -25,6 +29,7 @@ use bollard::{ image::ListImagesOptions, service::{ HostConfig, + ImageSummary, Mount, MountTypeEnum, }, @@ -40,8 +45,11 @@ use crate::{ BuildSteps, CrateMetadata, ExecuteArgs, + Features, Network, OptimizationPasses, + UnstableFlags, + Verbosity, DEFAULT_MAX_MEMORY_PAGES, }; @@ -59,7 +67,7 @@ pub fn docker_build(args: ExecuteArgs) -> Result { let ExecuteArgs { manifest_path, verbosity, - features, + mut features, network, unstable_flags, optimization_passes, @@ -73,21 +81,30 @@ pub fn docker_build(args: ExecuteArgs) -> Result { tokio::runtime::Runtime::new()?.block_on(async { let mut build_steps = BuildSteps::new(); - build_steps.set_total_steps(5); + build_steps.set_total_steps(3); if build_artifact == BuildArtifacts::CodeOnly { - build_steps.set_total_steps(4); + build_steps.set_total_steps(2); } - maybe_println!( - verbosity, - " {} {}", - format!("{build_steps}").bold(), - "Executing verifiable build".bright_green().bold() - ); - - let client = Docker::connect_with_socket_defaults()?; + let crate_metadata = CrateMetadata::collect(&manifest_path, target)?; + let host_folder = crate_metadata + .manifest_path + .absolute_directory()? + .as_path() + .to_owned(); + let file_path = host_folder.join(Path::new("target/build_result.json")); + let args = compose_build_args( + &mut features, + keep_debug_symbols, + optimization_passes.as_ref(), + &network, + &unstable_flags, + &build_artifact, + max_memory_pages, + )?; // TODO: replace with image pulling, once the image is pushed to registry + let client = Docker::connect_with_socket_defaults()?; let images = client .list_images(Some(ListImagesOptions:: { all: true, @@ -103,244 +120,267 @@ pub fn docker_build(args: ExecuteArgs) -> Result { } let build_image = build_image.unwrap(); - let crate_metadata = CrateMetadata::collect(&manifest_path, target)?; - let host_folder = crate_metadata - .manifest_path - .absolute_directory()? - .as_path() - .to_owned(); + run_build( + args, + build_image, + &crate_metadata.contract_artifact_name, + &host_folder, + &verbosity, + &mut build_steps, + ) + .await?; - let file_path = host_folder.join(Path::new("target/build_result.json")); - - let mount = Mount { - target: Some(String::from("/contract")), - source: Some( - host_folder - .to_str() - .context("Cannot convert path to string.")? - .to_string(), - ), - typ: Some(MountTypeEnum::BIND), - ..Default::default() - }; - let host_cfg = Some(HostConfig { - mounts: Some(vec![mount]), - ..Default::default() - }); + let build_result = read_build_result(&host_folder, &file_path)?; - let mut entrypoint = vec!["/bin/bash".to_string(), "-c".to_string()]; + update_metadata(&build_result, &verbosity, &mut build_steps, build_image)?; - let mut args: Vec = vec!["--release".to_string()]; - - features.append_to_args(&mut args); - if keep_debug_symbols { - args.push("--keep-debug-symbols".to_owned()); - } - - if let Some(passes) = optimization_passes { - if passes != OptimizationPasses::default() { - args.push(format!("--optimization-passes {}", passes)); - } - } - - if network == Network::Offline { - args.push("--offline".to_owned()); - } - - if unstable_flags.original_manifest { - args.push("-Z original-manifest".to_owned()); - } + Ok(BuildResult { + output_type, + verbosity, + ..build_result + }) + }) +} - let s = match build_artifact { - BuildArtifacts::CodeOnly => "--generate code-only".to_owned(), - BuildArtifacts::All => String::new(), - BuildArtifacts::CheckOnly => { - anyhow::bail!("--generate check-only is invalid flag for this command!"); +/// Reads the `BuildResult` produced by the docker execution +fn read_build_result(host_folder: &Path, file_path: &PathBuf) -> Result { + let file = std::fs::File::open(file_path)?; + let mut build_result: BuildResult = + match serde_json::from_reader(BufReader::new(file)) { + Ok(result) => result, + Err(_) => { + std::fs::remove_file(file_path)?; + anyhow::bail!( + "Error parsing output from docker build. The build probably failed!" + ) } }; - args.push(s); + let new_path = host_folder.join( + build_result + .target_directory + .as_path() + .strip_prefix("/contract")?, + ); + build_result.target_directory = new_path; + + let new_path = build_result.dest_wasm.as_ref().map(|p| { + host_folder.join( + p.as_path() + .strip_prefix("/contract") + .expect("cannot strip prefix"), + ) + }); + build_result.dest_wasm = new_path; + + build_result.metadata_result.as_mut().map(|mut m| { + m.dest_bundle = host_folder.join( + m.dest_bundle + .as_path() + .strip_prefix("/contract") + .expect("cannot strip prefix"), + ); + m.dest_metadata = host_folder.join( + m.dest_metadata + .as_path() + .strip_prefix("/contract") + .expect("cannot strip prefix"), + ); + m + }); + Ok(build_result) +} - if max_memory_pages != DEFAULT_MAX_MEMORY_PAGES { - args.push(format!("--max-memory-pages {}", max_memory_pages)); - } +fn update_metadata( + build_result: &BuildResult, + verbosity: &Verbosity, + build_steps: &mut BuildSteps, + build_image: &ImageSummary, +) -> Result<()> { + if let Some(metadata_artifacts) = &build_result.metadata_result { + let mut metadata = ContractMetadata::load(&metadata_artifacts.dest_bundle)?; + metadata.image = Some(build_image.id.to_string()); + + crate::metadata::write_metadata( + metadata_artifacts, + metadata, + build_steps, + verbosity, + )?; + } + Ok(()) +} - let joined_args = args.join(" "); +async fn run_build( + build_args: String, + build_image: &ImageSummary, + contract_name: &str, + host_folder: &Path, + verbosity: &Verbosity, + build_steps: &mut BuildSteps, +) -> Result<()> { + let client = Docker::connect_with_socket_defaults()?; - let mut cmds = vec![format!( + let mut entrypoint = vec!["/bin/bash".to_string(), "-c".to_string()]; + + let mut cmds = vec![format!( "mkdir -p target && cargo contract build {} --output-json > target/build_result.json", - joined_args + build_args )]; - entrypoint.append(&mut cmds); - - // in order to optimise the container usage - // we are hashing the inputted command - // in order to reuse containers for different permutations of arguments - let mut s = DefaultHasher::new(); - entrypoint.hash(&mut s); - let digest = s.finish(); - //taking the first 5 digits to be a unique identifier - let digest_code: String = digest.to_string().chars().take(5).collect(); + entrypoint.append(&mut cmds); - let container_name = - format!("ink-verified-{}-{}", crate_metadata.contract_artifact_name, digest_code); + // in order to optimise the container usage + // we are hashing the inputted command + // in order to reuse containers for different permutations of arguments + let mut s = DefaultHasher::new(); + entrypoint.hash(&mut s); + let digest = s.finish(); + // taking the first 5 digits to be a unique identifier + let digest_code: String = digest.to_string().chars().take(5).collect(); - let mut filters = HashMap::new(); - filters.insert("name".to_string(), vec![container_name.clone()]); + let container_name = format!("ink-verified-{}-{}", contract_name, digest_code); - let containers = client - .list_containers(Some(ListContainersOptions:: { - all: true, - filters, - ..Default::default() - })) - .await?; + let mut filters = HashMap::new(); + filters.insert("name".to_string(), vec![container_name.clone()]); - let mut labels = HashMap::new(); - labels.insert("cmd_digest".to_string(), digest.to_string()); - let config = Config { - image: Some(build_image.id.clone()), - entrypoint: Some(entrypoint), - cmd: None, - labels: Some(labels), - host_config: host_cfg, - attach_stderr: Some(true), - tty: Some(true), + let containers = client + .list_containers(Some(ListContainersOptions:: { + all: true, + filters, ..Default::default() - }; - let options = Some(CreateContainerOptions { - name: container_name.as_str(), - platform: Some("linux/amd64"), - }); - - let container_id: String = if containers.is_empty() { - client - .create_container(options.clone(), config.clone()) - .await? - .id - } else { - let c = containers - .first() - .context("Error finding existing container")? - .clone(); - if c.labels - .context("image does not have labels")? - .get("cmd_digest") - == Some(&digest.to_string()) - { - c.id.context("Container does not have an id")? - } else { - client - .create_container(options.clone(), config.clone()) - .await? - .id - } - }; - - client - .start_container::(&container_id, None) - .await?; - - build_steps.increment_current(); - maybe_println!( - verbosity, - " {} {}\n{}", - format!("{build_steps}").bold(), - "Started the build inside the container" - .bright_green() - .bold(), - "This might take a while. Check container logs for more details." - ); - - let options = Some(WaitContainerOptions { - condition: "not-running", - }); - - let mut wait_stream = client.wait_container(&container_id, options); - while wait_stream.next().await.is_some() {} - + })) + .await?; + + let container = containers.first(); + + let mount = Mount { + target: Some(String::from("/contract")), + source: Some( + host_folder + .to_str() + .context("Cannot convert path to string.")? + .to_string(), + ), + typ: Some(MountTypeEnum::BIND), + ..Default::default() + }; + let host_cfg = Some(HostConfig { + mounts: Some(vec![mount]), + ..Default::default() + }); + + let mut labels = HashMap::new(); + labels.insert("cmd_digest".to_string(), digest.to_string()); + let config = Config { + image: Some(build_image.id.clone()), + entrypoint: Some(entrypoint), + cmd: None, + labels: Some(labels), + host_config: host_cfg, + attach_stderr: Some(true), + tty: Some(true), + ..Default::default() + }; + let options = Some(CreateContainerOptions { + name: container_name.as_str(), + platform: Some("linux/amd64"), + }); + + let container_id = if container.is_none() { client - .stop_container(&container_id, Some(StopContainerOptions { t: 20 })) - .await?; + .create_container(options.clone(), config.clone()) + .await? + .id + } else { + container + .unwrap() + .id + .clone() + .context("Container does not have an ID")? + .to_owned() + }; + + client + .start_container::(&container_id, None) + .await?; + + maybe_println!( + verbosity, + " {} {}\n{}", + format!("{build_steps}").bold(), + "Started the build inside the container" + .bright_green() + .bold(), + "This might take a while. Check container logs for more details." + ); + + let options = Some(WaitContainerOptions { + condition: "not-running", + }); + + let mut wait_stream = client.wait_container(&container_id, options); + while wait_stream.next().await.is_some() {} + + client + .stop_container(&container_id, Some(StopContainerOptions { t: 20 })) + .await?; + + build_steps.increment_current(); + maybe_println!( + verbosity, + " {} {}", + format!("{build_steps}").bold(), + "Docker container has finished the build. Reading the results" + .bright_green() + .bold(), + ); + Ok(()) +} - build_steps.increment_current(); - maybe_println!( - verbosity, - " {} {}", - format!("{build_steps}").bold(), - "Docker container has finished the build." - .bright_green() - .bold(), - ); +fn compose_build_args( + features: &mut Features, + keep_debug_symbols: bool, + optimization_passes: Option<&OptimizationPasses>, + network: &Network, + unstable_flags: &UnstableFlags, + build_artifact: &BuildArtifacts, + max_memory_pages: u32, +) -> Result { + let mut args: Vec = vec!["--release".to_string()]; + features.append_to_args(&mut args); + if keep_debug_symbols { + args.push("--keep-debug-symbols".to_owned()); + } + + if let Some(passes) = optimization_passes { + if passes != &OptimizationPasses::default() { + args.push(format!("--optimization-passes {}", passes)); + } + } - let result_contents = match std::fs::read_to_string(&file_path) { - Ok(content) => { - std::fs::remove_file(&file_path)?; - content - } - Err(e) => { - std::fs::remove_file(&file_path)?; - anyhow::bail!(e); - } - }; + if network == &Network::Offline { + args.push("--offline".to_owned()); + } - let mut build_result: BuildResult = serde_json::from_str(&result_contents) - .map_err(|_| { - anyhow::anyhow!( - "Error parsing output from docker build. The build probably failed!" - ) - })?; + if unstable_flags.original_manifest { + args.push("-Z original-manifest".to_owned()); + } - let new_path = host_folder.join( - build_result - .target_directory - .as_path() - .strip_prefix("/contract")?, - ); - build_result.target_directory = new_path; - - let new_path = build_result.dest_wasm.as_ref().map(|p| { - host_folder.join( - p.as_path() - .strip_prefix("/contract") - .expect("cannot strip prefix"), - ) - }); - build_result.dest_wasm = new_path; - - build_result.metadata_result.as_mut().map(|mut m| { - m.dest_bundle = host_folder.join( - m.dest_bundle - .as_path() - .strip_prefix("/contract") - .expect("cannot strip prefix"), - ); - m.dest_metadata = host_folder.join( - m.dest_metadata - .as_path() - .strip_prefix("/contract") - .expect("cannot strip prefix"), - ); - m - }); + let s = match build_artifact { + BuildArtifacts::CodeOnly => "--generate code-only".to_owned(), + BuildArtifacts::All => String::new(), + BuildArtifacts::CheckOnly => { + anyhow::bail!("--generate check-only is invalid flag for this command!"); + } + }; - if let Some(metadata_artifacts) = &build_result.metadata_result { - let mut metadata = ContractMetadata::load(&metadata_artifacts.dest_bundle)?; - metadata.image = Some(build_image.id.to_string()); + args.push(s); - crate::metadata::write_metadata( - metadata_artifacts, - metadata, - &mut build_steps, - &verbosity, - )?; - } + if max_memory_pages != DEFAULT_MAX_MEMORY_PAGES { + args.push(format!("--max-memory-pages {}", max_memory_pages)); + } - Ok(BuildResult { - output_type, - verbosity, - ..build_result - }) - }) + let joined_args = args.join(" "); + Ok(joined_args) } From fd4773af003d87fdb409b03b7c84dfae240ee3ad Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 13 Jun 2023 19:45:03 +0100 Subject: [PATCH 20/64] correct argument passing in dockerfile --- build-image/Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build-image/Dockerfile b/build-image/Dockerfile index 50243b135..b6dcb1d41 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -1,3 +1,5 @@ +FROM docker.io/bitnami/minideb:bullseye-amd64 as slimmed-rust + # The rust version to use # The 1.69 toolchain is temporarily required to build ink! contracts because of # https://github.com/paritytech/cargo-contract/issues/1139 @@ -5,11 +7,9 @@ ARG RUST_VERSION=1.69 # The cargo contract version to use ARG CARGO_CONTRACT_VERSION=3.0.1 -FROM docker.io/bitnami/minideb:bullseye-amd64 as slimmed-rust - # metadata LABEL io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="contracts-verifiable" \ + io.parity.image.title="paritytech/contracts-verifiable" \ io.parity.image.description="Inherits from docker.io/bitnami/minideb:bullseye. \ rust nightly, clippy, rustfmt, miri, rust-src, rustc-dev, grcov, rust-covfix, \ llvm-tools-preview, cargo-contract, xargo, binaryen, parallel, codecov, ink, solang" \ @@ -39,6 +39,7 @@ RUN set -eux \ && rm -rf /var/lib/apt/lists/* FROM slimmed-rust as cc-builder +ARG CARGO_CONTRACT_VERSION # This is important, see https://github.com/rust-lang/docker-rust/issues/85 ENV RUSTFLAGS="-C target-feature=-crt-static" From b2eb769b481f3a212a342c2238925aca6b4051e6 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 13 Jun 2023 19:52:15 +0100 Subject: [PATCH 21/64] refactoring --- crates/build/src/docker.rs | 24 ++++++++++-------------- crates/metadata/src/lib.rs | 2 +- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index e3ff4c7ac..bc84e02e2 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -232,7 +232,7 @@ async fn run_build( // in order to optimise the container usage // we are hashing the inputted command - // in order to reuse containers for different permutations of arguments + // in order to reuse the container for the same permutation of arguments let mut s = DefaultHasher::new(); entrypoint.hash(&mut s); let digest = s.finish(); @@ -252,7 +252,7 @@ async fn run_build( })) .await?; - let container = containers.first(); + let container_option = containers.first(); let mount = Mount { target: Some(String::from("/contract")), @@ -287,18 +287,14 @@ async fn run_build( platform: Some("linux/amd64"), }); - let container_id = if container.is_none() { - client - .create_container(options.clone(), config.clone()) - .await? - .id - } else { - container - .unwrap() - .id - .clone() - .context("Container does not have an ID")? - .to_owned() + let container_id = match container_option { + Some(container) => { + container + .id + .clone() + .context("Container does not have an ID")? + } + None => client.create_container(options, config).await?.id, }; client diff --git a/crates/metadata/src/lib.rs b/crates/metadata/src/lib.rs index 1cb5b6a07..4d7105be4 100644 --- a/crates/metadata/src/lib.rs +++ b/crates/metadata/src/lib.rs @@ -749,7 +749,7 @@ mod tests { let metadata = ContractMetadata::new( source, contract, - Some(String::from("parity/ver-build:latest")), + Some(String::from("paritytech/contracts-verified:3.0.1")), Some(user), abi_json, ); From 23bd332c2adca0151f9d01a8e5bef15b29297467 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 13 Jun 2023 19:58:04 +0100 Subject: [PATCH 22/64] more fixes --- crates/metadata/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/metadata/src/lib.rs b/crates/metadata/src/lib.rs index 4d7105be4..02223d351 100644 --- a/crates/metadata/src/lib.rs +++ b/crates/metadata/src/lib.rs @@ -749,7 +749,7 @@ mod tests { let metadata = ContractMetadata::new( source, contract, - Some(String::from("paritytech/contracts-verified:3.0.1")), + Some(String::from("paritytech/contracts-verifiable:3.0.1")), Some(user), abi_json, ); @@ -768,7 +768,7 @@ mod tests { "example_name": "increment" } }, - "image": "parity/ver-build:latest", + "image": "paritytech/contracts-verifiable:3.0.1", "contract": { "name": "incrementer", "version": "2.1.0", From c706c83f17077b60262bcb5040cdf3288f3ab4e9 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 13 Jun 2023 22:38:45 +0100 Subject: [PATCH 23/64] using remote registry + bell & whistles (progress bars) --- Cargo.lock | 45 +++++++ crates/build/Cargo.toml | 1 + crates/build/src/docker.rs | 252 ++++++++++++++++++++++++++++--------- 3 files changed, 237 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc88384b7..f1f623a05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -705,6 +705,19 @@ dependencies = [ "winapi", ] +[[package]] +name = "console" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.45.0", +] + [[package]] name = "constant_time_eq" version = "0.2.5" @@ -726,6 +739,7 @@ dependencies = [ "heck", "hex", "impl-serde", + "indicatif", "parity-scale-codec", "parity-wasm", "pretty_assertions", @@ -1163,6 +1177,12 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "env_logger" version = "0.10.0" @@ -1772,6 +1792,19 @@ dependencies = [ "serde", ] +[[package]] +name = "indicatif" +version = "0.17.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ff8cc23a7393a397ed1d7f56e6365cba772aba9f9912ab968b03043c395d057" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "unicode-width", +] + [[package]] name = "ink" version = "4.2.0" @@ -2537,6 +2570,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.29.0" @@ -2745,6 +2784,12 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8d0eef3571242013a0d5dc84861c3ae4a652e56e12adf8bdc26ff5f8cb34c94" +[[package]] +name = "portable-atomic" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "767eb9f07d4a5ebcb39bbf2d452058a93c011373abf6832e24194a1c3f004794" + [[package]] name = "ppv-lite86" version = "0.2.17" diff --git a/crates/build/Cargo.toml b/crates/build/Cargo.toml index 43dfe13e0..ee1c652e2 100644 --- a/crates/build/Cargo.toml +++ b/crates/build/Cargo.toml @@ -41,6 +41,7 @@ strum = { version = "0.24", features = ["derive"] } tokio = { version = "1", features = ["macros", "rt-multi-thread"] } tokio-stream = "0.1" bollard = "0.14" +indicatif = "0.17" contract-metadata = { version = "3.0.1", path = "../metadata" } diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index bc84e02e2..6e3117df2 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -12,6 +12,7 @@ use std::{ Path, PathBuf, }, + time::Duration, }; use anyhow::{ @@ -26,7 +27,10 @@ use bollard::{ StopContainerOptions, WaitContainerOptions, }, - image::ListImagesOptions, + image::{ + CreateImageOptions, + ListImagesOptions, + }, service::{ HostConfig, ImageSummary, @@ -36,6 +40,10 @@ use bollard::{ Docker, }; use contract_metadata::ContractMetadata; +use indicatif::{ + ProgressBar, + ProgressStyle, +}; use tokio_stream::StreamExt; use crate::{ @@ -55,6 +63,8 @@ use crate::{ use colored::Colorize; +const IMAGE: &str = "paritytech/contracts-verifiable:latest"; + #[derive(Clone, Debug, Default)] pub enum ImageVariant { #[default] @@ -76,70 +86,67 @@ pub fn docker_build(args: ExecuteArgs) -> Result { target, max_memory_pages, build_artifact, + image, .. } = args; - tokio::runtime::Runtime::new()?.block_on(async { - let mut build_steps = BuildSteps::new(); - - build_steps.set_total_steps(3); - if build_artifact == BuildArtifacts::CodeOnly { - build_steps.set_total_steps(2); - } - - let crate_metadata = CrateMetadata::collect(&manifest_path, target)?; - let host_folder = crate_metadata - .manifest_path - .absolute_directory()? - .as_path() - .to_owned(); - let file_path = host_folder.join(Path::new("target/build_result.json")); - let args = compose_build_args( - &mut features, - keep_debug_symbols, - optimization_passes.as_ref(), - &network, - &unstable_flags, - &build_artifact, - max_memory_pages, - )?; + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()? + .block_on(async { + let mut build_steps = BuildSteps::new(); + + build_steps.set_total_steps(3); + if build_artifact == BuildArtifacts::CodeOnly { + build_steps.set_total_steps(2); + } - // TODO: replace with image pulling, once the image is pushed to registry - let client = Docker::connect_with_socket_defaults()?; - let images = client - .list_images(Some(ListImagesOptions:: { - all: true, - ..Default::default() - })) + let crate_metadata = CrateMetadata::collect(&manifest_path, target)?; + let host_folder = crate_metadata + .manifest_path + .absolute_directory()? + .as_path() + .to_owned(); + let file_path = host_folder.join(Path::new("target/build_result.json")); + let args = compose_build_args( + &mut features, + keep_debug_symbols, + optimization_passes.as_ref(), + &network, + &unstable_flags, + &build_artifact, + max_memory_pages, + )?; + + let image_variant = match image { + Some(i) => i, + None => ImageVariant::Default, + }; + + let client = Docker::connect_with_socket_defaults()?; + let build_image = + get_image(client.clone(), image_variant, &verbosity, &mut build_steps) + .await?; + + run_build( + args, + &build_image, + &crate_metadata.contract_artifact_name, + &host_folder, + &verbosity, + &mut build_steps, + ) .await?; - let build_image = images.iter().find(|i| { - i.labels.get("io.parity.image.title") - == Some(&"contracts-verifiable".to_string()) - }); - if build_image.is_none() { - return Err(anyhow::anyhow!("No image found")) - } - let build_image = build_image.unwrap(); - - run_build( - args, - build_image, - &crate_metadata.contract_artifact_name, - &host_folder, - &verbosity, - &mut build_steps, - ) - .await?; - let build_result = read_build_result(&host_folder, &file_path)?; + let build_result = read_build_result(&host_folder, &file_path)?; - update_metadata(&build_result, &verbosity, &mut build_steps, build_image)?; + update_metadata(&build_result, &verbosity, &mut build_steps, &build_image)?; - Ok(BuildResult { - output_type, - verbosity, - ..build_result + Ok(BuildResult { + output_type, + verbosity, + ..build_result + }) }) - }) } /// Reads the `BuildResult` produced by the docker execution @@ -199,7 +206,18 @@ fn update_metadata( ) -> Result<()> { if let Some(metadata_artifacts) = &build_result.metadata_result { let mut metadata = ContractMetadata::load(&metadata_artifacts.dest_bundle)?; - metadata.image = Some(build_image.id.to_string()); + + // find alternative unique identifier of the image, otherwise grab the digest + let image_tag = match build_image + .repo_tags + .iter() + .find(|t| !t.ends_with("latest")) + { + Some(tag) => tag.to_owned(), + None => build_image.id.clone(), + }; + + metadata.image = Some(image_tag); crate::metadata::write_metadata( metadata_artifacts, @@ -301,14 +319,15 @@ async fn run_build( .start_container::(&container_id, None) .await?; + build_steps.increment_current(); maybe_println!( verbosity, - " {} {}\n{}", + " {} {}\n {}", format!("{build_steps}").bold(), "Started the build inside the container" .bright_green() .bold(), - "This might take a while. Check container logs for more details." + "You can close this terminal session. The execution will be finished in the background" ); let options = Some(WaitContainerOptions { @@ -316,7 +335,22 @@ async fn run_build( }); let mut wait_stream = client.wait_container(&container_id, options); - while wait_stream.next().await.is_some() {} + if verbosity.is_verbose() { + let spinner_style = + ProgressStyle::with_template(" {spinner:.cyan.bold} {wide_msg}")? + .tick_chars("⠁⠂⠄⡀⢀⠠⠐⠈ "); + + let pb = ProgressBar::new(1000); + pb.enable_steady_tick(Duration::from_millis(100)); + pb.set_style(spinner_style); + pb.set_message("Build is being executed..."); + while wait_stream.next().await.is_some() { + pb.inc(1); + } + pb.finish_with_message("Done!") + } else { + while wait_stream.next().await.is_some() {} + } client .stop_container(&container_id, Some(StopContainerOptions { t: 20 })) @@ -380,3 +414,99 @@ fn compose_build_args( let joined_args = args.join(" "); Ok(joined_args) } + +/// Retrieve local of the image, otherwise pulls one from the registry +async fn get_image( + client: Docker, + custom_image: ImageVariant, + verbosity: &Verbosity, + build_steps: &mut BuildSteps, +) -> Result { + // if no custom image is specified, then we use the latest tag + let image = match custom_image { + ImageVariant::Custom(i) => i.clone(), + ImageVariant::Default => IMAGE.to_owned(), + }; + + let build_image = match find_local_image(client.clone(), image.clone()).await? { + Some(image_s) => image_s, + None => { + build_steps.total_steps = build_steps.total_steps.map(|s| s + 1); + pull_image(client.clone(), image.clone(), verbosity, build_steps).await?; + find_local_image(client.clone(), image.clone()) + .await? + .context("Could not pull the image from the registry")? + } + }; + + Ok(build_image) +} + +async fn find_local_image(client: Docker, image: String) -> Result> { + let images = client + .list_images(Some(ListImagesOptions:: { + all: true, + ..Default::default() + })) + .await?; + let build_image = images.iter().find(|i| i.repo_tags.contains(&image)); + + Ok(build_image.cloned()) +} + +async fn pull_image( + client: Docker, + image: String, + verbosity: &Verbosity, + build_steps: &mut BuildSteps, +) -> Result<()> { + let mut pull_image_stream = client.create_image( + Some(CreateImageOptions { + from_image: image, + ..Default::default() + }), + None, + None, + ); + + maybe_println!( + verbosity, + " {} {}", + format!("{build_steps}").bold(), + "Image does not exist. Pulling one from the registry" + .bright_green() + .bold() + ); + + if verbosity.is_verbose() { + let spinner_style = ProgressStyle::with_template( + " {spinner:.cyan} [{wide_bar:.cyan/blue}]\n {wide_msg}", + )? + .progress_chars("#>-") + .tick_chars("⠁⠂⠄⡀⢀⠠⠐⠈ "); + let pb = ProgressBar::new(1000); + pb.set_style(spinner_style); + pb.enable_steady_tick(Duration::from_millis(100)); + + while let Some(summary_result) = pull_image_stream.next().await { + let summary = summary_result?; + + if let Some(progress_detail) = summary.progress_detail { + let total = progress_detail.total.map_or(1000, |v| v) as u64; + let current_step = progress_detail.current.map_or(1000, |v| v) as u64; + pb.set_length(total); + pb.set_position(current_step); + + if let Some(msg) = summary.status { + pb.set_message(msg); + } + } + } + + pb.finish(); + } else { + while pull_image_stream.next().await.is_some() {} + } + + Ok(()) +} From 4c69b283bd002080123c2b3e07cd45b3e793f719 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 13 Jun 2023 22:56:13 +0100 Subject: [PATCH 24/64] changelog entry --- CHANGELOG.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e9d187ef..bce01d133 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,13 +6,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Standardised verifiable builds - [#1148](https://github.com/paritytech/cargo-contract/pull/1148) + ### Changed -- Dry-run result output improvements [1123](https://github.com/paritytech/cargo-contract/pull/1123) +- Dry-run result output improvements - [1123](https://github.com/paritytech/cargo-contract/pull/1123) ## [3.0.1] ### Fixed -- `[contract-build]` flush the remaining buffered bytes [1118](https://github.com/paritytech/cargo-contract/pull/1118) +- `[contract-build]` flush the remaining buffered bytes - [1118](https://github.com/paritytech/cargo-contract/pull/1118) ## [3.0.0] @@ -23,7 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Contracts are build as `bin` crate now (we used `cdylib` before) - [#1076](https://github.com/paritytech/cargo-contract/pull/1076) - BREAKING CHANGE: Make sure that your contract is `no_main` by having this on top of your contract: - `#![cfg_attr(not(feature = "std"), no_std, no_main)]` - - This will be detected and suggested for `error[E0601]` [#1113](https://github.com/paritytech/cargo-contract/pull/1113) + - This will be detected and suggested for `error[E0601]` - [#1113](https://github.com/paritytech/cargo-contract/pull/1113) - Update contracts node metadata (#1105) - Compatible with `substrate-contracts-node 0.25.0-a2b09462c7c` From 427f27b2caf2e128d160003f55b470de50fbd4d8 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 13 Jun 2023 23:23:20 +0100 Subject: [PATCH 25/64] fixes --- crates/build/src/docker.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index 6e3117df2..02ee10bda 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -63,7 +63,8 @@ use crate::{ use colored::Colorize; -const IMAGE: &str = "paritytech/contracts-verifiable:latest"; +const IMAGE: &str = "paritytech/contracts-verifiable"; +const VERSION: &str = env!("CARGO_PKG_VERSION"); #[derive(Clone, Debug, Default)] pub enum ImageVariant { @@ -319,7 +320,6 @@ async fn run_build( .start_container::(&container_id, None) .await?; - build_steps.increment_current(); maybe_println!( verbosity, " {} {}\n {}", @@ -422,10 +422,13 @@ async fn get_image( verbosity: &Verbosity, build_steps: &mut BuildSteps, ) -> Result { - // if no custom image is specified, then we use the latest tag + // if no custom image is specified, then we use the tag of the current version of + // `cargo-contract` let image = match custom_image { ImageVariant::Custom(i) => i.clone(), - ImageVariant::Default => IMAGE.to_owned(), + ImageVariant::Default => { + format!("{}:{}", IMAGE, VERSION) + } }; let build_image = match find_local_image(client.clone(), image.clone()).await? { @@ -477,6 +480,7 @@ async fn pull_image( .bright_green() .bold() ); + build_steps.increment_current(); if verbosity.is_verbose() { let spinner_style = ProgressStyle::with_template( From 000d302654f9673481aae06fc48790f1fe9052c7 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Wed, 14 Jun 2023 16:34:02 +0100 Subject: [PATCH 26/64] refactoring --- crates/build/src/docker.rs | 5 ++ crates/cargo-contract/src/cmd/build.rs | 2 +- .../src/cmd/extrinsics/error.rs | 6 ++ .../src/cmd/extrinsics/instantiate.rs | 75 +++++++++---------- .../src/cmd/extrinsics/remove.rs | 57 +++++++------- .../src/cmd/extrinsics/upload.rs | 6 +- crates/cargo-contract/src/cmd/info.rs | 63 +++++++--------- 7 files changed, 102 insertions(+), 112 deletions(-) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index 02ee10bda..da4ad9dab 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -199,6 +199,7 @@ fn read_build_result(host_folder: &Path, file_path: &PathBuf) -> Result Result> { let images = client .list_images(Some(ListImagesOptions:: { @@ -457,6 +461,7 @@ async fn find_local_image(client: Docker, image: String) -> Result for ErrorVariant { } } +impl From for ErrorVariant { + fn from(value: std::io::Error) -> Self { + Self::Generic(GenericError::from_message(value.to_string())) + } +} + #[derive(serde::Serialize)] pub struct ModuleError { pub pallet: String, diff --git a/crates/cargo-contract/src/cmd/extrinsics/instantiate.rs b/crates/cargo-contract/src/cmd/extrinsics/instantiate.rs index 4a500f6a7..6e7fb60f6 100644 --- a/crates/cargo-contract/src/cmd/extrinsics/instantiate.rs +++ b/crates/cargo-contract/src/cmd/extrinsics/instantiate.rs @@ -16,7 +16,6 @@ use super::{ display_contract_exec_result, - error::GenericError, prompt_confirm_tx, state_call, submit_extrinsic, @@ -129,45 +128,41 @@ impl InstantiateCommand { }; let salt = self.salt.clone().map(|s| s.0).unwrap_or_default(); - Runtime::new() - .map_err(|e| { - ErrorVariant::Generic(GenericError::from_message(e.to_string())) - })? - .block_on(async { - let client = OnlineClient::from_url(url.clone()).await?; - - let token_metadata = TokenMetadata::query(&client).await?; - - let args = InstantiateArgs { - constructor: self.constructor.clone(), - raw_args: self.args.clone(), - value: self.value.denominate_balance(&token_metadata)?, - gas_limit: self.gas_limit, - proof_size: self.proof_size, - storage_deposit_limit: self - .extrinsic_opts - .storage_deposit_limit - .as_ref() - .map(|bv| bv.denominate_balance(&token_metadata)) - .transpose()?, - code, - data, - salt, - }; - - let exec = Exec { - args, - opts: self.extrinsic_opts.clone(), - url, - client, - verbosity, - signer, - transcoder, - output_json: self.output_json, - }; - - exec.exec(self.extrinsic_opts.execute).await - }) + Runtime::new()?.block_on(async { + let client = OnlineClient::from_url(url.clone()).await?; + + let token_metadata = TokenMetadata::query(&client).await?; + + let args = InstantiateArgs { + constructor: self.constructor.clone(), + raw_args: self.args.clone(), + value: self.value.denominate_balance(&token_metadata)?, + gas_limit: self.gas_limit, + proof_size: self.proof_size, + storage_deposit_limit: self + .extrinsic_opts + .storage_deposit_limit + .as_ref() + .map(|bv| bv.denominate_balance(&token_metadata)) + .transpose()?, + code, + data, + salt, + }; + + let exec = Exec { + args, + opts: self.extrinsic_opts.clone(), + url, + client, + verbosity, + signer, + transcoder, + output_json: self.output_json, + }; + + exec.exec(self.extrinsic_opts.execute).await + }) } } diff --git a/crates/cargo-contract/src/cmd/extrinsics/remove.rs b/crates/cargo-contract/src/cmd/extrinsics/remove.rs index 93cd3c1df..50c981cdd 100644 --- a/crates/cargo-contract/src/cmd/extrinsics/remove.rs +++ b/crates/cargo-contract/src/cmd/extrinsics/remove.rs @@ -15,7 +15,6 @@ // along with cargo-contract. If not, see . use super::{ - error::GenericError, submit_extrinsic, Client, ContractMessageTranscoder, @@ -89,39 +88,35 @@ impl RemoveCommand { } }?; - Runtime::new() - .map_err(|e| { - ErrorVariant::Generic(GenericError::from_message(e.to_string())) - })? - .block_on(async { - let url = self.extrinsic_opts.url_to_string(); - let client = OnlineClient::from_url(url.clone()).await?; - if let Some(code_removed) = self - .remove_code( - &client, - sp_core::H256(final_code_hash), - &signer, - &transcoder, - ) - .await? - { - let remove_result = code_removed.code_hash; + Runtime::new()?.block_on(async { + let url = self.extrinsic_opts.url_to_string(); + let client = OnlineClient::from_url(url.clone()).await?; + if let Some(code_removed) = self + .remove_code( + &client, + sp_core::H256(final_code_hash), + &signer, + &transcoder, + ) + .await? + { + let remove_result = code_removed.code_hash; - if self.output_json { - println!("{}", &remove_result); - } else { - name_value_println!("Code hash", format!("{remove_result:?}")); - } - Result::<(), ErrorVariant>::Ok(()) + if self.output_json { + println!("{}", &remove_result); } else { - let error_code_hash = hex::encode(final_code_hash); - Err(anyhow::anyhow!( - "Error removing the code for the supplied code hash: {}", - error_code_hash - ) - .into()) + name_value_println!("Code hash", format!("{remove_result:?}")); } - }) + Result::<(), ErrorVariant>::Ok(()) + } else { + let error_code_hash = hex::encode(final_code_hash); + Err(anyhow::anyhow!( + "Error removing the code for the supplied code hash: {}", + error_code_hash + ) + .into()) + } + }) } async fn remove_code( diff --git a/crates/cargo-contract/src/cmd/extrinsics/upload.rs b/crates/cargo-contract/src/cmd/extrinsics/upload.rs index 6ef8436d2..9df2efb38 100644 --- a/crates/cargo-contract/src/cmd/extrinsics/upload.rs +++ b/crates/cargo-contract/src/cmd/extrinsics/upload.rs @@ -16,7 +16,6 @@ use super::{ display_dry_run_result_warning, - error::GenericError, state_call, submit_extrinsic, Client, @@ -79,10 +78,7 @@ impl UploadCommand { })?; let code_hash = code.code_hash(); - Runtime::new() - .map_err(|e| - ErrorVariant::Generic(GenericError::from_message(e.to_string())) - )? + Runtime::new()? .block_on(async { let url = self.extrinsic_opts.url_to_string(); let client = OnlineClient::from_url(url.clone()).await?; diff --git a/crates/cargo-contract/src/cmd/info.rs b/crates/cargo-contract/src/cmd/info.rs index 60aab54e0..bf1523376 100644 --- a/crates/cargo-contract/src/cmd/info.rs +++ b/crates/cargo-contract/src/cmd/info.rs @@ -21,10 +21,7 @@ use super::{ }; use crate::{ cmd::{ - extrinsics::{ - GenericError, - MAX_KEY_COL_WIDTH, - }, + extrinsics::MAX_KEY_COL_WIDTH, runtime_api::api::runtime_types::pallet_contracts::storage::ContractInfo, Balance, CodeHash, @@ -69,41 +66,37 @@ impl InfoCommand { self.contract ); - Runtime::new() - .map_err(|e| { - ErrorVariant::Generic(GenericError::from_message(e.to_string())) - })? - .block_on(async { - let url = self.url.clone(); - let client = OnlineClient::::from_url(url).await?; + Runtime::new()?.block_on(async { + let url = self.url.clone(); + let client = OnlineClient::::from_url(url).await?; - let info_result = self.fetch_contract_info(&client).await?; + let info_result = self.fetch_contract_info(&client).await?; - match info_result { - Some(info_result) => { - let convert_trie_id = hex::encode(info_result.trie_id.0); - let info_to_json = InfoToJson { - trie_id: convert_trie_id, - code_hash: info_result.code_hash, - storage_items: info_result.storage_items, - storage_item_deposit: info_result.storage_item_deposit, - }; - if self.output_json { - println!("{}", info_to_json.to_json()?); - } else { - info_to_json.basic_display_format_contract_info(); - } - Ok(()) - } - None => { - Err(anyhow!( - "No contract information was found for account id {}", - self.contract - ) - .into()) + match info_result { + Some(info_result) => { + let convert_trie_id = hex::encode(info_result.trie_id.0); + let info_to_json = InfoToJson { + trie_id: convert_trie_id, + code_hash: info_result.code_hash, + storage_items: info_result.storage_items, + storage_item_deposit: info_result.storage_item_deposit, + }; + if self.output_json { + println!("{}", info_to_json.to_json()?); + } else { + info_to_json.basic_display_format_contract_info(); } + Ok(()) + } + None => { + Err(anyhow!( + "No contract information was found for account id {}", + self.contract + ) + .into()) } - }) + } + }) } async fn fetch_contract_info(&self, client: &Client) -> Result> { From a8db873df75ee73735af2c94d7e23b6411478a3d Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Thu, 15 Jun 2023 16:26:59 +0100 Subject: [PATCH 27/64] explicitly specify versions of apt packages --- build-image/Dockerfile | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/build-image/Dockerfile b/build-image/Dockerfile index b6dcb1d41..fe2dc7c3a 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -6,6 +6,12 @@ FROM docker.io/bitnami/minideb:bullseye-amd64 as slimmed-rust ARG RUST_VERSION=1.69 # The cargo contract version to use ARG CARGO_CONTRACT_VERSION=3.0.1 +# gcc package version +ARG GCC_VERSION=4:10.2.1-1 +# wget package version +ARG WGET_VERSION=1.21-1+deb11u1 +# g++ package version +ARG G_VERSION=4:10.2.1-1 # metadata LABEL io.parity.image.vendor="Parity Technologies" \ @@ -16,7 +22,10 @@ LABEL io.parity.image.vendor="Parity Technologies" \ io.parity.image.documentation="https://github.com/paritytech/cargo-contract/blob/master/\ build-image/README.md" \ io.parity.version.rust=${RUST_VERSION} \ - io.parity.version.cargo-contract=${CARGO_CONTRACT_VERSION} + io.parity.version.cargo-contract=${CARGO_CONTRACT_VERSION} \ + io.parity.version.gcc=${GCC_VERSION} \ + io.parity.version.wget=${WGET_VERSION} \ + io.parity.version.g_plus_plus=${G_VERSION} ENV RUSTUP_HOME=/usr/local/rustup \ CARGO_HOME=/usr/local/cargo \ @@ -24,7 +33,7 @@ ENV RUSTUP_HOME=/usr/local/rustup \ # Minimal Rust dependencies. RUN set -eux \ - && apt-get update && apt-get -y install wget \ + && apt-get update && apt-get -y install wget=${WGET_VERSION} \ && url="https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init" \ && wget "$url" \ && chmod +x rustup-init \ @@ -35,17 +44,19 @@ RUN set -eux \ && cargo --version \ && rustc --version \ && apt-get remove -y --auto-remove wget \ - && apt-get -y install gcc \ + && apt-get -y install gcc=${GCC_VERSION} \ && rm -rf /var/lib/apt/lists/* FROM slimmed-rust as cc-builder ARG CARGO_CONTRACT_VERSION +ARG GCC_VERSION +ARG G_VERSION # This is important, see https://github.com/rust-lang/docker-rust/issues/85 ENV RUSTFLAGS="-C target-feature=-crt-static" # Install required packages for `cargo-contract` -RUN apt-get -y update && apt-get -y install gcc g++ \ +RUN apt-get -y update && apt-get -y install gcc=${GCC_VERSION} g++=${G_VERSION} \ # Install cargo contract && cargo install cargo-contract --locked --version $CARGO_CONTRACT_VERSION \ # Cleanup after `cargo install` From 8264529c5d59771484b119cb4c0e83d34b79c704 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 20 Jun 2023 14:40:44 +0100 Subject: [PATCH 28/64] refactoring --- crates/build/src/args.rs | 5 +- crates/build/src/docker.rs | 91 +++++++++----------------- crates/cargo-contract/src/cmd/build.rs | 25 ++++--- 3 files changed, 50 insertions(+), 71 deletions(-) diff --git a/crates/build/src/args.rs b/crates/build/src/args.rs index f238302a4..869476af5 100644 --- a/crates/build/src/args.rs +++ b/crates/build/src/args.rs @@ -231,8 +231,10 @@ pub enum BuildMode { /// Functionality to output debug messages is build into the contract. #[default] Debug, - /// The contract is build without any debugging functionality. + /// The contract is built without any debugging functionality. Release, + /// the contract is built in release mode and in a deterministic environment. + Verifiable, } impl fmt::Display for BuildMode { @@ -240,6 +242,7 @@ impl fmt::Display for BuildMode { match self { Self::Debug => write!(f, "debug"), Self::Release => write!(f, "release"), + Self::Verifiable => write!(f, "verifiable"), } } } diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index da4ad9dab..0d4206908 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -1,3 +1,19 @@ +// Copyright 2018-2023 Parity Technologies (UK) Ltd. +// This file is part of cargo-contract. +// +// cargo-contract is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// cargo-contract is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with cargo-contract. If not, see . + use std::{ collections::{ hash_map::DefaultHasher, @@ -53,17 +69,13 @@ use crate::{ BuildSteps, CrateMetadata, ExecuteArgs, - Features, - Network, - OptimizationPasses, - UnstableFlags, Verbosity, - DEFAULT_MAX_MEMORY_PAGES, }; use colored::Colorize; const IMAGE: &str = "paritytech/contracts-verifiable"; +// We assume the docker image contains the same tag as the current version of the crate const VERSION: &str = env!("CARGO_PKG_VERSION"); #[derive(Clone, Debug, Default)] @@ -73,19 +85,13 @@ pub enum ImageVariant { Custom(String), } -/// Launched the docker container to execute verifiable build +/// Launches the docker container to execute verifiable build pub fn docker_build(args: ExecuteArgs) -> Result { let ExecuteArgs { manifest_path, verbosity, - mut features, - network, - unstable_flags, - optimization_passes, - keep_debug_symbols, output_type, target, - max_memory_pages, build_artifact, image, .. @@ -108,15 +114,7 @@ pub fn docker_build(args: ExecuteArgs) -> Result { .as_path() .to_owned(); let file_path = host_folder.join(Path::new("target/build_result.json")); - let args = compose_build_args( - &mut features, - keep_debug_symbols, - optimization_passes.as_ref(), - &network, - &unstable_flags, - &build_artifact, - max_memory_pages, - )?; + let args = compose_build_args()?; let image_variant = match image { Some(i) => i, @@ -371,48 +369,21 @@ async fn run_build( } /// Takes CLI args from the host and appends them to the build command inside the docker -fn compose_build_args( - features: &mut Features, - keep_debug_symbols: bool, - optimization_passes: Option<&OptimizationPasses>, - network: &Network, - unstable_flags: &UnstableFlags, - build_artifact: &BuildArtifacts, - max_memory_pages: u32, -) -> Result { +fn compose_build_args() -> Result { let mut args: Vec = vec!["--release".to_string()]; - features.append_to_args(&mut args); - if keep_debug_symbols { - args.push("--keep-debug-symbols".to_owned()); - } - - if let Some(passes) = optimization_passes { - if passes != &OptimizationPasses::default() { - args.push(format!("--optimization-passes {}", passes)); - } - } - - if network == &Network::Offline { - args.push("--offline".to_owned()); - } - if unstable_flags.original_manifest { - args.push("-Z original-manifest".to_owned()); - } - - let s = match build_artifact { - BuildArtifacts::CodeOnly => "--generate code-only".to_owned(), - BuildArtifacts::All => String::new(), - BuildArtifacts::CheckOnly => { - anyhow::bail!("--generate check-only is invalid flag for this command!"); - } - }; - - args.push(s); + let mut os_args: Vec = std::env::args() + .filter(|a| { + a != "--verifiable" + && a != "--image" + && !a.contains("cargo-contract") + && a != "cargo" + && a != "contract" + && a != "build" + }) + .collect(); - if max_memory_pages != DEFAULT_MAX_MEMORY_PAGES { - args.push(format!("--max-memory-pages {}", max_memory_pages)); - } + args.append(&mut os_args); let joined_args = args.join(" "); Ok(joined_args) diff --git a/crates/cargo-contract/src/cmd/build.rs b/crates/cargo-contract/src/cmd/build.rs index bbd00f4b7..2c8335c71 100644 --- a/crates/cargo-contract/src/cmd/build.rs +++ b/crates/cargo-contract/src/cmd/build.rs @@ -138,9 +138,13 @@ impl BuildCommand { TryFrom::<&UnstableOptions>::try_from(&self.unstable_options)?; let mut verbosity = TryFrom::<&VerbosityFlags>::try_from(&self.verbosity)?; - let build_mode = match self.build_release { - true => BuildMode::Release, - false => BuildMode::Debug, + let build_mode = if self.verifiable { + BuildMode::Verifiable + } else { + match self.build_release { + true => BuildMode::Release, + false => BuildMode::Debug, + } }; let network = match self.build_offline { @@ -153,13 +157,14 @@ impl BuildCommand { false => OutputType::HumanReadable, }; - let image = if self.verifiable { - match &self.image { - Some(i) => Some(ImageVariant::Custom(i.clone())), - None => Some(ImageVariant::Default), - } - } else { - None + if self.image.is_some() && build_mode == BuildMode::Verifiable { + anyhow::bail!("--image flag can only be used with verifiable builds!"); + } + + let image = match &self.image { + Some(i) => Some(ImageVariant::Custom(i.clone())), + None if build_mode == BuildMode::Verifiable => Some(ImageVariant::Default), + None => None, }; // We want to ensure that the only thing in `STDOUT` is our JSON formatted string. From a86dcce0e90149180e37f7f80cc117ce40f7bba2 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 20 Jun 2023 14:50:59 +0100 Subject: [PATCH 29/64] add module docs --- crates/build/src/docker.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index 0d4206908..042db0575 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -14,6 +14,26 @@ // You should have received a copy of the GNU General Public License // along with cargo-contract. If not, see . +//! This module provides a simple interface to execute the verifiable build +//! inside the docker container. +//! +//! For the correct behaviour, the docker engine must be running, +//! and the socket to be accessible. +//! +//! It is also important that the docker registry contains the tag +//! that matches the current version of this crate. +//! +//! The process of the build is following: +//! 1. Pull the image from the registry or use the local copy if available +//! 2. Parse other arguments that were passed to the host execution context +//! 3. Calculate the digest of the command and use it +//! to uniquely identify the container +//! 4. If the container exists, we just start the build, if not, we create it it +//! 5. After the build, the docker container produces metadata with +//! paths relative to its internal storage structure, we parse the file +//! and overwrite those paths relative to the host machine. +//! 6. Done! + use std::{ collections::{ hash_map::DefaultHasher, From 2c419eb6060d367e269824ec87c1d875254dbeb2 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 20 Jun 2023 18:08:04 +0100 Subject: [PATCH 30/64] refactoring --- crates/build/src/docker.rs | 16 +++------------- crates/build/src/lib.rs | 5 ++--- crates/cargo-contract/src/cmd/build.rs | 7 +++---- 3 files changed, 8 insertions(+), 20 deletions(-) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index 042db0575..d8df3e294 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -128,23 +128,13 @@ pub fn docker_build(args: ExecuteArgs) -> Result { } let crate_metadata = CrateMetadata::collect(&manifest_path, target)?; - let host_folder = crate_metadata - .manifest_path - .absolute_directory()? - .as_path() - .to_owned(); - let file_path = host_folder.join(Path::new("target/build_result.json")); + let host_folder = crate_metadata.manifest_path.absolute_directory()?; + let file_path = host_folder.join("target/build_result.json"); let args = compose_build_args()?; - let image_variant = match image { - Some(i) => i, - None => ImageVariant::Default, - }; - let client = Docker::connect_with_socket_defaults()?; let build_image = - get_image(client.clone(), image_variant, &verbosity, &mut build_steps) - .await?; + get_image(client.clone(), image, &verbosity, &mut build_steps).await?; run_build( args, diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index 3ce69799e..89acddfa3 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -124,7 +124,7 @@ pub struct ExecuteArgs { pub skip_wasm_validation: bool, pub target: Target, pub max_memory_pages: u32, - pub image: Option, + pub image: ImageVariant, } impl Default for ExecuteArgs { @@ -684,12 +684,11 @@ pub fn execute(args: ExecuteArgs) -> Result { lint, output_type, target, - image, .. } = &args; // if image exists, then --verifiable was called and we need to build inside docker. - if image.is_some() { + if build_mode == &BuildMode::Verifiable { return docker_build(args) } diff --git a/crates/cargo-contract/src/cmd/build.rs b/crates/cargo-contract/src/cmd/build.rs index 2c8335c71..4b124630b 100644 --- a/crates/cargo-contract/src/cmd/build.rs +++ b/crates/cargo-contract/src/cmd/build.rs @@ -162,9 +162,8 @@ impl BuildCommand { } let image = match &self.image { - Some(i) => Some(ImageVariant::Custom(i.clone())), - None if build_mode == BuildMode::Verifiable => Some(ImageVariant::Default), - None => None, + Some(i) => ImageVariant::Custom(i.clone()), + None => ImageVariant::Default, }; // We want to ensure that the only thing in `STDOUT` is our JSON formatted string. @@ -231,7 +230,7 @@ impl CheckCommand { skip_wasm_validation: false, target: self.target, max_memory_pages: 0, - image: None, + image: ImageVariant::Default, }; contract_build::execute(args) From 085a744a17bdf1c4b07358c3f4668b175854ed55 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 20 Jun 2023 18:39:54 +0100 Subject: [PATCH 31/64] doc fix and optionally use git in docker image --- build-image/Dockerfile | 13 ++++++++++++- crates/build/README.md | 3 ++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/build-image/Dockerfile b/build-image/Dockerfile index fe2dc7c3a..db55ec3ad 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -12,6 +12,10 @@ ARG GCC_VERSION=4:10.2.1-1 ARG WGET_VERSION=1.21-1+deb11u1 # g++ package version ARG G_VERSION=4:10.2.1-1 +# Flag to indicate whether to install cargo-contract from git +ARG USE_GIT=false +# Url to the cargo-contract repository to install from +ARG CARGO_CONTRACT_GIT # metadata LABEL io.parity.image.vendor="Parity Technologies" \ @@ -51,6 +55,9 @@ FROM slimmed-rust as cc-builder ARG CARGO_CONTRACT_VERSION ARG GCC_VERSION ARG G_VERSION +ARG USE_GIT +ARG CARGO_CONTRACT_GIT + # This is important, see https://github.com/rust-lang/docker-rust/issues/85 ENV RUSTFLAGS="-C target-feature=-crt-static" @@ -58,7 +65,11 @@ ENV RUSTFLAGS="-C target-feature=-crt-static" # Install required packages for `cargo-contract` RUN apt-get -y update && apt-get -y install gcc=${GCC_VERSION} g++=${G_VERSION} \ # Install cargo contract - && cargo install cargo-contract --locked --version $CARGO_CONTRACT_VERSION \ + && if [ ${USE_GIT} = true ]; then \ + cargo install --git ${CARGO_CONTRACT_GIT} ;\ + else \ + cargo install cargo-contract --locked --version ${CARGO_CONTRACT_VERSION} ;\ + fi \ # Cleanup after `cargo install` && rm -rf ${CARGO_HOME}/"registry" ${CARGO_HOME}/"git" /root/.cache/sccache \ # apt clean up diff --git a/crates/build/README.md b/crates/build/README.md index 490f479ce..b8feec1ad 100644 --- a/crates/build/README.md +++ b/crates/build/README.md @@ -17,6 +17,7 @@ use contract_build::{ OutputType, UnstableFlags, Target, + ImageVariant, }; let manifest_path = ManifestPath::new("my-contract/Cargo.toml").unwrap(); @@ -36,7 +37,7 @@ let args = contract_build::ExecuteArgs { skip_wasm_validation: false, target: Target::Wasm, max_memory_pages: 16, - image: None, + image: ImageVariant::Default, }; contract_build::execute(args); From 913d2a99c9c1982ac91f319a09d5ec90dda171be Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 20 Jun 2023 19:06:32 +0100 Subject: [PATCH 32/64] Error handling for builds --- crates/build/src/docker.rs | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index d8df3e294..c5a6f9ea8 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -63,11 +63,13 @@ use bollard::{ StopContainerOptions, WaitContainerOptions, }, + errors::Error, image::{ CreateImageOptions, ListImagesOptions, }, service::{ + ContainerWaitResponse, HostConfig, ImageSummary, Mount, @@ -139,6 +141,7 @@ pub fn docker_build(args: ExecuteArgs) -> Result { run_build( args, &build_image, + &file_path, &crate_metadata.contract_artifact_name, &host_folder, &verbosity, @@ -243,6 +246,7 @@ fn update_metadata( async fn run_build( build_args: String, build_image: &ImageSummary, + file_path: &PathBuf, contract_name: &str, host_folder: &Path, verbosity: &Verbosity, @@ -345,6 +349,22 @@ async fn run_build( }); let mut wait_stream = client.wait_container(&container_id, options); + let handle_error = |r: Result| -> Result<()> { + let response = match r { + Ok(v) => v, + Err(e) => { + std::fs::remove_file(file_path)?; + anyhow::bail!( + "{}. Execution failed! Check logs for more details", + e.to_string() + ) + } + }; + if response.status_code != 0 { + anyhow::bail!("Execution failed! Status code: {}. Check the container's logs for more details", response.status_code); + } + Ok(()) + }; if verbosity.is_verbose() { let spinner_style = ProgressStyle::with_template(" {spinner:.cyan.bold} {wide_msg}")? @@ -354,12 +374,15 @@ async fn run_build( pb.enable_steady_tick(Duration::from_millis(100)); pb.set_style(spinner_style); pb.set_message("Build is being executed..."); - while wait_stream.next().await.is_some() { + while let Some(r) = wait_stream.next().await { + handle_error(r)?; pb.inc(1); } pb.finish_with_message("Done!") } else { - while wait_stream.next().await.is_some() {} + while let Some(r) = wait_stream.next().await { + handle_error(r)?; + } } client From 1d84b3cae8255b9211f354a67e4f5d5f60b230b2 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Wed, 21 Jun 2023 14:57:45 +0100 Subject: [PATCH 33/64] use single var in docker and refactor docker module --- build-image/Dockerfile | 7 +++---- crates/build/src/docker.rs | 22 ++++++++++++++-------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/build-image/Dockerfile b/build-image/Dockerfile index db55ec3ad..e729bc920 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -12,8 +12,6 @@ ARG GCC_VERSION=4:10.2.1-1 ARG WGET_VERSION=1.21-1+deb11u1 # g++ package version ARG G_VERSION=4:10.2.1-1 -# Flag to indicate whether to install cargo-contract from git -ARG USE_GIT=false # Url to the cargo-contract repository to install from ARG CARGO_CONTRACT_GIT @@ -64,10 +62,11 @@ ENV RUSTFLAGS="-C target-feature=-crt-static" # Install required packages for `cargo-contract` RUN apt-get -y update && apt-get -y install gcc=${GCC_VERSION} g++=${G_VERSION} \ - # Install cargo contract - && if [ ${USE_GIT} = true ]; then \ + # Install cargo contract from git if the arg is set + && if [ -n "$CARGO_CONTRACT_GIT" ]; then \ cargo install --git ${CARGO_CONTRACT_GIT} ;\ else \ + # Otherwise install from crates.io cargo install cargo-contract --locked --version ${CARGO_CONTRACT_VERSION} ;\ fi \ # Cleanup after `cargo install` diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index c5a6f9ea8..c1e52948e 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -131,7 +131,8 @@ pub fn docker_build(args: ExecuteArgs) -> Result { let crate_metadata = CrateMetadata::collect(&manifest_path, target)?; let host_folder = crate_metadata.manifest_path.absolute_directory()?; - let file_path = host_folder.join("target/build_result.json"); + let target_dir = crate_metadata.target_directory; + let build_result_path = target_dir.join("build_result.json"); let args = compose_build_args()?; let client = Docker::connect_with_socket_defaults()?; @@ -141,7 +142,7 @@ pub fn docker_build(args: ExecuteArgs) -> Result { run_build( args, &build_image, - &file_path, + &build_result_path, &crate_metadata.contract_artifact_name, &host_folder, &verbosity, @@ -149,7 +150,7 @@ pub fn docker_build(args: ExecuteArgs) -> Result { ) .await?; - let build_result = read_build_result(&host_folder, &file_path)?; + let build_result = read_build_result(&host_folder, &build_result_path)?; update_metadata(&build_result, &verbosity, &mut build_steps, &build_image)?; @@ -162,13 +163,17 @@ pub fn docker_build(args: ExecuteArgs) -> Result { } /// Reads the `BuildResult` produced by the docker execution -fn read_build_result(host_folder: &Path, file_path: &PathBuf) -> Result { - let file = std::fs::File::open(file_path)?; +fn read_build_result( + host_folder: &Path, + build_result_path: &PathBuf, +) -> Result { + let file = std::fs::File::open(build_result_path)?; let mut build_result: BuildResult = match serde_json::from_reader(BufReader::new(file)) { Ok(result) => result, Err(_) => { - std::fs::remove_file(file_path)?; + // sometimes we cannot remove the file due to privileged access + let _ = std::fs::remove_file(build_result_path); anyhow::bail!( "Error parsing output from docker build. The build probably failed!" ) @@ -246,7 +251,7 @@ fn update_metadata( async fn run_build( build_args: String, build_image: &ImageSummary, - file_path: &PathBuf, + build_result_path: &PathBuf, contract_name: &str, host_folder: &Path, verbosity: &Verbosity, @@ -353,7 +358,8 @@ async fn run_build( let response = match r { Ok(v) => v, Err(e) => { - std::fs::remove_file(file_path)?; + // sometimes we cannot remove the file due to privileged access + let _ = std::fs::remove_file(build_result_path); anyhow::bail!( "{}. Execution failed! Check logs for more details", e.to_string() From d237d73df3680e8fe9eb69504a4a7e703b5381c0 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Wed, 21 Jun 2023 15:22:27 +0100 Subject: [PATCH 34/64] notes on apple silicon --- build-image/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build-image/README.md b/build-image/README.md index b788e2e4e..241da3394 100644 --- a/build-image/README.md +++ b/build-image/README.md @@ -43,3 +43,10 @@ my-app/ Make sure to run the command inside `my-app` directory and specify a relative manifest path to the root contract: `cargo contract build --verifiable --release --manifest-path ink-project-a/Cargo.toml` + + +**Apple Silicon performance** + +It is a known issue that running AMD64 image on the ARM host architecture significantly impacts the performance +and build times. To solve this issues, enable _Use Rosetta for x86/amd64 emulation on Apple Silicon_ in +_Settings_ -> _Features in development_ tab in Docker Desktop App. From d0e00dda8119908a5ad8d73c409b21f0d4209de1 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Wed, 21 Jun 2023 15:26:51 +0100 Subject: [PATCH 35/64] remove unused arg in dockerfile --- build-image/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/build-image/Dockerfile b/build-image/Dockerfile index e729bc920..1c08e7f5b 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -53,7 +53,6 @@ FROM slimmed-rust as cc-builder ARG CARGO_CONTRACT_VERSION ARG GCC_VERSION ARG G_VERSION -ARG USE_GIT ARG CARGO_CONTRACT_GIT From 2f3569812f4704a19a12f40b3b13dbdb0cbd992a Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Wed, 21 Jun 2023 16:31:12 +0100 Subject: [PATCH 36/64] use target path correctly --- crates/build/src/docker.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index c1e52948e..fcd9fc047 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -131,11 +131,16 @@ pub fn docker_build(args: ExecuteArgs) -> Result { let crate_metadata = CrateMetadata::collect(&manifest_path, target)?; let host_folder = crate_metadata.manifest_path.absolute_directory()?; - let target_dir = crate_metadata.target_directory; + let target_dir = crate_metadata.cargo_meta.target_directory.as_std_path(); let build_result_path = target_dir.join("build_result.json"); let args = compose_build_args()?; - let client = Docker::connect_with_socket_defaults()?; + let client = Docker::connect_with_socket_defaults().map_err(|e| { + anyhow::anyhow!("{}\nDo you have the docker engine installed in path?", e) + })?; + let _ = client.ping().await.map_err(|e| { + anyhow::anyhow!("{}\nIs your docker engine up and running?", e) + })?; let build_image = get_image(client.clone(), image, &verbosity, &mut build_steps).await?; @@ -224,7 +229,6 @@ fn update_metadata( ) -> Result<()> { if let Some(metadata_artifacts) = &build_result.metadata_result { let mut metadata = ContractMetadata::load(&metadata_artifacts.dest_bundle)?; - // find alternative unique identifier of the image, otherwise grab the digest let image_tag = match build_image .repo_tags From 3e157f4cc6edeca92aa5efdc5d2efe0b792edd39 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Mon, 26 Jun 2023 16:37:55 +0100 Subject: [PATCH 37/64] handle building with relative paths + permissions --- crates/build/src/docker.rs | 23 +++++++++++++--------- crates/build/src/lib.rs | 40 +++++++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index fcd9fc047..488f48cf3 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -130,9 +130,11 @@ pub fn docker_build(args: ExecuteArgs) -> Result { } let crate_metadata = CrateMetadata::collect(&manifest_path, target)?; - let host_folder = crate_metadata.manifest_path.absolute_directory()?; - let target_dir = crate_metadata.cargo_meta.target_directory.as_std_path(); - let build_result_path = target_dir.join("build_result.json"); + let mut manifest_dir_option = crate_metadata.manifest_path.directory(); + let empty_path = PathBuf::new(); + let contract_dir = manifest_dir_option.get_or_insert(&empty_path); + let host_dir = std::env::current_dir()?; + let build_result_path = contract_dir.join("target/build_result.json"); let args = compose_build_args()?; let client = Docker::connect_with_socket_defaults().map_err(|e| { @@ -149,13 +151,13 @@ pub fn docker_build(args: ExecuteArgs) -> Result { &build_image, &build_result_path, &crate_metadata.contract_artifact_name, - &host_folder, + &host_dir, &verbosity, &mut build_steps, ) .await?; - let build_result = read_build_result(&host_folder, &build_result_path)?; + let build_result = read_build_result(&host_dir, &build_result_path)?; update_metadata(&build_result, &verbosity, &mut build_steps, &build_image)?; @@ -264,11 +266,14 @@ async fn run_build( let client = Docker::connect_with_socket_defaults()?; let mut entrypoint = vec!["/bin/bash".to_string(), "-c".to_string()]; - + let b_path_str = build_result_path + .as_os_str() + .to_str() + .context("Cannot convert Os String to String")?; let mut cmds = vec![format!( - "mkdir -p target && cargo contract build {} --output-json > target/build_result.json", - build_args - )]; + "mkdir -p target && cargo contract build {} --output-json > {}", + build_args, b_path_str + )]; entrypoint.append(&mut cmds); diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index 89acddfa3..5ef571dd4 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -795,6 +795,13 @@ pub fn execute(args: ExecuteArgs) -> Result { } }; + let cargo_target_path = crate_metadata + .cargo_meta + .target_directory + .as_std_path() + .to_path_buf(); + change_permissions(&cargo_target_path)?; + Ok(BuildResult { dest_wasm, metadata_result, @@ -808,6 +815,37 @@ pub fn execute(args: ExecuteArgs) -> Result { }) } +/// Modifies permissions of all entries in the dir to be read_only = false +#[allow(clippy::permissions_set_readonly_false)] +fn change_permissions(p: &PathBuf) -> Result<()> { + let files_and_paths = fs::read_dir(p)?; + + for e in files_and_paths { + let entry = e?; + let entry_path = entry.path(); + let meta = entry.metadata()?; + let mut perms = meta.permissions(); + if cfg!(unix) { + use std::os::unix::fs::PermissionsExt; + // give r-w-x to the owner + // give r-w to the group + // give r to all other users + perms.set_mode(0o764); + } else { + perms.set_readonly(false); + } + fs::set_permissions(&entry_path, perms)?; + + // if it's a directory recursively change permissions in there too + if meta.is_dir() { + change_permissions(&entry_path)?; + } + } + + Ok(()) +} + +/// Build the contract on host locally fn local_build( crate_metadata: &CrateMetadata, optimization_passes: &OptimizationPasses, @@ -849,7 +887,7 @@ fn local_build( target, )?; - // we persist the latest target we used so we trigger a rebuild when we switch + // We persist the latest target we used so we trigger a rebuild when we switch fs::write(&crate_metadata.target_file_path, target.llvm_target())?; let cargo_contract_version = if let Ok(version) = Version::parse(VERSION) { From 98004028bea6e9fa249d5831b31be60a5ebfeb60 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Mon, 26 Jun 2023 18:19:19 +0100 Subject: [PATCH 38/64] more flags in docker image + fixes --- Cargo.lock | 1 + build-image/Dockerfile | 41 +++++++++++++++++++++++--- crates/build/Cargo.toml | 1 + crates/build/src/docker.rs | 5 +++- crates/build/src/lib.rs | 3 +- crates/cargo-contract/src/cmd/build.rs | 2 +- 6 files changed, 46 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 75d17067d..8e6c84e95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -749,6 +749,7 @@ dependencies = [ "parity-scale-codec", "parity-wasm", "pretty_assertions", + "regex", "rustc_version", "semver", "serde", diff --git a/build-image/Dockerfile b/build-image/Dockerfile index 1c08e7f5b..ba2db9388 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -14,6 +14,12 @@ ARG WGET_VERSION=1.21-1+deb11u1 ARG G_VERSION=4:10.2.1-1 # Url to the cargo-contract repository to install from ARG CARGO_CONTRACT_GIT +# Branch to use in git repository +ARG CARGO_CONTRACT_BRANCH +# Commit to use in git repository +ARG CARGO_CONTRACT_REV +# Tag to use in git repository +ARG CARGO_CONTRACT_TAG # metadata LABEL io.parity.image.vendor="Parity Technologies" \ @@ -53,21 +59,48 @@ FROM slimmed-rust as cc-builder ARG CARGO_CONTRACT_VERSION ARG GCC_VERSION ARG G_VERSION +ARG CARGO_CONTRACT_BRANCH +ARG CARGO_CONTRACT_TAG +ARG CARGO_CONTRACT_REV ARG CARGO_CONTRACT_GIT # This is important, see https://github.com/rust-lang/docker-rust/issues/85 ENV RUSTFLAGS="-C target-feature=-crt-static" - # Install required packages for `cargo-contract` +# Install required packages for `cargo-contract` RUN apt-get -y update && apt-get -y install gcc=${GCC_VERSION} g++=${G_VERSION} \ # Install cargo contract from git if the arg is set + # If branch is specified use it + + # && if [ -n "$CARGO_CONTRACT_BRANCH" ] && [ -n "$CARGO_CONTRACT_GIT" ]; then \ + # echo "cargo-contract git: ${CARGO_CONTRACT_GIT} and branch: ${CARGO_CONTRACT_BRANCH}" && \ + # cargo install --git ${CARGO_CONTRACT_GIT} --branch ${CARGO_CONTRACT_BRANCH} ; \ + # # Else if tag is specified use it + # elif [ -n "$CARGO_CONTRACT_TAG" ] && [ -n "$CARGO_CONTRACT_GIT" ]; then \ + # echo "cargo-contract git: ${CARGO_CONTRACT_GIT} and tag: ${CARGO_CONTRACT_TAG}" && \ + # cargo install --git ${CARGO_CONTRACT_GIT} --tag ${CARGO_CONTRACT_TAG} ;\ + # # Else if commit is specified use it + # elif [ -n "$CARGO_CONTRACT_REV" ] && [ -n "$CARGO_CONTRACT_GIT" ]; then \ + # echo "cargo-contract git: ${CARGO_CONTRACT_GIT} and commit: ${CARGO_CONTRACT_REV}" && \ + # cargo install --git ${CARGO_CONTRACT_GIT} --rev ${CARGO_CONTRACT_REV} ;\ + # # Else if just git is specified, install from master && if [ -n "$CARGO_CONTRACT_GIT" ]; then \ - cargo install --git ${CARGO_CONTRACT_GIT} ;\ + COMMAND="cargo install --git ${CARGO_CONTRACT_GIT}" ; \ else \ - # Otherwise install from crates.io - cargo install cargo-contract --locked --version ${CARGO_CONTRACT_VERSION} ;\ + COMMAND="cargo install cargo-contract --locked --version ${CARGO_CONTRACT_VERSION}" ;\ + fi \ + && if [ -n "$CARGO_CONTRACT_BRANCH" ] && [ -n "$CARGO_CONTRACT_GIT" ]; then \ + COMMAND="${COMMAND} --branch ${CARGO_CONTRACT_BRANCH}" ; \ + fi \ + && if [ -n "$CARGO_CONTRACT_REV" ] && [ -n "$CARGO_CONTRACT_GIT" ]; then \ + COMMAND="${COMMAND} --rev ${CARGO_CONTRACT_REV}" ; \ + fi \ + && if [ -n "$CARGO_CONTRACT_TAG" ] && [ -n "$CARGO_CONTRACT_GIT" ]; then \ + COMMAND="cargo install --git ${CARGO_CONTRACT_GIT} --tag ${CARGO_CONTRACT_TAG}" ; \ fi \ + && echo "Executing ${COMMAND}" \ + && eval "${COMMAND}" \ # Cleanup after `cargo install` && rm -rf ${CARGO_HOME}/"registry" ${CARGO_HOME}/"git" /root/.cache/sccache \ # apt clean up diff --git a/crates/build/Cargo.toml b/crates/build/Cargo.toml index 2a0750f3d..4db9f99bc 100644 --- a/crates/build/Cargo.toml +++ b/crates/build/Cargo.toml @@ -14,6 +14,7 @@ keywords = ["wasm", "parity", "webassembly", "blockchain", "edsl"] include = ["Cargo.toml", "src/**/*.rs", "README.md", "LICENSE", "build.rs", "templates", "riscv_memory_layout.ld"] [dependencies] +regex = "1" anyhow = "1.0.71" blake2 = "0.10.6" cargo_metadata = "0.15.4" diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index 488f48cf3..dc76a8ba8 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -418,12 +418,15 @@ async fn run_build( /// Takes CLI args from the host and appends them to the build command inside the docker fn compose_build_args() -> Result { + use regex::Regex; let mut args: Vec = vec!["--release".to_string()]; + let rex = Regex::new(r"--image [.*]*")?; + let mut os_args: Vec = std::env::args() .filter(|a| { a != "--verifiable" - && a != "--image" + && !rex.is_match(&a) && !a.contains("cargo-contract") && a != "cargo" && a != "contract" diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index 5ef571dd4..38d9038ba 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -831,7 +831,8 @@ fn change_permissions(p: &PathBuf) -> Result<()> { // give r-w to the group // give r to all other users perms.set_mode(0o764); - } else { + } + if cfg!(not(unix)) { perms.set_readonly(false); } fs::set_permissions(&entry_path, perms)?; diff --git a/crates/cargo-contract/src/cmd/build.rs b/crates/cargo-contract/src/cmd/build.rs index 4b124630b..1dea0202f 100644 --- a/crates/cargo-contract/src/cmd/build.rs +++ b/crates/cargo-contract/src/cmd/build.rs @@ -157,7 +157,7 @@ impl BuildCommand { false => OutputType::HumanReadable, }; - if self.image.is_some() && build_mode == BuildMode::Verifiable { + if self.image.is_some() && build_mode != BuildMode::Verifiable { anyhow::bail!("--image flag can only be used with verifiable builds!"); } From 2fd00cf26e87f1d75fa177968ed86b0350a287be Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Mon, 26 Jun 2023 18:21:44 +0100 Subject: [PATCH 39/64] remove unused code --- build-image/Dockerfile | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/build-image/Dockerfile b/build-image/Dockerfile index ba2db9388..fc9af7065 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -71,20 +71,6 @@ ENV RUSTFLAGS="-C target-feature=-crt-static" # Install required packages for `cargo-contract` RUN apt-get -y update && apt-get -y install gcc=${GCC_VERSION} g++=${G_VERSION} \ # Install cargo contract from git if the arg is set - # If branch is specified use it - - # && if [ -n "$CARGO_CONTRACT_BRANCH" ] && [ -n "$CARGO_CONTRACT_GIT" ]; then \ - # echo "cargo-contract git: ${CARGO_CONTRACT_GIT} and branch: ${CARGO_CONTRACT_BRANCH}" && \ - # cargo install --git ${CARGO_CONTRACT_GIT} --branch ${CARGO_CONTRACT_BRANCH} ; \ - # # Else if tag is specified use it - # elif [ -n "$CARGO_CONTRACT_TAG" ] && [ -n "$CARGO_CONTRACT_GIT" ]; then \ - # echo "cargo-contract git: ${CARGO_CONTRACT_GIT} and tag: ${CARGO_CONTRACT_TAG}" && \ - # cargo install --git ${CARGO_CONTRACT_GIT} --tag ${CARGO_CONTRACT_TAG} ;\ - # # Else if commit is specified use it - # elif [ -n "$CARGO_CONTRACT_REV" ] && [ -n "$CARGO_CONTRACT_GIT" ]; then \ - # echo "cargo-contract git: ${CARGO_CONTRACT_GIT} and commit: ${CARGO_CONTRACT_REV}" && \ - # cargo install --git ${CARGO_CONTRACT_GIT} --rev ${CARGO_CONTRACT_REV} ;\ - # # Else if just git is specified, install from master && if [ -n "$CARGO_CONTRACT_GIT" ]; then \ COMMAND="cargo install --git ${CARGO_CONTRACT_GIT}" ; \ else \ From 02aa0af90aae807d25aaf33e8594844e6e42f326 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Mon, 26 Jun 2023 18:28:14 +0100 Subject: [PATCH 40/64] add git info to docker metadata --- build-image/Dockerfile | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/build-image/Dockerfile b/build-image/Dockerfile index fc9af7065..93da667f1 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -6,12 +6,6 @@ FROM docker.io/bitnami/minideb:bullseye-amd64 as slimmed-rust ARG RUST_VERSION=1.69 # The cargo contract version to use ARG CARGO_CONTRACT_VERSION=3.0.1 -# gcc package version -ARG GCC_VERSION=4:10.2.1-1 -# wget package version -ARG WGET_VERSION=1.21-1+deb11u1 -# g++ package version -ARG G_VERSION=4:10.2.1-1 # Url to the cargo-contract repository to install from ARG CARGO_CONTRACT_GIT # Branch to use in git repository @@ -20,6 +14,12 @@ ARG CARGO_CONTRACT_BRANCH ARG CARGO_CONTRACT_REV # Tag to use in git repository ARG CARGO_CONTRACT_TAG +# gcc package version +ARG GCC_VERSION=4:10.2.1-1 +# wget package version +ARG WGET_VERSION=1.21-1+deb11u1 +# g++ package version +ARG G_VERSION=4:10.2.1-1 # metadata LABEL io.parity.image.vendor="Parity Technologies" \ @@ -30,7 +30,11 @@ LABEL io.parity.image.vendor="Parity Technologies" \ io.parity.image.documentation="https://github.com/paritytech/cargo-contract/blob/master/\ build-image/README.md" \ io.parity.version.rust=${RUST_VERSION} \ - io.parity.version.cargo-contract=${CARGO_CONTRACT_VERSION} \ + io.parity.version.cargo-contract.version=${CARGO_CONTRACT_VERSION} \ + io.parity.version.cargo-contract.git.repository=${CARGO_CONTRACT_GIT} \ + io.parity.version.cargo-contract.git.branch=${CARGO_CONTRACT_BRANCH} \ + io.parity.version.cargo-contract.git.revision=${CARGO_CONTRACT_REV} \ + io.parity.version.cargo-contract.git.tag=${CARGO_CONTRACT_TAG} \ io.parity.version.gcc=${GCC_VERSION} \ io.parity.version.wget=${WGET_VERSION} \ io.parity.version.g_plus_plus=${G_VERSION} From fa95d79e27b7fcb638f9ae0ec8301b106ce4ee4c Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Mon, 26 Jun 2023 19:10:00 +0100 Subject: [PATCH 41/64] use env to detect running OS --- crates/build/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index 38d9038ba..2b487fc8a 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -825,14 +825,14 @@ fn change_permissions(p: &PathBuf) -> Result<()> { let entry_path = entry.path(); let meta = entry.metadata()?; let mut perms = meta.permissions(); - if cfg!(unix) { + if std::env::consts::OS == "macos" || std::env::consts::OS == "linux" { use std::os::unix::fs::PermissionsExt; // give r-w-x to the owner // give r-w to the group // give r to all other users perms.set_mode(0o764); } - if cfg!(not(unix)) { + if std::env::consts::OS == "windows" { perms.set_readonly(false); } fs::set_permissions(&entry_path, perms)?; From ea251da2e7f90644b5c9467c66f70274954c7aca Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Mon, 26 Jun 2023 20:22:17 +0100 Subject: [PATCH 42/64] detect host OS --- crates/build/build.rs | 8 ++++++++ crates/build/src/docker.rs | 15 +++++++++------ crates/build/src/lib.rs | 6 +++--- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/crates/build/build.rs b/crates/build/build.rs index 20610289d..c9a5d571c 100644 --- a/crates/build/build.rs +++ b/crates/build/build.rs @@ -48,6 +48,14 @@ fn main() { .into(); let res = zip_template(&manifest_dir, &out_dir); + // detect running host OS + if cfg!(windows) { + println!("cargo:rustc-cfg=windows_host"); + } + if cfg!(unix) { + println!("cargo:rustc-cfg=unix_host"); + } + match res { Ok(()) => std::process::exit(0), Err(err) => { diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index dc76a8ba8..45a20ca57 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -422,16 +422,19 @@ fn compose_build_args() -> Result { let mut args: Vec = vec!["--release".to_string()]; let rex = Regex::new(r"--image [.*]*")?; + let args_string: String = std::env::args().collect(); + let args_string = rex.replace_all(&args_string, "").to_string(); - let mut os_args: Vec = std::env::args() + let mut os_args: Vec = args_string + .split_ascii_whitespace() .filter(|a| { - a != "--verifiable" - && !rex.is_match(&a) + a != &"--verifiable" && !a.contains("cargo-contract") - && a != "cargo" - && a != "contract" - && a != "build" + && a != &"cargo" + && a != &"contract" + && a != &"build" }) + .map(|s| s.to_string()) .collect(); args.append(&mut os_args); diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index 2b487fc8a..606122b7a 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -825,14 +825,14 @@ fn change_permissions(p: &PathBuf) -> Result<()> { let entry_path = entry.path(); let meta = entry.metadata()?; let mut perms = meta.permissions(); - if std::env::consts::OS == "macos" || std::env::consts::OS == "linux" { + if cfg!(unix_host) { + println!("unix system is running"); use std::os::unix::fs::PermissionsExt; // give r-w-x to the owner // give r-w to the group // give r to all other users perms.set_mode(0o764); - } - if std::env::consts::OS == "windows" { + } else if cfg!(windows_host) { perms.set_readonly(false); } fs::set_permissions(&entry_path, perms)?; From 696a5d102e3cc727ddb97d1b4a7729a7e38492b5 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Wed, 28 Jun 2023 09:53:32 +0200 Subject: [PATCH 43/64] non-root docker user --- build-image/Dockerfile | 7 ++++--- crates/build/build.rs | 8 -------- crates/build/src/lib.rs | 38 -------------------------------------- 3 files changed, 4 insertions(+), 49 deletions(-) diff --git a/build-image/Dockerfile b/build-image/Dockerfile index 93da667f1..b24b538fe 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -68,7 +68,6 @@ ARG CARGO_CONTRACT_TAG ARG CARGO_CONTRACT_REV ARG CARGO_CONTRACT_GIT - # This is important, see https://github.com/rust-lang/docker-rust/issues/85 ENV RUSTFLAGS="-C target-feature=-crt-static" @@ -99,12 +98,14 @@ RUN apt-get -y update && apt-get -y install gcc=${GCC_VERSION} g++=${G_VERSION} && apt-get clean \ && rm -rf /var/lib/apt/lists/* - FROM slimmed-rust as ink-dev COPY --from=cc-builder /usr/local/cargo/bin/cargo-contract /usr/local/bin/cargo-contract -WORKDIR /contract +RUN useradd --create-home homeuser +RUN mkdir /home/myuser/contract +WORKDIR /home/myuser/contract +USER homeuser # default entry point ENTRYPOINT ["cargo", "contract", "build", "--release"] diff --git a/crates/build/build.rs b/crates/build/build.rs index c9a5d571c..20610289d 100644 --- a/crates/build/build.rs +++ b/crates/build/build.rs @@ -48,14 +48,6 @@ fn main() { .into(); let res = zip_template(&manifest_dir, &out_dir); - // detect running host OS - if cfg!(windows) { - println!("cargo:rustc-cfg=windows_host"); - } - if cfg!(unix) { - println!("cargo:rustc-cfg=unix_host"); - } - match res { Ok(()) => std::process::exit(0), Err(err) => { diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index 606122b7a..329a81865 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -795,13 +795,6 @@ pub fn execute(args: ExecuteArgs) -> Result { } }; - let cargo_target_path = crate_metadata - .cargo_meta - .target_directory - .as_std_path() - .to_path_buf(); - change_permissions(&cargo_target_path)?; - Ok(BuildResult { dest_wasm, metadata_result, @@ -815,37 +808,6 @@ pub fn execute(args: ExecuteArgs) -> Result { }) } -/// Modifies permissions of all entries in the dir to be read_only = false -#[allow(clippy::permissions_set_readonly_false)] -fn change_permissions(p: &PathBuf) -> Result<()> { - let files_and_paths = fs::read_dir(p)?; - - for e in files_and_paths { - let entry = e?; - let entry_path = entry.path(); - let meta = entry.metadata()?; - let mut perms = meta.permissions(); - if cfg!(unix_host) { - println!("unix system is running"); - use std::os::unix::fs::PermissionsExt; - // give r-w-x to the owner - // give r-w to the group - // give r to all other users - perms.set_mode(0o764); - } else if cfg!(windows_host) { - perms.set_readonly(false); - } - fs::set_permissions(&entry_path, perms)?; - - // if it's a directory recursively change permissions in there too - if meta.is_dir() { - change_permissions(&entry_path)?; - } - } - - Ok(()) -} - /// Build the contract on host locally fn local_build( crate_metadata: &CrateMetadata, From acf89adfdbb61fcc22b5bcb6992cf25f58d86d98 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Wed, 28 Jun 2023 10:30:56 +0200 Subject: [PATCH 44/64] mount contract to home dir in docker --- build-image/Dockerfile | 4 ++-- crates/build/src/docker.rs | 12 +++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/build-image/Dockerfile b/build-image/Dockerfile index b24b538fe..4a9e8bd73 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -103,8 +103,8 @@ FROM slimmed-rust as ink-dev COPY --from=cc-builder /usr/local/cargo/bin/cargo-contract /usr/local/bin/cargo-contract RUN useradd --create-home homeuser -RUN mkdir /home/myuser/contract -WORKDIR /home/myuser/contract +RUN mkdir /home/homeuser/contract +WORKDIR /home/homeuser/contract USER homeuser # default entry point diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index 45a20ca57..99822cd71 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -100,6 +100,8 @@ const IMAGE: &str = "paritytech/contracts-verifiable"; // We assume the docker image contains the same tag as the current version of the crate const VERSION: &str = env!("CARGO_PKG_VERSION"); +const MOUNT_DIR: &str = "/home/homeuser/contract"; + #[derive(Clone, Debug, Default)] pub enum ImageVariant { #[default] @@ -191,14 +193,14 @@ fn read_build_result( build_result .target_directory .as_path() - .strip_prefix("/contract")?, + .strip_prefix(MOUNT_DIR)?, ); build_result.target_directory = new_path; let new_path = build_result.dest_wasm.as_ref().map(|p| { host_folder.join( p.as_path() - .strip_prefix("/contract") + .strip_prefix(MOUNT_DIR) .expect("cannot strip prefix"), ) }); @@ -208,13 +210,13 @@ fn read_build_result( m.dest_bundle = host_folder.join( m.dest_bundle .as_path() - .strip_prefix("/contract") + .strip_prefix(MOUNT_DIR) .expect("cannot strip prefix"), ); m.dest_metadata = host_folder.join( m.dest_metadata .as_path() - .strip_prefix("/contract") + .strip_prefix(MOUNT_DIR) .expect("cannot strip prefix"), ); m @@ -302,7 +304,7 @@ async fn run_build( let container_option = containers.first(); let mount = Mount { - target: Some(String::from("/contract")), + target: Some(String::from(MOUNT_DIR)), source: Some( host_folder .to_str() From 217ab6a49338e35ba7ba19b4fb4ca6cb970db10c Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Wed, 28 Jun 2023 15:34:30 +0200 Subject: [PATCH 45/64] give permission to target to everyone --- build-image/Dockerfile | 4 +--- crates/build/src/docker.rs | 2 +- crates/build/src/lib.rs | 30 ++++++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/build-image/Dockerfile b/build-image/Dockerfile index 4a9e8bd73..c75ef5c86 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -102,9 +102,7 @@ FROM slimmed-rust as ink-dev COPY --from=cc-builder /usr/local/cargo/bin/cargo-contract /usr/local/bin/cargo-contract -RUN useradd --create-home homeuser -RUN mkdir /home/homeuser/contract -WORKDIR /home/homeuser/contract +WORKDIR /contract USER homeuser # default entry point diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index 99822cd71..442fcc82a 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -100,7 +100,7 @@ const IMAGE: &str = "paritytech/contracts-verifiable"; // We assume the docker image contains the same tag as the current version of the crate const VERSION: &str = env!("CARGO_PKG_VERSION"); -const MOUNT_DIR: &str = "/home/homeuser/contract"; +const MOUNT_DIR: &str = "/contract"; #[derive(Clone, Debug, Default)] pub enum ImageVariant { diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index 329a81865..83958b6d1 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -795,6 +795,13 @@ pub fn execute(args: ExecuteArgs) -> Result { } }; + let cargo_target_path = crate_metadata + .cargo_meta + .target_directory + .as_std_path() + .to_path_buf(); + change_permissions(&cargo_target_path)?; + Ok(BuildResult { dest_wasm, metadata_result, @@ -808,6 +815,29 @@ pub fn execute(args: ExecuteArgs) -> Result { }) } +/// Modifies permissions of all entries in the dir to be read_only = false +#[allow(clippy::permissions_set_readonly_false)] +fn change_permissions(p: &PathBuf) -> Result<()> { + let files_and_paths = fs::read_dir(p)?; + + for e in files_and_paths { + let entry = e?; + let entry_path = entry.path(); + let meta = entry.metadata()?; + let mut perms = meta.permissions(); + perms.set_readonly(false); + + fs::set_permissions(&entry_path, perms)?; + + // if it's a directory, then recursively change permissions in there too + if meta.is_dir() { + change_permissions(&entry_path)?; + } + } + + Ok(()) +} + /// Build the contract on host locally fn local_build( crate_metadata: &CrateMetadata, From 707bfb1390dd18209c3ac2b4e8179d818451fcc7 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Wed, 28 Jun 2023 15:37:45 +0200 Subject: [PATCH 46/64] fix typo --- build-image/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/build-image/Dockerfile b/build-image/Dockerfile index c75ef5c86..0f9945536 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -103,7 +103,6 @@ FROM slimmed-rust as ink-dev COPY --from=cc-builder /usr/local/cargo/bin/cargo-contract /usr/local/bin/cargo-contract WORKDIR /contract -USER homeuser # default entry point ENTRYPOINT ["cargo", "contract", "build", "--release"] From f620dccd4412ef9f8adbc54ad87155b79ec1ea09 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Thu, 29 Jun 2023 11:17:24 +0200 Subject: [PATCH 47/64] use current uid and gid for docker --- Cargo.lock | 11 +++++++++++ crates/build/Cargo.toml | 1 + crates/build/src/docker.rs | 7 +++++++ crates/build/src/lib.rs | 30 ------------------------------ 4 files changed, 19 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8ae8de7d6..f90efe64c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -762,6 +762,7 @@ dependencies = [ "toml 0.7.5", "tracing", "url", + "users", "wabt", "walkdir", "wasm-opt", @@ -5247,6 +5248,16 @@ dependencies = [ "serde", ] +[[package]] +name = "users" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24cc0f6d6f267b73e5a2cadf007ba8f9bc39c6a6f9666f8cf25ea809a153b032" +dependencies = [ + "libc", + "log", +] + [[package]] name = "utf8parse" version = "0.2.1" diff --git a/crates/build/Cargo.toml b/crates/build/Cargo.toml index 545a1f542..91d038e65 100644 --- a/crates/build/Cargo.toml +++ b/crates/build/Cargo.toml @@ -43,6 +43,7 @@ tokio = { version = "1", features = ["macros", "rt-multi-thread"] } tokio-stream = "0.1" bollard = "0.14" indicatif = "0.17" +users = "0.11" contract-metadata = { version = "3.0.1", path = "../metadata" } diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index 442fcc82a..8e21a6887 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -319,6 +319,12 @@ async fn run_build( ..Default::default() }); + let user = Some(format!( + "{}:{}", + users::get_current_uid(), + users::get_current_gid() + )); + let mut labels = HashMap::new(); labels.insert("cmd_digest".to_string(), digest.to_string()); let config = Config { @@ -329,6 +335,7 @@ async fn run_build( host_config: host_cfg, attach_stderr: Some(true), tty: Some(true), + user, ..Default::default() }; let options = Some(CreateContainerOptions { diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index 83958b6d1..329a81865 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -795,13 +795,6 @@ pub fn execute(args: ExecuteArgs) -> Result { } }; - let cargo_target_path = crate_metadata - .cargo_meta - .target_directory - .as_std_path() - .to_path_buf(); - change_permissions(&cargo_target_path)?; - Ok(BuildResult { dest_wasm, metadata_result, @@ -815,29 +808,6 @@ pub fn execute(args: ExecuteArgs) -> Result { }) } -/// Modifies permissions of all entries in the dir to be read_only = false -#[allow(clippy::permissions_set_readonly_false)] -fn change_permissions(p: &PathBuf) -> Result<()> { - let files_and_paths = fs::read_dir(p)?; - - for e in files_and_paths { - let entry = e?; - let entry_path = entry.path(); - let meta = entry.metadata()?; - let mut perms = meta.permissions(); - perms.set_readonly(false); - - fs::set_permissions(&entry_path, perms)?; - - // if it's a directory, then recursively change permissions in there too - if meta.is_dir() { - change_permissions(&entry_path)?; - } - } - - Ok(()) -} - /// Build the contract on host locally fn local_build( crate_metadata: &CrateMetadata, From 37ae55d29d0bcf9cbf7f18f2ff23b92af83de6c1 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Thu, 29 Jun 2023 12:15:00 +0200 Subject: [PATCH 48/64] conditional compilation --- crates/build/Cargo.toml | 4 +++- crates/build/src/docker.rs | 14 +++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/crates/build/Cargo.toml b/crates/build/Cargo.toml index 91d038e65..f8a96ac48 100644 --- a/crates/build/Cargo.toml +++ b/crates/build/Cargo.toml @@ -43,10 +43,12 @@ tokio = { version = "1", features = ["macros", "rt-multi-thread"] } tokio-stream = "0.1" bollard = "0.14" indicatif = "0.17" -users = "0.11" contract-metadata = { version = "3.0.1", path = "../metadata" } +[target.'cfg(unix)'.dependencies] +users = "0.11" + [build-dependencies] anyhow = "1.0.71" walkdir = "2.3.3" diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index 8e21a6887..6d6521518 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -319,11 +319,15 @@ async fn run_build( ..Default::default() }); - let user = Some(format!( - "{}:{}", - users::get_current_uid(), - users::get_current_gid() - )); + let user = if cfg!(unix) { + Some(format!( + "{}:{}", + users::get_current_uid(), + users::get_current_gid() + )) + } else { + None + }; let mut labels = HashMap::new(); labels.insert("cmd_digest".to_string(), digest.to_string()); From dd6a89ddf1e47a5235f40032dd88980f6557d8ad Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Thu, 29 Jun 2023 17:00:08 +0200 Subject: [PATCH 49/64] dont cache ci template and use musl in docker --- .github/workflows/ci.yml | 2 ++ build-image/Dockerfile | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7a655621..d4fb4815e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,6 +189,8 @@ jobs: - name: Cache uses: Swatinem/rust-cache@v2 + with: + cache-targets: "false" - name: Check Template run: >- diff --git a/build-image/Dockerfile b/build-image/Dockerfile index 0f9945536..bbb768f6c 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -20,6 +20,7 @@ ARG GCC_VERSION=4:10.2.1-1 ARG WGET_VERSION=1.21-1+deb11u1 # g++ package version ARG G_VERSION=4:10.2.1-1 +ARG MUSL_V=1.2.2-1 # metadata LABEL io.parity.image.vendor="Parity Technologies" \ @@ -37,7 +38,8 @@ LABEL io.parity.image.vendor="Parity Technologies" \ io.parity.version.cargo-contract.git.tag=${CARGO_CONTRACT_TAG} \ io.parity.version.gcc=${GCC_VERSION} \ io.parity.version.wget=${WGET_VERSION} \ - io.parity.version.g_plus_plus=${G_VERSION} + io.parity.version.g_plus_plus=${G_VERSION} \ + io.parity.version.musl=${MUSL_V} ENV RUSTUP_HOME=/usr/local/rustup \ CARGO_HOME=/usr/local/cargo \ @@ -63,6 +65,7 @@ FROM slimmed-rust as cc-builder ARG CARGO_CONTRACT_VERSION ARG GCC_VERSION ARG G_VERSION +ARG MUSL_V ARG CARGO_CONTRACT_BRANCH ARG CARGO_CONTRACT_TAG ARG CARGO_CONTRACT_REV @@ -72,7 +75,7 @@ ARG CARGO_CONTRACT_GIT ENV RUSTFLAGS="-C target-feature=-crt-static" # Install required packages for `cargo-contract` -RUN apt-get -y update && apt-get -y install gcc=${GCC_VERSION} g++=${G_VERSION} \ +RUN apt-get -y update && apt-get -y install gcc=${GCC_VERSION} g++=${G_VERSION} musl-dev=${MUSL_V} \ # Install cargo contract from git if the arg is set && if [ -n "$CARGO_CONTRACT_GIT" ]; then \ COMMAND="cargo install --git ${CARGO_CONTRACT_GIT}" ; \ From 5ec303bdc62e652dd84be3d4475d6b97db2781b7 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Thu, 29 Jun 2023 18:24:39 +0200 Subject: [PATCH 50/64] OS specific code blocks --- .github/workflows/ci.yml | 2 -- crates/build/src/docker.rs | 14 +++++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4fb4815e..e7a655621 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,8 +189,6 @@ jobs: - name: Cache uses: Swatinem/rust-cache@v2 - with: - cache-targets: "false" - name: Check Template run: >- diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index 6d6521518..f91e9abb0 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -319,15 +319,19 @@ async fn run_build( ..Default::default() }); - let user = if cfg!(unix) { - Some(format!( + let user; + #[cfg(unix)] + { + user = Some(format!( "{}:{}", users::get_current_uid(), users::get_current_gid() - )) - } else { - None + )); }; + #[cfg(windows)] + { + user = None; + } let mut labels = HashMap::new(); labels.insert("cmd_digest".to_string(), digest.to_string()); From 7209fa3bac6793f5f2c7c5a0e06dd79c7e7be5c3 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Mon, 3 Jul 2023 15:28:31 +0100 Subject: [PATCH 51/64] print error logs --- crates/build/src/docker.rs | 52 +++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index f91e9abb0..e9e3d0535 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -28,11 +28,10 @@ //! 2. Parse other arguments that were passed to the host execution context //! 3. Calculate the digest of the command and use it //! to uniquely identify the container -//! 4. If the container exists, we just start the build, if not, we create it it +//! 4. If the container exists, we just start the build, if not, we create it //! 5. After the build, the docker container produces metadata with //! paths relative to its internal storage structure, we parse the file //! and overwrite those paths relative to the host machine. -//! 6. Done! use std::{ collections::{ @@ -60,6 +59,8 @@ use bollard::{ Config, CreateContainerOptions, ListContainersOptions, + LogOutput, + LogsOptions, StopContainerOptions, WaitContainerOptions, }, @@ -283,7 +284,9 @@ async fn run_build( // we are hashing the inputted command // in order to reuse the container for the same permutation of arguments let mut s = DefaultHasher::new(); - entrypoint.hash(&mut s); + // the data is set of commands and args and the image digest + let data = (entrypoint.clone(), build_image.id.clone()); + data.hash(&mut s); let digest = s.finish(); // taking the first 5 digits to be a unique identifier let digest_code: String = digest.to_string().chars().take(5).collect(); @@ -365,6 +368,11 @@ async fn run_build( .start_container::(&container_id, None) .await?; + let start_time = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .context("Invalid start time")? + .as_secs() as i64; + maybe_println!( verbosity, " {} {}\n {}", @@ -386,14 +394,11 @@ async fn run_build( Err(e) => { // sometimes we cannot remove the file due to privileged access let _ = std::fs::remove_file(build_result_path); - anyhow::bail!( - "{}. Execution failed! Check logs for more details", - e.to_string() - ) + anyhow::bail!("{}. Execution failed!", e.to_string()) } }; if response.status_code != 0 { - anyhow::bail!("Execution failed! Status code: {}. Check the container's logs for more details", response.status_code); + anyhow::bail!("Execution failed! Status code: {}.", response.status_code); } Ok(()) }; @@ -407,7 +412,36 @@ async fn run_build( pb.set_style(spinner_style); pb.set_message("Build is being executed..."); while let Some(r) = wait_stream.next().await { - handle_error(r)?; + let res = handle_error(r); + if let Some(e) = res.err() { + let err_logs: Vec = client + .logs::( + &container_id, + Some(LogsOptions { + follow: false, + stdout: true, + stderr: true, + since: start_time, + ..Default::default() + }), + ) + .filter_map(|l| l.ok()) + .collect() + .await; + let err_string = err_logs + .iter() + .filter_map(|l| { + if let LogOutput::Console { message } = l { + let msg = String::from_utf8(message.to_vec()).unwrap(); + Some(msg) + } else { + None + } + }) + .collect::>() + .join("\n"); + anyhow::bail!("{}\n{}", e, err_string); + } pb.inc(1); } pb.finish_with_message("Done!") From 6b8a028644649fa131d420f4f888a5d1e931ffc4 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Mon, 3 Jul 2023 18:28:21 +0100 Subject: [PATCH 52/64] remove the message --- crates/build/src/docker.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index e9e3d0535..8a670881a 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -375,12 +375,11 @@ async fn run_build( maybe_println!( verbosity, - " {} {}\n {}", + " {} {}", format!("{build_steps}").bold(), "Started the build inside the container" .bright_green() .bold(), - "You can close this terminal session. The execution will be finished in the background" ); let options = Some(WaitContainerOptions { From 00d82b4e0a134473c90370bb846c16bde7d8e178 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Mon, 3 Jul 2023 18:35:43 +0100 Subject: [PATCH 53/64] calculate digest in separate func --- crates/build/src/docker.rs | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index 8a670881a..eac27026e 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -280,18 +280,9 @@ async fn run_build( entrypoint.append(&mut cmds); - // in order to optimise the container usage - // we are hashing the inputted command - // in order to reuse the container for the same permutation of arguments - let mut s = DefaultHasher::new(); - // the data is set of commands and args and the image digest - let data = (entrypoint.clone(), build_image.id.clone()); - data.hash(&mut s); - let digest = s.finish(); - // taking the first 5 digits to be a unique identifier - let digest_code: String = digest.to_string().chars().take(5).collect(); - - let container_name = format!("ink-verified-{}-{}", contract_name, digest_code); + let digest_code = container_digest(entrypoint.clone(), build_image.id.clone()); + let container_name = + format!("ink-verified-{}-{}", contract_name, digest_code.clone()); let mut filters = HashMap::new(); filters.insert("name".to_string(), vec![container_name.clone()]); @@ -337,7 +328,7 @@ async fn run_build( } let mut labels = HashMap::new(); - labels.insert("cmd_digest".to_string(), digest.to_string()); + labels.insert("digest-code".to_string(), digest_code); let config = Config { image: Some(build_image.id.clone()), entrypoint: Some(entrypoint), @@ -594,3 +585,18 @@ async fn pull_image( Ok(()) } + +/// Calculates the unique container's code +fn container_digest(entrypoint: Vec, image_digest: String) -> String { + // in order to optimise the container usage + // we are hashing the inputted command + // in order to reuse the container for the same permutation of arguments + let mut s = DefaultHasher::new(); + // the data is set of commands and args and the image digest + let data = (entrypoint, image_digest); + data.hash(&mut s); + let digest = s.finish(); + // taking the first 5 digits to be a unique identifier + let digest_code: String = digest.to_string().chars().take(5).collect(); + digest_code +} From 14d12367bb144dfe2e5e266414462497f400d381 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 4 Jul 2023 00:00:43 +0100 Subject: [PATCH 54/64] filter our status msgs in error --- crates/build/src/docker.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index eac27026e..1aad555f4 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -418,18 +418,23 @@ async fn run_build( .filter_map(|l| l.ok()) .collect() .await; + // cargo dumps compilation status together with other logs + // we need to filter our those messages + let rex = regex::Regex::new(r"\[=*> \]")?; let err_string = err_logs .iter() .filter_map(|l| { if let LogOutput::Console { message } = l { let msg = String::from_utf8(message.to_vec()).unwrap(); + let msg = + msg.split('\r').filter(|m| !rex.is_match(m)).collect(); Some(msg) } else { None } }) .collect::>() - .join("\n"); + .join(""); anyhow::bail!("{}\n{}", e, err_string); } pb.inc(1); From b2faaffd85b254b376e491e645254fe99d9993a4 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 4 Jul 2023 14:01:12 +0100 Subject: [PATCH 55/64] update comments --- crates/metadata/src/lib.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/metadata/src/lib.rs b/crates/metadata/src/lib.rs index 02223d351..0e3b30f76 100644 --- a/crates/metadata/src/lib.rs +++ b/crates/metadata/src/lib.rs @@ -54,10 +54,8 @@ //! let user = User::new(user_json); //! // contract abi raw json generated by contract compilation //! let abi_json: Map = Map::new(); -//! // image digest used for the verifiable build. -//! let image = String::from( -//! "sha256:824b6fde00873a146f52b41e7154e98c9d371609f9fd77e1b22ccd530f052d66", -//! ); +//! // image name and tag used for the verifiable build. +//! let image = String::from("paritytech/contracts-verifiable:3.0.1"); //! //! let metadata = //! ContractMetadata::new(source, contract, Some(image), Some(user), abi_json); @@ -104,7 +102,8 @@ pub struct ContractMetadata { pub source: Source, /// Metadata about the contract. pub contract: Contract, - /// If the contract is verifiable, then the Docker image id is specified. + /// If the contract is meant to be verifiable, + /// then the Docker image is specified. pub image: Option, /// Additional user-defined metadata. #[serde(skip_serializing_if = "Option::is_none")] From 470047243b4ddd9cf01b44f0aed58db74ec9868f Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Tue, 4 Jul 2023 19:34:15 +0100 Subject: [PATCH 56/64] resolve other merge conflicts --- Cargo.lock | 227 +------------------------------ crates/cargo-contract/Cargo.toml | 6 - crates/extrinsics/Cargo.toml | 2 +- crates/extrinsics/src/call.rs | 22 +-- crates/extrinsics/src/upload.rs | 11 -- 5 files changed, 3 insertions(+), 265 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 94562b463..6ce24fd37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -198,77 +198,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" -[[package]] -name = "async-attributes" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5" -dependencies = [ - "quote", - "syn 1.0.109", -] - -[[package]] -name = "async-channel" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833" -dependencies = [ - "concurrent-queue", - "event-listener", - "futures-core", -] - -[[package]] -name = "async-executor" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fa3dc5f2a8564f07759c008b9109dc0d39de92a88d5588b8a5036d286383afb" -dependencies = [ - "async-lock", - "async-task", - "concurrent-queue", - "fastrand", - "futures-lite", - "slab", -] - -[[package]] -name = "async-global-executor" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" -dependencies = [ - "async-channel", - "async-executor", - "async-io", - "async-lock", - "blocking", - "futures-lite", - "once_cell", - "tokio", -] - -[[package]] -name = "async-io" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" -dependencies = [ - "async-lock", - "autocfg", - "cfg-if", - "concurrent-queue", - "futures-lite", - "log", - "parking", - "polling", - "rustix 0.37.22", - "slab", - "socket2", - "waker-fn", -] - [[package]] name = "async-lock" version = "2.7.0" @@ -278,39 +207,6 @@ dependencies = [ "event-listener", ] -[[package]] -name = "async-std" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" -dependencies = [ - "async-attributes", - "async-channel", - "async-global-executor", - "async-io", - "async-lock", - "crossbeam-utils", - "futures-channel", - "futures-core", - "futures-io", - "futures-lite", - "gloo-timers", - "kv-log-macro", - "log", - "memchr", - "once_cell", - "pin-project-lite", - "pin-utils", - "slab", - "wasm-bindgen-futures", -] - -[[package]] -name = "async-task" -version = "4.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc7ab41815b3c653ccd2978ec3255c81349336702dfdf62ee6f7069b12a3aae" - [[package]] name = "async-trait" version = "0.1.69" @@ -322,12 +218,6 @@ dependencies = [ "syn 2.0.23", ] -[[package]] -name = "atomic-waker" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1181e1e0d1fce796a03db1ae795d67167da795f9cf4a39c37589e85ef57f26d3" - [[package]] name = "atty" version = "0.2.14" @@ -479,21 +369,6 @@ dependencies = [ "byte-tools", ] -[[package]] -name = "blocking" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77231a1c8f801696fc0123ec6150ce92cffb8e164a02afb9c8ddee0e9b65ad65" -dependencies = [ - "async-channel", - "async-lock", - "async-task", - "atomic-waker", - "fastrand", - "futures-lite", - "log", -] - [[package]] name = "bollard" version = "0.14.0" @@ -691,15 +566,9 @@ dependencies = [ "contract-transcode", "current_platform", "hex", - "pallet-contracts-primitives", "predicates", "regex", - "rust_decimal", - "scale-info", "serde_json", - "sp-core", - "sp-runtime", - "sp-weights", "substrate-build-script-utils", "subxt", "tempfile", @@ -838,15 +707,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "concurrent-queue" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "console" version = "0.15.7" @@ -911,7 +771,6 @@ name = "contract-extrinsics" version = "3.0.1" dependencies = [ "anyhow", - "async-std", "clap", "colored", "contract-build", @@ -929,6 +788,7 @@ dependencies = [ "sp-runtime", "sp-weights", "subxt", + "tokio", "tracing", "url", ] @@ -1569,21 +1429,6 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" -[[package]] -name = "futures-lite" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" -dependencies = [ - "fastrand", - "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite", - "waker-fn", -] - [[package]] name = "futures-macro" version = "0.3.28" @@ -1691,18 +1536,6 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" -[[package]] -name = "gloo-timers" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "h2" version = "0.3.20" @@ -2484,15 +2317,6 @@ dependencies = [ "cpufeatures", ] -[[package]] -name = "kv-log-macro" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" -dependencies = [ - "log", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -2595,9 +2419,6 @@ name = "log" version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" -dependencies = [ - "value-bag", -] [[package]] name = "mach" @@ -2894,12 +2715,6 @@ version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1ad0aff30c1da14b1254fcb2af73e1fa9a28670e584a626f53a369d0e157304" -[[package]] -name = "parking" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e" - [[package]] name = "parking_lot" version = "0.12.1" @@ -2991,22 +2806,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8d0eef3571242013a0d5dc84861c3ae4a652e56e12adf8bdc26ff5f8cb34c94" -[[package]] -name = "polling" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" -dependencies = [ - "autocfg", - "bitflags 1.3.2", - "cfg-if", - "concurrent-queue", - "libc", - "log", - "pin-project-lite", - "windows-sys 0.48.0", -] - [[package]] name = "portable-atomic" version = "1.3.3" @@ -5099,12 +4898,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" -[[package]] -name = "value-bag" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d92ccd67fb88503048c01b59152a04effd0782d035a83a6d256ce6085f08f4a3" - [[package]] name = "version_check" version = "0.9.4" @@ -5143,12 +4936,6 @@ dependencies = [ "libc", ] -[[package]] -name = "waker-fn" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" - [[package]] name = "walkdir" version = "2.3.3" @@ -5205,18 +4992,6 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "wasm-bindgen-macro" version = "0.2.87" diff --git a/crates/cargo-contract/Cargo.toml b/crates/cargo-contract/Cargo.toml index adba008d7..f55a5da81 100644 --- a/crates/cargo-contract/Cargo.toml +++ b/crates/cargo-contract/Cargo.toml @@ -30,16 +30,10 @@ which = "4.4.0" colored = "2.0.1" serde_json = "1.0.99" url = { version = "2.4.0", features = ["serde"] } -rust_decimal = "1.30" # dependencies for extrinsics (deploying and calling a contract) tokio = { version = "1", features = ["macros", "rt-multi-thread"] } -sp-core = "21.0.0" -sp-runtime = "24.0.0" -sp-weights = "20.0.0" -pallet-contracts-primitives = "24.0.0" -scale-info = "2.8.0" subxt = "0.29.0" hex = "0.4.3" diff --git a/crates/extrinsics/Cargo.toml b/crates/extrinsics/Cargo.toml index 9d165d68b..be9dfc74f 100644 --- a/crates/extrinsics/Cargo.toml +++ b/crates/extrinsics/Cargo.toml @@ -27,7 +27,7 @@ serde = { version = "1.0.164", default-features = false, features = ["derive"] } serde_json = "1.0.97" url = { version = "2.4.0", features = ["serde"] } rust_decimal = "1.30" -async-std = { version = "1.12.0", features = ["attributes", "tokio1"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } sp-core = "21.0.0" sp-runtime = "24.0.0" sp-weights = "20.0.0" diff --git a/crates/extrinsics/src/call.rs b/crates/extrinsics/src/call.rs index a6f053319..6ac3a248f 100644 --- a/crates/extrinsics/src/call.rs +++ b/crates/extrinsics/src/call.rs @@ -37,23 +37,6 @@ use super::{ MAX_KEY_COL_WIDTH, }; -<<<<<<< HEAD:crates/cargo-contract/src/cmd/extrinsics/call.rs -use crate::{ - cmd::{ - extrinsics::{ - display_contract_exec_result_debug, - display_dry_run_result_warning, - error::GenericError, - events::DisplayEvents, - ErrorVariant, - }, - runtime_api::api, - Balance, - }, - DEFAULT_KEY_COL_WIDTH, -}; -======= ->>>>>>> master:crates/extrinsics/src/call.rs use contract_build::name_value_println; use anyhow::{ @@ -120,10 +103,7 @@ impl CallCommand { let signer = super::pair_signer(self.extrinsic_opts.signer()?); - Runtime::new() - .map_err(|e| { - ErrorVariant::Generic(GenericError::from_message(e.to_string())) - })? + Runtime::new()? .block_on(async { let url = self.extrinsic_opts.url_to_string(); let client = OnlineClient::from_url(url.clone()).await?; diff --git a/crates/extrinsics/src/upload.rs b/crates/extrinsics/src/upload.rs index b38b43020..b6bb66cf3 100644 --- a/crates/extrinsics/src/upload.rs +++ b/crates/extrinsics/src/upload.rs @@ -120,18 +120,7 @@ impl UploadCommand { ) .into()) } -<<<<<<< HEAD:crates/cargo-contract/src/cmd/extrinsics/upload.rs Ok(()) -======= - } else { - let code_hash = hex::encode(code_hash); - return Err(anyhow::anyhow!( - "This contract has already been uploaded with code hash: 0x{code_hash}" - ) - .into()); - } - Ok(()) ->>>>>>> master:crates/extrinsics/src/upload.rs }) } From 0c4009200d4029a65e080678b0a174a6ea873b54 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Mon, 10 Jul 2023 15:44:09 +0100 Subject: [PATCH 57/64] display container name --- crates/build/src/docker.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index 1aad555f4..719d5bbf8 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -368,7 +368,7 @@ async fn run_build( verbosity, " {} {}", format!("{build_steps}").bold(), - "Started the build inside the container" + format!("Started the build inside the container: {container_name}") .bright_green() .bold(), ); From 92a786722038aa2441c31dd7a19eb23032c09373 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Mon, 10 Jul 2023 17:25:30 +0100 Subject: [PATCH 58/64] properly trancate error output --- crates/build/src/docker.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index 719d5bbf8..5bbb2ca2c 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -418,17 +418,18 @@ async fn run_build( .filter_map(|l| l.ok()) .collect() .await; - // cargo dumps compilation status together with other logs - // we need to filter our those messages - let rex = regex::Regex::new(r"\[=*> \]")?; let err_string = err_logs .iter() .filter_map(|l| { if let LogOutput::Console { message } = l { let msg = String::from_utf8(message.to_vec()).unwrap(); - let msg = - msg.split('\r').filter(|m| !rex.is_match(m)).collect(); - Some(msg) + // as the logs also contain compilation status output, + // we need to take the slice from the start of the error + // message + let remainder = msg + [msg.find(r"error\[.*\]").unwrap_or_default()..] + .to_string(); + Some(remainder) } else { None } From b6b66e90c24c41ba4e41c90003a99ffdb8cfca38 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Wed, 12 Jul 2023 16:27:08 +0100 Subject: [PATCH 59/64] apply Andrews suggestions --- build-image/Dockerfile | 2 +- crates/build/src/docker.rs | 362 +++++++++++++++---------------------- 2 files changed, 149 insertions(+), 215 deletions(-) diff --git a/build-image/Dockerfile b/build-image/Dockerfile index bbb768f6c..94e8b4eba 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -108,4 +108,4 @@ COPY --from=cc-builder /usr/local/cargo/bin/cargo-contract /usr/local/bin/cargo- WORKDIR /contract # default entry point -ENTRYPOINT ["cargo", "contract", "build", "--release"] +ENTRYPOINT ["cargo", "contract"] diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index c2e7fe20c..bffdb07c1 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -42,11 +42,11 @@ use std::{ Hash, Hasher, }, - io::BufReader, - path::{ - Path, - PathBuf, + io::{ + BufReader, + Write, }, + path::Path, time::Duration, }; @@ -56,21 +56,18 @@ use anyhow::{ }; use bollard::{ container::{ + AttachContainerOptions, + AttachContainerResults, Config, CreateContainerOptions, ListContainersOptions, LogOutput, - LogsOptions, - StopContainerOptions, - WaitContainerOptions, }, - errors::Error, image::{ CreateImageOptions, ListImagesOptions, }, service::{ - ContainerWaitResponse, HostConfig, ImageSummary, Mount, @@ -133,11 +130,7 @@ pub fn docker_build(args: ExecuteArgs) -> Result { } let crate_metadata = CrateMetadata::collect(&manifest_path, target)?; - let mut manifest_dir_option = crate_metadata.manifest_path.directory(); - let empty_path = PathBuf::new(); - let contract_dir = manifest_dir_option.get_or_insert(&empty_path); - let host_dir = std::env::current_dir()?; - let build_result_path = contract_dir.join("target/build_result.json"); + let host_folder = std::env::current_dir()?; let args = compose_build_args()?; let client = Docker::connect_with_socket_defaults().map_err(|e| { @@ -146,23 +139,32 @@ pub fn docker_build(args: ExecuteArgs) -> Result { let _ = client.ping().await.map_err(|e| { anyhow::anyhow!("{}\nIs your docker engine up and running?", e) })?; - let build_image = - get_image(client.clone(), image, &verbosity, &mut build_steps).await?; - run_build( - args, - &build_image, - &build_result_path, + let image = match image { + ImageVariant::Custom(i) => i.clone(), + ImageVariant::Default => { + format!("{}:{}", IMAGE, VERSION) + } + }; + + let container = create_container( + &client, + args.clone(), + &image, &crate_metadata.contract_artifact_name, - &host_dir, + &host_folder, &verbosity, &mut build_steps, ) .await?; - let build_result = read_build_result(&host_dir, &build_result_path)?; + let mut build_result = + run_build(&client, &container, &verbosity, &mut build_steps).await?; + + update_build_result(&host_folder, &mut build_result)?; - update_metadata(&build_result, &verbosity, &mut build_steps, &build_image)?; + update_metadata(&build_result, &verbosity, &mut build_steps, &image, &client) + .await?; Ok(BuildResult { output_type, @@ -173,23 +175,7 @@ pub fn docker_build(args: ExecuteArgs) -> Result { } /// Reads the `BuildResult` produced by the docker execution -fn read_build_result( - host_folder: &Path, - build_result_path: &PathBuf, -) -> Result { - let file = std::fs::File::open(build_result_path)?; - let mut build_result: BuildResult = - match serde_json::from_reader(BufReader::new(file)) { - Ok(result) => result, - Err(_) => { - // sometimes we cannot remove the file due to privileged access - let _ = std::fs::remove_file(build_result_path); - anyhow::bail!( - "Error parsing output from docker build. The build probably failed!" - ) - } - }; - +fn update_build_result(host_folder: &Path, build_result: &mut BuildResult) -> Result<()> { let new_path = host_folder.join( build_result .target_directory @@ -222,18 +208,23 @@ fn read_build_result( ); m }); - Ok(build_result) + Ok(()) } /// Overwrites `build_result` and `image` fields in the metadata -fn update_metadata( +async fn update_metadata( build_result: &BuildResult, verbosity: &Verbosity, build_steps: &mut BuildSteps, - build_image: &ImageSummary, + build_image: &str, + client: &Docker, ) -> Result<()> { if let Some(metadata_artifacts) = &build_result.metadata_result { let mut metadata = ContractMetadata::load(&metadata_artifacts.dest_bundle)?; + + let build_image = find_local_image(client, build_image.to_string()) + .await? + .context("Image summary does not exist")?; // find alternative unique identifier of the image, otherwise grab the digest let image_tag = match build_image .repo_tags @@ -256,31 +247,45 @@ fn update_metadata( Ok(()) } -/// Creates the container and executed the build inside it -async fn run_build( - build_args: String, - build_image: &ImageSummary, - build_result_path: &PathBuf, +/// Searches for the local copy of the docker image +async fn find_local_image( + client: &Docker, + image: String, +) -> Result> { + let images = client + .list_images(Some(ListImagesOptions:: { + all: true, + ..Default::default() + })) + .await?; + let build_image = images.iter().find(|i| i.repo_tags.contains(&image)); + + Ok(build_image.cloned()) +} + +/// Creates the container, returning the container id if successful. +/// +/// If the image is not available locally, it will be pulled from the registry. +async fn create_container( + client: &Docker, + mut build_args: Vec, + build_image: &str, contract_name: &str, host_folder: &Path, verbosity: &Verbosity, build_steps: &mut BuildSteps, -) -> Result<()> { - let client = Docker::connect_with_socket_defaults()?; +) -> Result { + let entrypoint = vec!["cargo".to_string(), "contract".to_string()]; - let mut entrypoint = vec!["/bin/bash".to_string(), "-c".to_string()]; - let b_path_str = build_result_path - .as_os_str() - .to_str() - .context("Cannot convert Os String to String")?; - let mut cmds = vec![format!( - "mkdir -p target && cargo contract build {} --output-json > {}", - build_args, b_path_str - )]; + let mut cmd = vec![ + "build".to_string(), + "--release".to_string(), + "--output-json".to_string(), + ]; - entrypoint.append(&mut cmds); + cmd.append(&mut build_args); - let digest_code = container_digest(entrypoint.clone(), build_image.id.clone()); + let digest_code = container_digest(entrypoint.clone(), build_image.to_string()); let container_name = format!("ink-verified-{}-{}", contract_name, digest_code.clone()); @@ -297,6 +302,10 @@ async fn run_build( let container_option = containers.first(); + if let Some(summary) = container_option { + return summary.id.clone().context("container does not have an id") + } + let mount = Mount { target: Some(String::from(MOUNT_DIR)), source: Some( @@ -327,16 +336,12 @@ async fn run_build( user = None; } - let mut labels = HashMap::new(); - labels.insert("digest-code".to_string(), digest_code); let config = Config { - image: Some(build_image.id.clone()), + image: Some(build_image.to_string()), entrypoint: Some(entrypoint), - cmd: None, - labels: Some(labels), + cmd: Some(cmd), host_config: host_cfg, attach_stderr: Some(true), - tty: Some(true), user, ..Default::default() }; @@ -345,128 +350,100 @@ async fn run_build( platform: Some("linux/amd64"), }); - let container_id = match container_option { - Some(container) => { - container - .id - .clone() - .context("Container does not have an ID")? + match client + .create_container(options.clone(), config.clone()) + .await + { + Ok(container) => Ok(container.id), + Err(err) => { + if matches!( + err, + bollard::errors::Error::DockerResponseServerError { + status_code: 404, + .. + } + ) { + // no such image locally, so pull and try again + pull_image(client, build_image.to_string(), verbosity, build_steps) + .await?; + client + .create_container(options, config) + .await + .context("Failed to create docker container") + .map(|o| o.id) + } else { + Err(err.into()) + } } - None => client.create_container(options, config).await?.id, - }; + } +} - client - .start_container::(&container_id, None) +/// Creates the container and executed the build inside it +async fn run_build( + client: &Docker, + container_id: &str, + verbosity: &Verbosity, + build_steps: &mut BuildSteps, +) -> Result { + client.start_container::(container_id, None).await?; + + let AttachContainerResults { mut output, .. } = client + .attach_container( + container_id, + Some(AttachContainerOptions:: { + stdout: Some(true), + stderr: Some(true), + stream: Some(true), + ..Default::default() + }), + ) .await?; - let start_time = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .context("Invalid start time")? - .as_secs() as i64; - verbose_eprintln!( verbosity, " {} {}", format!("{build_steps}").bold(), - format!("Started the build inside the container: {container_name}") + "Started the build inside the container" .bright_green() .bold(), ); - let options = Some(WaitContainerOptions { - condition: "not-running", - }); - - let mut wait_stream = client.wait_container(&container_id, options); - let handle_error = |r: Result| -> Result<()> { - let response = match r { - Ok(v) => v, - Err(e) => { - // sometimes we cannot remove the file due to privileged access - let _ = std::fs::remove_file(build_result_path); - anyhow::bail!("{}. Execution failed!", e.to_string()) + // pipe docker attach output into stdout + let stderr = std::io::stderr(); + let mut stderr = stderr.lock(); + let mut build_result = None; + while let Some(Ok(output)) = output.next().await { + match output { + LogOutput::StdOut { message } => { + build_result = Some( + serde_json::from_reader(BufReader::new(message.as_ref())) + .context("Error decoding BuildResult"), + ); } - }; - if response.status_code != 0 { - anyhow::bail!("Execution failed! Status code: {}.", response.status_code); - } - Ok(()) - }; - if verbosity.is_verbose() { - let spinner_style = - ProgressStyle::with_template(" {spinner:.cyan.bold} {wide_msg}")? - .tick_chars("⠁⠂⠄⡀⢀⠠⠐⠈ "); - - let pb = ProgressBar::new(1000); - pb.enable_steady_tick(Duration::from_millis(100)); - pb.set_style(spinner_style); - pb.set_message("Build is being executed..."); - while let Some(r) = wait_stream.next().await { - let res = handle_error(r); - if let Some(e) = res.err() { - let err_logs: Vec = client - .logs::( - &container_id, - Some(LogsOptions { - follow: false, - stdout: true, - stderr: true, - since: start_time, - ..Default::default() - }), - ) - .filter_map(|l| l.ok()) - .collect() - .await; - let err_string = err_logs - .iter() - .filter_map(|l| { - if let LogOutput::Console { message } = l { - let msg = String::from_utf8(message.to_vec()).unwrap(); - // as the logs also contain compilation status output, - // we need to take the slice from the start of the error - // message - let remainder = msg - [msg.find(r"error\[.*\]").unwrap_or_default()..] - .to_string(); - Some(remainder) - } else { - None - } - }) - .collect::>() - .join(""); - anyhow::bail!("{}\n{}", e, err_string); + LogOutput::StdErr { message } => { + stderr.write_all(message.as_ref())?; + stderr.flush()?; } - pb.inc(1); - } - pb.finish_with_message("Done!") - } else { - while let Some(r) = wait_stream.next().await { - handle_error(r)?; + LogOutput::Console { message: _ } => { + panic!("LogOutput::Console") + } + LogOutput::StdIn { message: _ } => panic!("LogOutput::StdIn"), } } - client - .stop_container(&container_id, Some(StopContainerOptions { t: 20 })) - .await?; - - build_steps.increment_current(); - verbose_eprintln!( - verbosity, - " {} {}", - format!("{build_steps}").bold(), - "Docker container has finished the build. Reading the results" - .bright_green() - .bold(), - ); - Ok(()) + if let Some(build_result) = build_result { + build_result + } else { + Err(anyhow::anyhow!( + "Failed to read build result from docker build" + )) + } } /// Takes CLI args from the host and appends them to the build command inside the docker -fn compose_build_args() -> Result { +fn compose_build_args() -> Result> { use regex::Regex; - let mut args: Vec = vec!["--release".to_string()]; + let mut args: Vec = Vec::new(); let rex = Regex::new(r"--image [.*]*")?; let args_string: String = std::env::args().collect(); @@ -480,62 +457,19 @@ fn compose_build_args() -> Result { && a != &"cargo" && a != &"contract" && a != &"build" + && a != &"--output-json" }) .map(|s| s.to_string()) .collect(); args.append(&mut os_args); - let joined_args = args.join(" "); - Ok(joined_args) -} - -/// Retrieve local of the image, otherwise pulls one from the registry -async fn get_image( - client: Docker, - custom_image: ImageVariant, - verbosity: &Verbosity, - build_steps: &mut BuildSteps, -) -> Result { - // if no custom image is specified, then we use the tag of the current version of - // `cargo-contract` - let image = match custom_image { - ImageVariant::Custom(i) => i.clone(), - ImageVariant::Default => { - format!("{}:{}", IMAGE, VERSION) - } - }; - - let build_image = match find_local_image(client.clone(), image.clone()).await? { - Some(image_s) => image_s, - None => { - build_steps.total_steps = build_steps.total_steps.map(|s| s + 1); - pull_image(client.clone(), image.clone(), verbosity, build_steps).await?; - find_local_image(client.clone(), image.clone()) - .await? - .context("Could not pull the image from the registry")? - } - }; - - Ok(build_image) -} - -/// Searches for the local copy of the docker image -async fn find_local_image(client: Docker, image: String) -> Result> { - let images = client - .list_images(Some(ListImagesOptions:: { - all: true, - ..Default::default() - })) - .await?; - let build_image = images.iter().find(|i| i.repo_tags.contains(&image)); - - Ok(build_image.cloned()) + Ok(args) } /// Pulls the docker image from the registry async fn pull_image( - client: Docker, + client: &Docker, image: String, verbosity: &Verbosity, build_steps: &mut BuildSteps, From 590b5b8d360b47b96ffc184d9a3eefbebfa3b7e1 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Wed, 12 Jul 2023 17:50:37 +0100 Subject: [PATCH 60/64] proper build step logs --- crates/build/src/docker.rs | 30 ++++++++++++++++++++---------- crates/build/src/metadata.rs | 2 +- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index bffdb07c1..87f7659cb 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -166,6 +166,14 @@ pub fn docker_build(args: ExecuteArgs) -> Result { update_metadata(&build_result, &verbosity, &mut build_steps, &image, &client) .await?; + build_steps.increment_current(); + verbose_eprintln!( + verbosity, + " {} {}", + format!("{build_steps}").bold(), + "Displaying results".bright_cyan().bold(), + ); + Ok(BuildResult { output_type, verbosity, @@ -302,8 +310,8 @@ async fn create_container( let container_option = containers.first(); - if let Some(summary) = container_option { - return summary.id.clone().context("container does not have an id") + if container_option.is_some() { + return Ok(container_name) } let mount = Mount { @@ -354,7 +362,7 @@ async fn create_container( .create_container(options.clone(), config.clone()) .await { - Ok(container) => Ok(container.id), + Ok(_) => Ok(container_name), Err(err) => { if matches!( err, @@ -370,7 +378,7 @@ async fn create_container( .create_container(options, config) .await .context("Failed to create docker container") - .map(|o| o.id) + .map(|_| container_name) } else { Err(err.into()) } @@ -381,15 +389,17 @@ async fn create_container( /// Creates the container and executed the build inside it async fn run_build( client: &Docker, - container_id: &str, + container_name: &str, verbosity: &Verbosity, build_steps: &mut BuildSteps, ) -> Result { - client.start_container::(container_id, None).await?; + client + .start_container::(container_name, None) + .await?; let AttachContainerResults { mut output, .. } = client .attach_container( - container_id, + container_name, Some(AttachContainerOptions:: { stdout: Some(true), stderr: Some(true), @@ -403,8 +413,8 @@ async fn run_build( verbosity, " {} {}", format!("{build_steps}").bold(), - "Started the build inside the container" - .bright_green() + format!("Started the build inside the container: {}", container_name) + .bright_cyan() .bold(), ); @@ -488,7 +498,7 @@ async fn pull_image( " {} {}", format!("{build_steps}").bold(), "Image does not exist. Pulling one from the registry" - .bright_green() + .bright_cyan() .bold() ); build_steps.increment_current(); diff --git a/crates/build/src/metadata.rs b/crates/build/src/metadata.rs index e8aaef211..f356252a8 100644 --- a/crates/build/src/metadata.rs +++ b/crates/build/src/metadata.rs @@ -212,7 +212,7 @@ pub fn write_metadata( verbosity, " {} {}", format!("{build_steps}").bold(), - "Generating bundle".bright_green().bold() + "Updating paths".bright_cyan().bold() ); let contents = serde_json::to_string(&metadata)?; fs::write(&metadata_artifacts.dest_bundle, contents)?; From d5155044bb8abb10c7001e62900db5a18d46e860 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Wed, 12 Jul 2023 18:13:18 +0100 Subject: [PATCH 61/64] fix generation of ABI --- crates/build/src/docker.rs | 1 + crates/build/src/metadata.rs | 39 +++++++++++++++++++++--------------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index 87f7659cb..5c57821db 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -250,6 +250,7 @@ async fn update_metadata( metadata, build_steps, verbosity, + true, )?; } Ok(()) diff --git a/crates/build/src/metadata.rs b/crates/build/src/metadata.rs index f356252a8..cf829fe6d 100644 --- a/crates/build/src/metadata.rs +++ b/crates/build/src/metadata.rs @@ -162,16 +162,13 @@ pub fn execute( serde_json::from_slice(&output.stdout)?; let metadata = ContractMetadata::new(source, contract, None, user, ink_meta); - build_steps.increment_current(); - - verbose_eprintln!( - verbosity, - " {} {}", - format!("{build_steps}").bold(), - "Generating bundle".bright_green().bold() - ); - let contents = serde_json::to_string(&metadata)?; - fs::write(&metadata_artifacts.dest_bundle, contents)?; + write_metadata( + metadata_artifacts, + metadata, + &mut build_steps, + &verbosity, + false, + )?; Ok(()) }; @@ -199,6 +196,7 @@ pub fn write_metadata( metadata: ContractMetadata, build_steps: &mut BuildSteps, verbosity: &Verbosity, + overwrite: bool, ) -> Result<()> { { let mut metadata = metadata.clone(); @@ -208,12 +206,21 @@ pub fn write_metadata( build_steps.increment_current(); } - verbose_eprintln!( - verbosity, - " {} {}", - format!("{build_steps}").bold(), - "Updating paths".bright_cyan().bold() - ); + if overwrite { + verbose_eprintln!( + verbosity, + " {} {}", + format!("{build_steps}").bold(), + "Updating paths".bright_cyan().bold() + ); + } else { + verbose_eprintln!( + verbosity, + " {} {}", + format!("{build_steps}").bold(), + "Generating bundle".bright_green().bold() + ); + } let contents = serde_json::to_string(&metadata)?; fs::write(&metadata_artifacts.dest_bundle, contents)?; From 44958e3149042c1425c7681c14e575b670f48415 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Wed, 12 Jul 2023 18:18:32 +0100 Subject: [PATCH 62/64] hash the cmd, not entrypoint for digest --- crates/build/src/docker.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index 5c57821db..ce74b68a5 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -294,7 +294,7 @@ async fn create_container( cmd.append(&mut build_args); - let digest_code = container_digest(entrypoint.clone(), build_image.to_string()); + let digest_code = container_digest(cmd.clone(), build_image.to_string()); let container_name = format!("ink-verified-{}-{}", contract_name, digest_code.clone()); From ad1ddc6d498094570d6c6ec1087615afacaf0eec Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Thu, 13 Jul 2023 15:11:35 +0100 Subject: [PATCH 63/64] fix args parsing and add docs --- crates/build/src/docker.rs | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index ce74b68a5..96373d499 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -93,21 +93,24 @@ use crate::{ }; use colored::Colorize; - +/// Default image to be used for the build. const IMAGE: &str = "paritytech/contracts-verifiable"; -// We assume the docker image contains the same tag as the current version of the crate +/// We assume the docker image contains the same tag as the current version of the crate. const VERSION: &str = env!("CARGO_PKG_VERSION"); - +/// The default directory to be mounted in the container. const MOUNT_DIR: &str = "/contract"; +/// The image to be used. #[derive(Clone, Debug, Default)] pub enum ImageVariant { + /// The default image is used, specified in the `IMAGE` constant. #[default] Default, + /// Custom image is used. Custom(String), } -/// Launches the docker container to execute verifiable build +/// Launches the docker container to execute verifiable build. pub fn docker_build(args: ExecuteArgs) -> Result { let ExecuteArgs { manifest_path, @@ -182,7 +185,7 @@ pub fn docker_build(args: ExecuteArgs) -> Result { }) } -/// Reads the `BuildResult` produced by the docker execution +/// Updates `build_result` paths to the artefacts. fn update_build_result(host_folder: &Path, build_result: &mut BuildResult) -> Result<()> { let new_path = host_folder.join( build_result @@ -219,7 +222,7 @@ fn update_build_result(host_folder: &Path, build_result: &mut BuildResult) -> Re Ok(()) } -/// Overwrites `build_result` and `image` fields in the metadata +/// Overwrites `build_result` and `image` fields in the metadata. async fn update_metadata( build_result: &BuildResult, verbosity: &Verbosity, @@ -256,7 +259,7 @@ async fn update_metadata( Ok(()) } -/// Searches for the local copy of the docker image +/// Searches for the local copy of the docker image. async fn find_local_image( client: &Docker, image: String, @@ -387,7 +390,7 @@ async fn create_container( } } -/// Creates the container and executed the build inside it +/// Starts the container and executed the build inside it. async fn run_build( client: &Docker, container_name: &str, @@ -451,15 +454,18 @@ async fn run_build( } } -/// Takes CLI args from the host and appends them to the build command inside the docker +/// Takes CLI args from the host and appends them to the build command inside the docker. fn compose_build_args() -> Result> { use regex::Regex; let mut args: Vec = Vec::new(); - - let rex = Regex::new(r"--image [.*]*")?; - let args_string: String = std::env::args().collect(); + // match --image with arg with 1 or more white spaces surrounded + let rex = Regex::new(r#"--image[ ]*[^ ]*[ ]*"#)?; + // we join the args together, so we can remove `--image ` + let args_string: String = std::env::args().collect::>().join(" "); let args_string = rex.replace_all(&args_string, "").to_string(); + // and then we turn it back to the vec, filtering out commands and arguments + // that should not be passed to the docker build command let mut os_args: Vec = args_string .split_ascii_whitespace() .filter(|a| { @@ -478,7 +484,7 @@ fn compose_build_args() -> Result> { Ok(args) } -/// Pulls the docker image from the registry +/// Pulls the docker image from the registry. async fn pull_image( client: &Docker, image: String, @@ -537,7 +543,7 @@ async fn pull_image( Ok(()) } -/// Calculates the unique container's code +/// Calculates the unique container's code. fn container_digest(entrypoint: Vec, image_digest: String) -> String { // in order to optimise the container usage // we are hashing the inputted command From 6e02958ec1472da06158e610b78b456940080229 Mon Sep 17 00:00:00 2001 From: SkymanOne Date: Thu, 13 Jul 2023 16:35:01 +0100 Subject: [PATCH 64/64] clippy fix --- crates/build/src/docker.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/build/src/docker.rs b/crates/build/src/docker.rs index 96373d499..e9c35018f 100644 --- a/crates/build/src/docker.rs +++ b/crates/build/src/docker.rs @@ -204,7 +204,7 @@ fn update_build_result(host_folder: &Path, build_result: &mut BuildResult) -> Re }); build_result.dest_wasm = new_path; - build_result.metadata_result.as_mut().map(|mut m| { + build_result.metadata_result.as_mut().map(|m| { m.dest_bundle = host_folder.join( m.dest_bundle .as_path()