Consensus verification failure

[hu55a1n1]

Crate

ibc

Summary of Bug

Module verification fails to verify correct consensus proofs from gaiad.

Version

master

Steps to Reproduce

To test with basecoin-rs ->

1. Checkout the hu55a1n1/module-verification branch
2. Try to create a connection between the basecoin chain and a gaia chain using these instructions -> Phase4 informalsystems/basecoin-rs#20 (comment)

$ RUST_BACKTRACE=1 hermes create connection ibc-0 basecoin-0
# ...
2022-01-05T10:53:10.739840Z ERROR ThreadId(01) Failed ConnTry ConnectionSide { chain: ProdChainHandle { chain_id: ChainId { id: "basecoin-0", version: 0 }, runtime_sender: Sender { .. } }, client_id: ClientId("07-tendermint-0"), connection_id: None }: 
   0: tx response event consists of an error: deliver_tx for A1426866A616BE813907ED5042011F41612627A633E74CD07B7CEAE75F353D52 reports error: code=Err(2), log=Log("deliver failed with error: IBC module error\n\nCaused by:\n    ICS03 connection error\n\n    Caused by:\n       0: the consensus proof verification failed (height: 0-10)\n       1: tendermint error\n       2: ics23 commitment error\n       3: proof verification failed\n\n    Location:\n        /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/flex-error-0.4.4/src/tracer_impl/eyre.rs:10:9\n\n    Stack backtrace:\n       0: eyre::DefaultHandler::default_with\n                 at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/eyre-0.6.5/src/lib.rs:689:25\n       1: core::ops::function::Fn::call\n                 at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/core/src/ops/function.rs:70:5\n       2: eyre::capture_handler\n                 at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/eyre-0.6.5/src/lib.rs:551:23\n       3: eyre::error::<impl eyre::Report>::from_adhoc\n                 at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/eyre-0.6.5/src/error.rs:113:28\n       4: eyre::error::<impl eyre::Report>::msg\n                 at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/eyre-0.6.5/src/error.rs:72:9\n       5: flex_error::tracer_impl::eyre::<impl flex_error::tracer::ErrorMessageTracer for eyre::Report>::new_message\n                 at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/flex-error-0.4.4/src/tracer_impl/eyre.rs:10:9\n       6: ibc::core::ics23_commitment::error::Error::verification_failure\n                 at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/flex-error-0.4.4/src/macros.rs:823:21\n       7: ibc::core::ics23_commitment::merkle::MerkleProof::verify_membership\n                 at /home/hussaini/Documents/ibc-rs/modules/src/core/ics23_commitment/merkle.rs:111:24\n       8: ibc::clients::ics07_tendermint::client_def::verify_membership\n                 at /home/hussaini/Documents/ibc-rs/modules/src/clients/ics07_tendermint/client_def.rs:419:5\n       9: <ibc::clients::ics07_tendermint::client_def::TendermintClient as ibc::core::ics02_client::client_def::ClientDef>::verify_client_consensus_state\n                 at /home/hussaini/Documents/ibc-rs/modules/src/clients/ics07_tendermint/client_def.rs:212:9\n      10: <ibc::core::ics02_client::client_def::AnyClient as ibc::core::ics02_client::client_def::ClientDef>::verify_client_consensus_state\n                 at /home/hussaini/Documents/ibc-rs/modules/src/core/ics02_client/client_def.rs:257:17\n      11: ibc::core::ics03_connection::handler::verify::verify_consensus_proof\n                 at /home/hussaini/Documents/ibc-rs/modules/src/core/ics03_connection/handler/verify.rs:158:5\n      12: ibc::core::ics03_connection::handler::verify::verify_proofs\n                 at /home/hussaini/Documents/ibc-rs/modules/src/core/ics03_connection/handler/verify.rs:48:12\n      13: ibc::core::ics03_connection::handler::conn_open_try::process\n                 at /home/hussaini/Documents/ibc-rs/modules/src/core/ics03_connection/handler/conn_open_try.rs:81:5\n      14: ibc::core::ics03_connection::handler::dispatch\n                 at /home/hussaini/Documents/ibc-rs/modules/src/core/ics03_connection/handler.rs:53:50\n      15: ibc::core::ics26_routing::handler::dispatch\n                 at /home/hussaini/Documents/ibc-rs/modules/src/core/ics26_routing/handler.rs:71:34\n      16: <tendermint_basecoin::app::modules::ibc::Ibc<S> as tendermint_basecoin::app::modules::Module>::deliver\n                 at ./src/app/modules/ibc.rs:806:15\n      17: tendermint_basecoin::app::BaseCoinApp<S>::deliver_msg\n                 at ./src/app/mod.rs:123:19\n      18: <tendermint_basecoin::app::BaseCoinApp<S> as tendermint_abci::application::Application>::deliver_tx\n                 at ./src/app/mod.rs:263:19\n      19: <A as tendermint_abci::application::RequestDispatcher>::handle\n                 at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/tendermint-abci-0.23.2/src/application.rs:132:69\n      20: tendermint_abci::server::Server<App>::handle_client\n                 at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/tendermint-abci-0.23.2/src/server.rs:114:28\n      21: tendermint_abci::server::Server<App>::spawn_client_handler::{{closure}}\n                 at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/tendermint-abci-0.23.2/src/server.rs:91:39\n      22: std::sys_common::backtrace::__rust_begin_short_backtrace\n                 at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/std/src/sys_common/backtrace.rs:123:18\n      23: std::thread::Builder::spawn_unchecked::{{closure}}::{{closure}}\n                 at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/std/src/thread/mod.rs:477:17\n      24: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n                 at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/core/src/panic/unwind_safe.rs:271:9\n      25: std::panicking::try::do_call\n                 at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/std/src/panicking.rs:406:40\n      26: __rust_try\n      27: std::panicking::try\n                 at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/std/src/panicking.rs:370:19\n      28: std::panic::catch_unwind\n                 at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/std/src/panic.rs:133:14\n      29: std::thread::Builder::spawn_unchecked::{{closure}}\n                 at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/std/src/thread/mod.rs:476:30\n      30: core::ops::function::FnOnce::call_once{{vtable.shim}}\n                 at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/core/src/ops/function.rs:227:5\n      31: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n                 at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/alloc/src/boxed.rs:1811:9\n      32: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n                 at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/alloc/src/boxed.rs:1811:9\n      33: std::sys::unix::thread::Thread::new::thread_start\n                 at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/std/src/sys/unix/thread.rs:108:17\n      34: start_thread\n                 at /build/glibc-eX1tMB/glibc-2.31/nptl/pthread_create.c:477:8\n      35: clone\n\n\nLocation:\n    /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/flex-error-0.4.4/src/tracer_impl/eyre.rs:30:9\n\nStack backtrace:\n   0: eyre::DefaultHandler::default_with\n             at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/eyre-0.6.5/src/lib.rs:689:25\n   1: core::ops::function::Fn::call\n             at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/core/src/ops/function.rs:70:5\n   2: eyre::capture_handler\n             at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/eyre-0.6.5/src/lib.rs:551:23\n   3: eyre::error::<impl eyre::Report>::from_adhoc\n             at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/eyre-0.6.5/src/error.rs:113:28\n   4: eyre::error::<impl eyre::Report>::msg\n             at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/eyre-0.6.5/src/error.rs:72:9\n   5: flex_error::tracer_impl::eyre::<impl flex_error::tracer::ErrorTracer<E> for eyre::Report>::new_trace\n             at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/flex-error-0.4.4/src/tracer_impl/eyre.rs:30:9\n   6: <flex_error::source::TraceError<E> as flex_error::source::ErrorSource<Tracer>>::error_details\n             at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/flex-error-0.4.4/src/source.rs:170:21\n   7: tendermint_basecoin::app::modules::Error::trace_from\n             at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/flex-error-0.4.4/src/macros.rs:547:39\n   8: tendermint_basecoin::app::modules::Error::ibc\n             at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/flex-error-0.4.4/src/macros.rs:863:40\n   9: <tendermint_basecoin::app::modules::ibc::Ibc<S> as tendermint_basecoin::app::modules::Module>::deliver\n             at ./src/app/modules/ibc.rs:812:27\n  10: tendermint_basecoin::app::BaseCoinApp<S>::deliver_msg\n             at ./src/app/mod.rs:123:19\n  11: <tendermint_basecoin::app::BaseCoinApp<S> as tendermint_abci::application::Application>::deliver_tx\n             at ./src/app/mod.rs:263:19\n  12: <A as tendermint_abci::application::RequestDispatcher>::handle\n             at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/tendermint-abci-0.23.2/src/application.rs:132:69\n  13: tendermint_abci::server::Server<App>::handle_client\n             at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/tendermint-abci-0.23.2/src/server.rs:114:28\n  14: tendermint_abci::server::Server<App>::spawn_client_handler::{{closure}}\n             at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/tendermint-abci-0.23.2/src/server.rs:91:39\n  15: std::sys_common::backtrace::__rust_begin_short_backtrace\n             at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/std/src/sys_common/backtrace.rs:123:18\n  16: std::thread::Builder::spawn_unchecked::{{closure}}::{{closure}}\n             at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/std/src/thread/mod.rs:477:17\n  17: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/core/src/panic/unwind_safe.rs:271:9\n  18: std::panicking::try::do_call\n             at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/std/src/panicking.rs:406:40\n  19: __rust_try\n  20: std::panicking::try\n             at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/std/src/panicking.rs:370:19\n  21: std::panic::catch_unwind\n             at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/std/src/panic.rs:133:14\n  22: std::thread::Builder::spawn_unchecked::{{closure}}\n             at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/std/src/thread/mod.rs:476:30\n  23: core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/core/src/ops/function.rs:227:5\n  24: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/alloc/src/boxed.rs:1811:9\n  25: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/alloc/src/boxed.rs:1811:9\n  26: std::sys::unix::thread::Thread::new::thread_start\n             at /rustc/cfa3fe5af339e724209b25715282adae0c61628f/library/std/src/sys/unix/thread.rs:108:17\n  27: start_thread\n             at /build/glibc-eX1tMB/glibc-2.31/nptl/pthread_create.c:477:8\n  28: clone\n")

Location:
   /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/flex-error-0.4.4/src/tracer_impl/eyre.rs:10

  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ BACKTRACE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
                                ⋮ 5 frames hidden ⋮                               
   6: flex_error::tracer_impl::eyre::<impl flex_error::tracer::ErrorMessageTracer for eyre::Report>::new_message::h11e7ae1769953703
      at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/flex-error-0.4.4/src/tracer_impl/eyre.rs:10
   7: ibc_relayer::connection::ConnectionError::tx_response::hdf979b8d029274fb
      at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/flex-error-0.4.4/src/macros.rs:823
   8: ibc_relayer::connection::Connection<ChainA,ChainB>::build_conn_try_and_send::h8ebfc3d6a942804b
      at /home/hussaini/Documents/ibc-rs/relayer/src/connection.rs:913
   9: ibc_relayer::connection::Connection<ChainA,ChainB>::handshake::h2908b1f68e3fb673
      at /home/hussaini/Documents/ibc-rs/relayer/src/connection.rs:513
  10: ibc_relayer::connection::Connection<ChainA,ChainB>::new::hd9c71f2c8cfd7192
      at /home/hussaini/Documents/ibc-rs/relayer/src/connection.rs:281
  11: ibc_relayer_cli::commands::create::connection::CreateConnectionCommand::run_using_new_clients::hbcaef0edaae8d96c
      at /home/hussaini/Documents/ibc-rs/relayer-cli/src/commands/create/connection.rs:93
  12: <ibc_relayer_cli::commands::create::connection::CreateConnectionCommand as abscissa_core::runnable::Runnable>::run::h6601230148e4b942
      at /home/hussaini/Documents/ibc-rs/relayer-cli/src/commands/create/connection.rs:53
  13: ibc_relayer_cli::commands::create::_DERIVE_Runnable_FOR_CreateCmds::<impl abscissa_core::runnable::Runnable for ibc_relayer_cli::commands::create::CreateCmds>::run::h3852698ad88ae6a0
      at /home/hussaini/Documents/ibc-rs/relayer-cli/src/commands/create.rs:12
  14: ibc_relayer_cli::commands::_DERIVE_Runnable_FOR_CliCmd::<impl abscissa_core::runnable::Runnable for ibc_relayer_cli::commands::CliCmd>::run::h847a46cb9bf02e87
      at /home/hussaini/Documents/ibc-rs/relayer-cli/src/commands.rs:42
  15: <ibc_relayer_cli::entry::EntryPoint as abscissa_core::runnable::Runnable>::run::h6d7d9dbb4c247b63
      at /home/hussaini/Documents/ibc-rs/relayer-cli/src/entry.rs:33
  16: abscissa_core::application::Application::run::h0b3a2a4a06e6c6a9
      at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/abscissa_core-0.6.0-beta.1/src/application.rs:60
  17: abscissa_core::application::boot::hd5255c0116a4d0c5
      at /home/hussaini/.cargo/registry/src/github.com-1ecc6299db9ec823/abscissa_core-0.6.0-beta.1/src/application.rs:175
  18: hermes::main::hfafdd5f872202568
      at /home/hussaini/Documents/ibc-rs/relayer-cli/src/bin/hermes/main.rs:12
  19: core::ops::function::FnOnce::call_once::h4627ebf198cd8f7b
      at /rustc/f1edd0429582dd29cccacaf50fd134b05593bd9c/library/core/src/ops/function.rs:227
                                ⋮ 15 frames hidden ⋮                              

Run with COLORBT_SHOW_HIDDEN=1 environment variable to disable frame filtering.
Run with RUST_BACKTRACE=full to include source snippets.

Acceptance Criteria

Consensus proof verification succeeds.

---

For Admin Use

• Not duplicate issue
• Appropriate labels applied
• Appropriate milestone (priority) applied
• Appropriate contributors tagged
• Contributor assigned/self-assigned

[hu55a1n1]

Some findings ->

1. The failure happens when trying to call ibc::core::ics23_commitment::merkle::MerkleProof::verify_membership() while trying to verify_client_consensus_state() because the final calculated hash doesn't match the root hash.
2. The merkle path seems to be alright ->

MerklePath {
    key_path: [
        "ibc",
        "clients/07-tendermint-0/consensusStates/0-10",
    ],
}

3. Looks like the consensus state for basecoin-0 stored on the gaiad chain ibc-0 matches the host consensus state exactly. Verified as follows ->

$ curl "http://localhost:26657/abci_query?path=\"store/ibc/key\"&data=\"clients/07-tendermint-0/consensusStates/0-10\"&height=976&prove=true" | jq
{
  "jsonrpc": "2.0",
  "id": -1,
  "result": {
    "response": {
      "code": 0,
      "log": "",
      "info": "",
      "index": "0",
      "key": "Y2xpZW50cy8wNy10ZW5kZXJtaW50LTAvY29uc2Vuc3VzU3RhdGVzLzAtMTA=",
      "value": "Ci4vaWJjLmxpZ2h0Y2xpZW50cy50ZW5kZXJtaW50LnYxLkNvbnNlbnN1c1N0YXRlElQKDAiR8dWOBhDnz4/BAhIiCiCXJkGhFY+fsTBlzNu5V+eoArxRBfUrGeviZrSX3IdGxxog0VWp+Tma6TLMfEBOBaKp7qIGjKV3JEYmHK7pEyvIaNE=",
      "proofOps": {
        "ops": [
          {
            "type": "ics23:iavl",
            "key": "Y2xpZW50cy8wNy10ZW5kZXJtaW50LTAvY29uc2Vuc3VzU3RhdGVzLzAtMTA=",
            "data": "CvsCCixjbGllbnRzLzA3LXRlbmRlcm1pbnQtMC9jb25zZW5zdXNTdGF0ZXMvMC0xMBKGAQouL2liYy5saWdodGNsaWVudHMudGVuZGVybWludC52MS5Db25zZW5zdXNTdGF0ZRJUCgwIkfHVjgYQ58+PwQISIgoglyZBoRWPn7EwZczbuVfnqAK8UQX1Kxnr4ma0l9yHRscaINFVqfk5mukyzHxATgWiqe6iBoyldyRGJhyu6RMryGjRGgwIARgBIAEqBAACjgQiLAgBEgUCBJYEIBohILfdYaL9qCfwNbCJkaq3uQvaEmNBoNGNmw01WTUrz7tkIioIARImBAiWBCDNAwXOxLlFHHeJqfOdZcKjC76nzRahRSFzzTG3O7O5fiAiLAgBEgUGDpYEIBohICTItZCLYCtVmw7Z6GdmUgwIN7SHSf83BUGP2/59+XrpIiwIARIFCBiWBCAaISDV2CipvgDjQLINM6r8gSNMp5hLfII3zm37YzdFIIW4ZQ=="
          },
          {
            "type": "ics23:simple",
            "key": "aWJj",
            "data": "CtIBCgNpYmMSIJIXrxNUY3zvvtvzIwyd3vjEF1uZCyzQUTRAHS42TJSWGgkIARgBIAEqAQAiJwgBEgEBGiC7WEXil5YOS01FaKW7trW7P/4keJnqJu1TsUhfKQvdnyIlCAESIQHE9fPjQgNd+1LyG5frMQ5vVjJeKlhk4wlb8lEMwdZLMSIlCAESIQFgYVMHv9X4KGvV315nwjujk3GrxaXh6OAqwIelEcOSBiInCAESAQEaIL/FdNVXiVe58tFaefjvqnHXkrBhBVZSrWDOUDGU8+oC"
          }
        ]
      },
      "height": "976",
      "codespace": ""
    }
  }
}
$ echo "Ci4vaWJjLmxpZ2h0Y2xpZW50cy50ZW5kZXJtaW50LnYxLkNvbnNlbnN1c1N0YXRlElQKDAiR8dWOBhDnz4/BAhIiCiCXJkGhFY+fsTBlzNu5V+eoArxRBfUrGeviZrSX3IdGxxog0VWp+Tma6TLMfEBOBaKp7qIGjKV3JEYmHK7pEyvIaNE=" | base64 -d | ibc-proto-decode
Tendermint(
    ConsensusState {
        timestamp: Time(
            PrimitiveDateTime {
                date: Date {
                    year: 2022,
                    ordinal: 5,
                },
                time: Time {
                    hour: 10,
                    minute: 53,
                    second: 5,
                    nanosecond: 673441767,
                },
            },
        ),
        root: CommitmentRoot(
            "972641A1158F9FB13065CCDBB957E7A802BC5105F52B19EBE266B497DC8746C7",
        ),
        next_validators_hash: Hash::Sha256(D155A9F9399AE932CC7C404E05A2A9EEA2068CA5772446261CAEE9132BC868D1),
    },
)

And here's basecoin-0's consensus state for the same height ->

ConsensusState {
    timestamp: Time(
        PrimitiveDateTime {
            date: Date {
                year: 2022,
                ordinal: 5,
            },
            time: Time {
                hour: 10,
                minute: 53,
                second: 5,
                nanosecond: 673441767,
            },
        },
    ),
    root: CommitmentRoot(
        "972641A1158F9FB13065CCDBB957E7A802BC5105F52B19EBE266B497DC8746C7",
    ),
    next_validators_hash: Hash::Sha256(D155A9F9399AE932CC7C404E05A2A9EEA2068CA5772446261CAEE9132BC868D1),
}

4. It is unlikely that there's a problem in the verificaiton code because the verificaiton is otherwise working fine for client-state, connection, etc.

PS: ibc-proto-decode is a small utility I wrote which is just decoding protobug bin ->

use std::io::{self, Read};
use tendermint_proto::Protobuf;
use ibc::core::ics02_client::client_consensus::AnyConsensusState;

fn main() {
    let stdin = io::stdin();
    let mut buf = vec![];
    stdin.lock().read_to_end(&mut buf).expect("failed to read stdin");
    println!("{:#?}", AnyConsensusState::decode_vec(&buf).expect("failed to parse input"));
}