From db27e4047fadac94e36733edff87d5bb05305f2a Mon Sep 17 00:00:00 2001 From: Rim Rakhimov Date: Wed, 20 Dec 2023 17:41:07 +0300 Subject: [PATCH] [SIG_PROVIDER] Add eth-bytecode-db source. Add batched event retrievals (#707) * Add event descriptions search * Insertion of the event signatures into database implementation * Add tests * Bump blockscout-service-launcher to v0.9.0. Add BatchGetEventAbis method template * Update existing events parser to support eth-bytecode-db * Add batch_get_event_abi implementation * Update BatchGetEventAbis proto to bypass initial 'responses' object. Make requests in batch_get_event_abis concurrent * Update tests * Add BatchSearchEventDescriptions endpoint definition. Add test cases for batch-search * Add BatchSearchEventDescriptions implementation * Make use of batch search * Remove supports_batched method --- sig-provider/Cargo.lock | 656 +++++++++++++++++- sig-provider/sig-provider-proto/build.rs | 3 +- .../proto/api_config_http.yaml | 5 + .../proto/sig-provider.proto | 10 + .../swagger/sig-provider.swagger.yaml | 71 +- sig-provider/sig-provider-server/Cargo.toml | 5 +- .../sig-provider-server/config/base.toml | 11 +- sig-provider/sig-provider-server/src/main.rs | 3 +- sig-provider/sig-provider-server/src/run.rs | 83 +-- .../sig-provider-server/src/service.rs | 70 +- .../sig-provider-server/src/settings.rs | 96 +-- .../sig-provider-server/tests/mock.rs | 262 ++++--- .../sig-provider-server/tests/settings.rs | 3 +- sig-provider/sig-provider/Cargo.toml | 4 +- sig-provider/sig-provider/src/aggregator.rs | 187 ++++- sig-provider/sig-provider/src/lib.rs | 2 +- .../src/sources/eth_bytecode_db.rs | 165 +++++ sig-provider/sig-provider/src/sources/mod.rs | 35 + .../sig-provider/src/sources/sigeth.rs | 12 +- 19 files changed, 1387 insertions(+), 296 deletions(-) create mode 100644 sig-provider/sig-provider/src/sources/eth_bytecode_db.rs diff --git a/sig-provider/Cargo.lock b/sig-provider/Cargo.lock index e5830aa0d..42647bfab 100644 --- a/sig-provider/Cargo.lock +++ b/sig-provider/Cargo.lock @@ -296,6 +296,55 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "alloy-json-abi" +version = "0.5.2" +source = "git+https://github.com/alloy-rs/core?rev=398d7e2#398d7e27b235f85f776c1b6b8fe2c9b081f2907a" +dependencies = [ + "alloy-primitives", + "alloy-sol-type-parser", + "serde", + "serde_json", +] + +[[package]] +name = "alloy-primitives" +version = "0.5.2" +source = "git+https://github.com/alloy-rs/core?rev=398d7e2#398d7e27b235f85f776c1b6b8fe2c9b081f2907a" +dependencies = [ + "alloy-rlp", + "bytes", + "cfg-if", + "const-hex", + "derive_more", + "hex-literal", + "itoa", + "proptest", + "rand", + "ruint", + "serde", + "tiny-keccak", +] + +[[package]] +name = "alloy-rlp" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc0fac0fc16baf1f63f78b47c3d24718f3619b0714076f6a02957d808d52cbef" +dependencies = [ + "arrayvec", + "bytes", + "smol_str", +] + +[[package]] +name = "alloy-sol-type-parser" +version = "0.5.2" +source = "git+https://github.com/alloy-rs/core?rev=398d7e2#398d7e27b235f85f776c1b6b8fe2c9b081f2907a" +dependencies = [ + "winnow", +] + [[package]] name = "android-tzdata" version = "0.1.1" @@ -317,6 +366,130 @@ version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +[[package]] +name = "ark-ff" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b3235cc41ee7a12aaaf2c575a2ad7b46713a8a50bda2fc3b003a04845c05dd6" +dependencies = [ + "ark-ff-asm 0.3.0", + "ark-ff-macros 0.3.0", + "ark-serialize 0.3.0", + "ark-std 0.3.0", + "derivative", + "num-bigint", + "num-traits", + "paste", + "rustc_version 0.3.3", + "zeroize", +] + +[[package]] +name = "ark-ff" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec847af850f44ad29048935519032c33da8aa03340876d351dfab5660d2966ba" +dependencies = [ + "ark-ff-asm 0.4.2", + "ark-ff-macros 0.4.2", + "ark-serialize 0.4.2", + "ark-std 0.4.0", + "derivative", + "digest 0.10.7", + "itertools", + "num-bigint", + "num-traits", + "paste", + "rustc_version 0.4.0", + "zeroize", +] + +[[package]] +name = "ark-ff-asm" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db02d390bf6643fb404d3d22d31aee1c4bc4459600aef9113833d17e786c6e44" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ark-ff-asm" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed4aa4fe255d0bc6d79373f7e31d2ea147bcf486cba1be5ba7ea85abdb92348" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ark-ff-macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fd794a08ccb318058009eefdf15bcaaaaf6f8161eb3345f907222bac38b20" +dependencies = [ + "num-bigint", + "num-traits", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ark-ff-macros" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7abe79b0e4288889c4574159ab790824d0033b9fdcb2a112a3182fac2e514565" +dependencies = [ + "num-bigint", + "num-traits", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ark-serialize" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d6c2b318ee6e10f8c2853e73a83adc0ccb88995aa978d8a3408d492ab2ee671" +dependencies = [ + "ark-std 0.3.0", + "digest 0.9.0", +] + +[[package]] +name = "ark-serialize" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb7b85a02b83d2f22f89bd5cac66c9c89474240cb6207cb1efc16d098e822a5" +dependencies = [ + "ark-std 0.4.0", + "digest 0.10.7", + "num-bigint", +] + +[[package]] +name = "ark-std" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1df2c09229cbc5a028b1d70e00fdb2acee28b1055dfb5ca73eea49c5a25c4e7c" +dependencies = [ + "num-traits", + "rand", +] + +[[package]] +name = "ark-std" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94893f1e0c6eeab764ade8dc4c0db24caf4fe7cbbaafc0eba0a9030f447b5185" +dependencies = [ + "num-traits", + "rand", +] + [[package]] name = "arrayvec" version = "0.7.4" @@ -521,6 +694,18 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1181e1e0d1fce796a03db1ae795d67167da795f9cf4a39c37589e85ef57f26d3" +[[package]] +name = "auto_impl" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fee3da8ef1276b0bee5dd1c7258010d8fffd31801447323115a25560e1327b89" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -675,22 +860,24 @@ dependencies = [ [[package]] name = "blockscout-service-launcher" -version = "0.1.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce1834d52614001133c0f10dac910cb3bba8f1755ac0e1046855ceff4400054" +checksum = "662512331ca49faf635aab7d25d29b2e66c020a807354d8cbd3d8a921600aa24" dependencies = [ "actix-web", "actix-web-prom", "anyhow", + "config", "futures", - "opentelemetry", - "opentelemetry-jaeger", + "opentelemetry 0.19.0", + "opentelemetry-jaeger 0.18.0", "prometheus", + "reqwest", "serde", "tokio", "tonic", "tracing", - "tracing-opentelemetry", + "tracing-opentelemetry 0.19.0", "tracing-subscriber", ] @@ -738,6 +925,9 @@ name = "bytes" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +dependencies = [ + "serde", +] [[package]] name = "bytestring" @@ -811,6 +1001,19 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "const-hex" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5104de16b218eddf8e34ffe2f86f74bfa4e61e95a1b89732fccf6325efd0557" +dependencies = [ + "cfg-if", + "cpufeatures", + "hex", + "proptest", + "serde", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -985,6 +1188,17 @@ dependencies = [ "serde", ] +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -994,7 +1208,7 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "rustc_version", + "rustc_version 0.4.0", "syn 1.0.109", ] @@ -1010,6 +1224,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + [[package]] name = "digest" version = "0.10.7" @@ -1106,9 +1329,9 @@ dependencies = [ [[package]] name = "ethabi" -version = "17.2.0" +version = "18.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4966fba78396ff92db3b817ee71143eccd98acf0f876b8d600e585a670c5d1b" +checksum = "7413c5f74cc903ea37386a8965a936cbeb334bd270862fdece542c1b2dcbc898" dependencies = [ "ethereum-types", "hex", @@ -1123,9 +1346,9 @@ dependencies = [ [[package]] name = "ethbloom" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11da94e443c60508eb62cf256243a64da87304c2802ac2528847f79d750007ef" +checksum = "c22d4b5885b6aa2fe5e8b9329fb8d232bf739e434e6b87347c63bdd00c120f60" dependencies = [ "crunchy", "fixed-hash", @@ -1136,9 +1359,9 @@ dependencies = [ [[package]] name = "ethereum-types" -version = "0.13.1" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2827b94c556145446fcce834ca86b7abf0c39a805883fe20e72c5bfdb5a0dc6" +checksum = "02d215cbf040552efcbe99a38372fe80ab9d00268e20012b79fcd0f073edd8ee" dependencies = [ "ethbloom", "fixed-hash", @@ -1169,11 +1392,22 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +[[package]] +name = "fastrlp" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139834ddba373bbdd213dffe02c8d110508dcf1726c2be27e8d1f7d7e1856418" +dependencies = [ + "arrayvec", + "auto_impl", + "bytes", +] + [[package]] name = "fixed-hash" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" +checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534" dependencies = [ "byteorder", "rand", @@ -1448,6 +1682,15 @@ name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +dependencies = [ + "serde", +] + +[[package]] +name = "hex-literal" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" [[package]] name = "http" @@ -1619,9 +1862,9 @@ dependencies = [ [[package]] name = "impl-serde" -version = "0.3.2" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4551f042f3438e64dbd6226b20527fc84a6e1fe65688b58746a2f53623f25f5c" +checksum = "ebc88fc67028ae3db0c853baa36269d398d5f45b6982f95549ff5def78c935cd" dependencies = [ "serde", ] @@ -1845,6 +2088,12 @@ version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + [[package]] name = "libnghttp2-sys" version = "0.1.8+1.55.1" @@ -2069,6 +2318,27 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-bigint" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.16" @@ -2076,6 +2346,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -2153,8 +2424,18 @@ version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69d6c3d7288a106c0a363e4b0e8d308058d56902adefb16f4936f417ffef086e" dependencies = [ - "opentelemetry_api", - "opentelemetry_sdk", + "opentelemetry_api 0.18.0", + "opentelemetry_sdk 0.18.0", +] + +[[package]] +name = "opentelemetry" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f4b8347cc26099d3aeee044065ecc3ae11469796b4d65d065a23a584ed92a6f" +dependencies = [ + "opentelemetry_api 0.19.0", + "opentelemetry_sdk 0.19.0", ] [[package]] @@ -2167,10 +2448,27 @@ dependencies = [ "futures", "futures-executor", "once_cell", - "opentelemetry", - "opentelemetry-semantic-conventions", + "opentelemetry 0.18.0", + "opentelemetry-semantic-conventions 0.10.0", "thiserror", - "thrift", + "thrift 0.16.0", + "tokio", +] + +[[package]] +name = "opentelemetry-jaeger" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08e028dc9f4f304e9320ce38c80e7cf74067415b1ad5a8750a38bae54a4d450d" +dependencies = [ + "async-trait", + "futures", + "futures-executor", + "once_cell", + "opentelemetry 0.19.0", + "opentelemetry-semantic-conventions 0.11.0", + "thiserror", + "thrift 0.17.0", "tokio", ] @@ -2180,7 +2478,16 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b02e0230abb0ab6636d18e2ba8fa02903ea63772281340ccac18e0af3ec9eeb" dependencies = [ - "opentelemetry", + "opentelemetry 0.18.0", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e33428e6bf08c6f7fcea4ddb8e358fab0fe48ab877a87c70c6ebe20f673ce5" +dependencies = [ + "opentelemetry 0.19.0", ] [[package]] @@ -2199,6 +2506,22 @@ dependencies = [ "thiserror", ] +[[package]] +name = "opentelemetry_api" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed41783a5bf567688eb38372f2b7a8530f5a607a4b49d38dd7573236c23ca7e2" +dependencies = [ + "fnv", + "futures-channel", + "futures-util", + "indexmap 1.9.3", + "once_cell", + "pin-project-lite", + "thiserror", + "urlencoding", +] + [[package]] name = "opentelemetry_sdk" version = "0.18.0" @@ -2213,7 +2536,29 @@ dependencies = [ "futures-executor", "futures-util", "once_cell", - "opentelemetry_api", + "opentelemetry_api 0.18.0", + "percent-encoding", + "rand", + "thiserror", + "tokio", + "tokio-stream", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b3a2a91fdbfdd4d212c0dcc2ab540de2c2bcbbd90be17de7a7daf8822d010c1" +dependencies = [ + "async-trait", + "crossbeam-channel", + "dashmap", + "fnv", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry_api 0.19.0", "percent-encoding", "rand", "thiserror", @@ -2230,6 +2575,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-multimap" version = "0.4.3" @@ -2500,9 +2854,9 @@ dependencies = [ [[package]] name = "primitive-types" -version = "0.11.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a" +checksum = "0b34d9fd68ae0b74a41b21c03c2f62847aa0ffea044eee893b4c140b37e244e2" dependencies = [ "fixed-hash", "impl-codec", @@ -2521,6 +2875,30 @@ dependencies = [ "toml_edit", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" version = "1.0.66" @@ -2545,6 +2923,26 @@ dependencies = [ "thiserror", ] +[[package]] +name = "proptest" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.4.0", + "lazy_static", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax 0.8.2", + "rusty-fork", + "tempfile", + "unarray", +] + [[package]] name = "prost" version = "0.11.9" @@ -2605,6 +3003,12 @@ version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "1.0.33" @@ -2650,6 +3054,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -2723,6 +3136,12 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + [[package]] name = "reqwest" version = "0.11.20" @@ -2838,7 +3257,7 @@ dependencies = [ "futures", "futures-timer", "rstest_macros", - "rustc_version", + "rustc_version 0.4.0", ] [[package]] @@ -2850,11 +3269,41 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "rustc_version", + "rustc_version 0.4.0", "syn 1.0.109", "unicode-ident", ] +[[package]] +name = "ruint" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608a5726529f2f0ef81b8fde9873c4bb829d6b5b5ca6be4d97345ddf0749c825" +dependencies = [ + "alloy-rlp", + "ark-ff 0.3.0", + "ark-ff 0.4.2", + "bytes", + "fastrlp", + "num-bigint", + "num-traits", + "parity-scale-codec", + "primitive-types", + "proptest", + "rand", + "rlp", + "ruint-macro", + "serde", + "valuable", + "zeroize", +] + +[[package]] +name = "ruint-macro" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e666a5496a0b2186dbcd0ff6106e29e093c15591bde62c20d3842007c6978a09" + [[package]] name = "rust-ini" version = "0.18.0" @@ -2877,13 +3326,22 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" +[[package]] +name = "rustc_version" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" +dependencies = [ + "semver 0.11.0", +] + [[package]] name = "rustc_version" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver", + "semver 1.0.18", ] [[package]] @@ -2919,6 +3377,18 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +[[package]] +name = "rusty-fork" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "ryu" version = "1.0.15" @@ -2963,12 +3433,30 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" +dependencies = [ + "semver-parser", +] + [[package]] name = "semver" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" +[[package]] +name = "semver-parser" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" +dependencies = [ + "pest", +] + [[package]] name = "serde" version = "1.0.188" @@ -3071,7 +3559,7 @@ checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ "cfg-if", "cpufeatures", - "digest", + "digest 0.10.7", ] [[package]] @@ -3082,7 +3570,7 @@ checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" dependencies = [ "cfg-if", "cpufeatures", - "digest", + "digest 0.10.7", ] [[package]] @@ -3091,7 +3579,7 @@ version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" dependencies = [ - "digest", + "digest 0.10.7", "keccak", ] @@ -3108,6 +3596,7 @@ dependencies = [ name = "sig-provider" version = "0.1.0" dependencies = [ + "alloy-json-abi", "anyhow", "async-recursion", "async-trait", @@ -3125,6 +3614,7 @@ dependencies = [ "serde_json", "sig-provider-proto", "tokio", + "tokio-stream", "tracing", "url", ] @@ -3162,8 +3652,8 @@ dependencies = [ "hex", "http", "httpmock", - "opentelemetry", - "opentelemetry-jaeger", + "opentelemetry 0.18.0", + "opentelemetry-jaeger 0.17.0", "pretty_assertions", "prometheus", "serde", @@ -3175,7 +3665,7 @@ dependencies = [ "toml", "tonic", "tracing", - "tracing-opentelemetry", + "tracing-opentelemetry 0.18.0", "tracing-subscriber", "url", ] @@ -3237,6 +3727,15 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +[[package]] +name = "smol_str" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74212e6bbe9a4352329b2f68ba3130c15a3f26fe88ff22dbdc6cdd58fa85e99c" +dependencies = [ + "serde", +] + [[package]] name = "socket2" version = "0.4.9" @@ -3403,7 +3902,20 @@ dependencies = [ "byteorder", "integer-encoding", "log", - "ordered-float", + "ordered-float 1.1.1", + "threadpool", +] + +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "log", + "ordered-float 2.10.1", "threadpool", ] @@ -3699,13 +4211,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21ebb87a95ea13271332df069020513ab70bdb5637ca42d6e492dc3bbbad48de" dependencies = [ "once_cell", - "opentelemetry", + "opentelemetry 0.18.0", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00a39dcf9bfc1742fa4d6215253b33a6e474be78275884c216fc2a06267b3600" +dependencies = [ + "once_cell", + "opentelemetry 0.19.0", "tracing", "tracing-core", "tracing-log", "tracing-subscriber", ] +[[package]] +name = "tracing-serde" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.17" @@ -3716,12 +4252,15 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] @@ -3754,6 +4293,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicase" version = "2.7.0" @@ -3808,6 +4353,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "valuable" version = "0.1.0" @@ -3832,6 +4383,15 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + [[package]] name = "waker-fn" version = "1.1.0" @@ -4039,9 +4599,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" -version = "0.5.15" +version = "0.5.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" +checksum = "6c830786f7720c2fd27a1a0e27a709dbd3c4d009b56d098fc742d4f4eab91fe2" dependencies = [ "memchr", ] @@ -4080,6 +4640,26 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.29", +] + [[package]] name = "zstd" version = "0.12.4" diff --git a/sig-provider/sig-provider-proto/build.rs b/sig-provider/sig-provider-proto/build.rs index fdf2f616f..8c76fd47c 100644 --- a/sig-provider/sig-provider-proto/build.rs +++ b/sig-provider/sig-provider-proto/build.rs @@ -16,7 +16,8 @@ fn compile( .protoc_arg("--openapiv2_opt") .protoc_arg("grpc_api_configuration=proto/api_config_http.yaml,output_format=yaml,allow_merge=true,merge_file_name=sig-provider") .bytes(["."]) - .type_attribute(".", "#[actix_prost_macros::serde]"); + .type_attribute(".", "#[actix_prost_macros::serde]") + .message_attribute(".", "#[derive(Eq, Hash)]"); config.compile_protos(protos, includes)?; Ok(()) } diff --git a/sig-provider/sig-provider-proto/proto/api_config_http.yaml b/sig-provider/sig-provider-proto/proto/api_config_http.yaml index 26032c9e9..6f497e289 100644 --- a/sig-provider/sig-provider-proto/proto/api_config_http.yaml +++ b/sig-provider/sig-provider-proto/proto/api_config_http.yaml @@ -13,5 +13,10 @@ http: get: /api/v1/abi/event response_body: "abi" + - selector: blockscout.sig_provider.v1.AbiService.BatchGetEventAbis + post: /api/v1/abi/events:batch-get + body: "*" + response_body: "responses" + - selector: blockscout.sig_provider.v1.Health.Check get: /health diff --git a/sig-provider/sig-provider-proto/proto/sig-provider.proto b/sig-provider/sig-provider-proto/proto/sig-provider.proto index 8621d09da..347b2ea51 100644 --- a/sig-provider/sig-provider-proto/proto/sig-provider.proto +++ b/sig-provider/sig-provider-proto/proto/sig-provider.proto @@ -12,6 +12,8 @@ service SignatureService { service AbiService { rpc GetFunctionAbi(GetFunctionAbiRequest) returns (GetFunctionAbiResponse) {} rpc GetEventAbi(GetEventAbiRequest) returns (GetEventAbiResponse) {} + + rpc BatchGetEventAbis(BatchGetEventAbisRequest) returns (BatchGetEventAbisResponse) {} } message CreateSignaturesRequest { string abi = 1; } @@ -44,3 +46,11 @@ message GetEventAbiRequest { } message GetEventAbiResponse { repeated Abi abi = 1; } + +message BatchGetEventAbisRequest { + repeated GetEventAbiRequest requests = 1; +} + +message BatchGetEventAbisResponse { + repeated GetEventAbiResponse responses = 1; +} \ No newline at end of file diff --git a/sig-provider/sig-provider-proto/swagger/sig-provider.swagger.yaml b/sig-provider/sig-provider-proto/swagger/sig-provider.swagger.yaml index ca79347b4..8938706df 100644 --- a/sig-provider/sig-provider-proto/swagger/sig-provider.swagger.yaml +++ b/sig-provider/sig-provider-proto/swagger/sig-provider.swagger.yaml @@ -20,6 +20,7 @@ paths: schema: type: array items: + type: object $ref: '#/definitions/v1Abi' default: description: An unexpected error response. @@ -37,6 +38,29 @@ paths: type: string tags: - AbiService + /api/v1/abi/events:batch-get: + post: + operationId: AbiService_BatchGetEventAbis + responses: + "200": + description: "" + schema: + type: array + items: + type: object + $ref: '#/definitions/v1GetEventAbiResponse' + default: + description: An unexpected error response. + schema: + $ref: '#/definitions/rpcStatus' + parameters: + - name: body + in: body + required: true + schema: + $ref: '#/definitions/v1BatchGetEventAbisRequest' + tags: + - AbiService /api/v1/abi/function: get: operationId: AbiService_GetFunctionAbi @@ -46,6 +70,7 @@ paths: schema: type: array items: + type: object $ref: '#/definitions/v1Abi' default: description: An unexpected error response. @@ -109,6 +134,7 @@ definitions: - NOT_SERVING - SERVICE_UNKNOWN default: UNKNOWN + description: ' - SERVICE_UNKNOWN: Used only by the Watch method.' protobufAny: type: object properties: @@ -121,38 +147,57 @@ definitions: code: type: integer format: int32 + message: + type: string details: type: array items: + type: object $ref: '#/definitions/protobufAny' - message: - type: string v1Abi: type: object properties: + name: + type: string inputs: type: array items: + type: object $ref: '#/definitions/v1Argument' - name: - type: string v1Argument: type: object properties: + name: + type: string + type: + type: string components: type: array items: + type: object $ref: '#/definitions/v1Argument' indexed: type: boolean title: this is present only in events - name: - type: string - type: - type: string value: type: string title: decoded value + v1BatchGetEventAbisRequest: + type: object + properties: + requests: + type: array + items: + type: object + $ref: '#/definitions/v1GetEventAbiRequest' + v1BatchGetEventAbisResponse: + type: object + properties: + responses: + type: array + items: + type: object + $ref: '#/definitions/v1GetEventAbiResponse' v1CreateSignaturesRequest: type: object properties: @@ -160,12 +205,21 @@ definitions: type: string v1CreateSignaturesResponse: type: object + v1GetEventAbiRequest: + type: object + properties: + data: + type: string + topics: + type: string + title: comma separated hex values, ex. `0x0000..1234,0x0000...5678` v1GetEventAbiResponse: type: object properties: abi: type: array items: + type: object $ref: '#/definitions/v1Abi' v1GetFunctionAbiResponse: type: object @@ -173,6 +227,7 @@ definitions: abi: type: array items: + type: object $ref: '#/definitions/v1Abi' v1HealthCheckResponse: type: object diff --git a/sig-provider/sig-provider-server/Cargo.toml b/sig-provider/sig-provider-server/Cargo.toml index 991c68285..173c4bbb8 100644 --- a/sig-provider/sig-provider-server/Cargo.toml +++ b/sig-provider/sig-provider-server/Cargo.toml @@ -25,10 +25,11 @@ futures = "0.3" anyhow = "1.0" url = { version = "2", features = ["serde"] } hex = "0.4" -ethabi = "17" -blockscout-service-launcher = "0.1" +ethabi = "18.0.0" +blockscout-service-launcher = "0.9.0" [dev-dependencies] +blockscout-service-launcher = { version = "*", features = ["test-server"] } http = "0.2" httpmock = "0.6" pretty_assertions = "1.3" diff --git a/sig-provider/sig-provider-server/config/base.toml b/sig-provider/sig-provider-server/config/base.toml index d454352a5..7d9d97168 100644 --- a/sig-provider/sig-provider-server/config/base.toml +++ b/sig-provider/sig-provider-server/config/base.toml @@ -1,15 +1,20 @@ [server.http] enabled = true addr = "0.0.0.0:8050" +max_body_size = 2097152 [server.grpc] -enabled = true +enabled = false addr = "0.0.0.0:8051" [sources] fourbyte = "https://www.4byte.directory/" sigeth = "https://sig.eth.samczsun.com/" +[sources.eth_bytecode_db] +enabled = true +url = "https://eth-bytecode-db.services.blockscout.com/" + [metrics] enabled = false addr = "0.0.0.0:6060" @@ -18,3 +23,7 @@ route = "/metrics" [jaeger] enabled = false agent_endpoint = "127.0.0.1:6831" + +[tracing] +enabled = true +format = "default" diff --git a/sig-provider/sig-provider-server/src/main.rs b/sig-provider/sig-provider-server/src/main.rs index 9ad65bfb1..4976ce7e7 100644 --- a/sig-provider/sig-provider-server/src/main.rs +++ b/sig-provider/sig-provider-server/src/main.rs @@ -1,7 +1,8 @@ +use blockscout_service_launcher::launcher::ConfigSettings; use sig_provider_server::{sig_provider, Settings}; #[tokio::main] async fn main() -> Result<(), anyhow::Error> { - let settings = Settings::new().expect("failed to read config"); + let settings = Settings::build().expect("failed to read config"); sig_provider(settings).await } diff --git a/sig-provider/sig-provider-server/src/run.rs b/sig-provider/sig-provider-server/src/run.rs index 09d098884..4afc331ca 100644 --- a/sig-provider/sig-provider-server/src/run.rs +++ b/sig-provider/sig-provider-server/src/run.rs @@ -1,7 +1,8 @@ use crate::{health::HealthService, settings::SourcesSettings, Service, Settings}; -use actix_web::web::ServiceConfig; -use blockscout_service_launcher::LaunchSettings; -use sig_provider::{fourbyte, sigeth, SourceAggregator}; +use blockscout_service_launcher::{launcher, launcher::LaunchSettings, tracing}; +use sig_provider::{ + eth_bytecode_db, fourbyte, sigeth, CompleteSignatureSource, SignatureSource, SourceAggregator, +}; use sig_provider_proto::blockscout::sig_provider::v1::{ abi_service_actix::route_abi_service, abi_service_server::{AbiService, AbiServiceServer}, @@ -12,67 +13,71 @@ use sig_provider_proto::blockscout::sig_provider::v1::{ }; use std::sync::Arc; -pub fn http_configure( - config: &mut ServiceConfig, - signature: Arc, - abi: Arc, -) { - route_signature_service(config, signature); - route_abi_service(config, abi); -} +const SERVICE_NAME: &str = "sig_provider"; #[derive(Clone)] -struct HttpRouter { +struct Router { signature: Arc, abi: Arc, health: Arc, } -impl blockscout_service_launcher::HttpRouter - for HttpRouter -{ +impl Router { + pub fn grpc_router(&self) -> tonic::transport::server::Router { + tonic::transport::Server::builder() + .add_service(HealthServer::from_arc(self.health.clone())) + .add_service(SignatureServiceServer::from_arc(self.signature.clone())) + .add_service(AbiServiceServer::from_arc(self.abi.clone())) + } +} + +impl launcher::HttpRouter for Router { fn register_routes(&self, service_config: &mut actix_web::web::ServiceConfig) { service_config .configure(|config| route_health(config, self.health.clone())) - .configure(|config| http_configure(config, self.signature.clone(), self.abi.clone())); + .configure(|config| route_signature_service(config, self.signature.clone())) + .configure(|config| route_abi_service(config, self.abi.clone())); } } -fn grpc_router( - signature: Arc, - abi: Arc, - health: Arc, -) -> tonic::transport::server::Router { - tonic::transport::Server::builder() - .add_service(HealthServer::from_arc(health)) - .add_service(SignatureServiceServer::from_arc(signature)) - .add_service(AbiServiceServer::from_arc(abi)) -} - -pub fn new_service(sources: SourcesSettings) -> Arc { - let aggregator = Arc::new(SourceAggregator::new(vec![ - Arc::new(sigeth::Source::new(sources.sigeth)), - Arc::new(fourbyte::Source::new(sources.fourbyte)), - ])); +pub fn new_service(settings: SourcesSettings) -> Arc { + let sources: Vec> = vec![ + Arc::new(sigeth::Source::new(settings.sigeth)), + Arc::new(fourbyte::Source::new(settings.fourbyte)), + ]; + let complete_sources = { + let mut sources: Vec> = vec![]; + if settings.eth_bytecode_db.enabled { + sources.push(Arc::new(eth_bytecode_db::Source::new( + settings.eth_bytecode_db.url, + ))) + }; + sources + }; + let aggregator = Arc::new(SourceAggregator::new(sources, complete_sources)); Arc::new(Service::new(aggregator)) } pub async fn sig_provider(settings: Settings) -> Result<(), anyhow::Error> { - let service = new_service(settings.sources); + tracing::init_logs(SERVICE_NAME, &settings.tracing, &settings.jaeger)?; + let health = Arc::new(HealthService::default()); + let service = new_service(settings.sources); - let grpc_router = grpc_router(service.clone(), service.clone(), health.clone()); - let http_router = HttpRouter { - signature: service.clone(), + let router = Router { abi: service.clone(), + signature: service.clone(), health, }; + + let grpc_router = router.grpc_router(); + let http_router = router; + let launch_settings = LaunchSettings { - service_name: "sig_provider".to_owned(), + service_name: SERVICE_NAME.to_string(), server: settings.server, metrics: settings.metrics, - jaeger: settings.jaeger, }; - blockscout_service_launcher::launch(&launch_settings, http_router, grpc_router).await + launcher::launch(&launch_settings, http_router, grpc_router).await } diff --git a/sig-provider/sig-provider-server/src/service.rs b/sig-provider/sig-provider-server/src/service.rs index abeb6164d..82a93a610 100644 --- a/sig-provider/sig-provider-server/src/service.rs +++ b/sig-provider/sig-provider-server/src/service.rs @@ -3,8 +3,9 @@ use ethabi::{ethereum_types::H256, RawLog}; use sig_provider::SourceAggregator; use sig_provider_proto::blockscout::sig_provider::v1::{ abi_service_server::AbiService, signature_service_server::SignatureService, - CreateSignaturesRequest, CreateSignaturesResponse, GetEventAbiRequest, GetEventAbiResponse, - GetFunctionAbiRequest, GetFunctionAbiResponse, + BatchGetEventAbisRequest, BatchGetEventAbisResponse, CreateSignaturesRequest, + CreateSignaturesResponse, GetEventAbiRequest, GetEventAbiResponse, GetFunctionAbiRequest, + GetFunctionAbiResponse, }; use std::sync::Arc; @@ -59,27 +60,62 @@ impl AbiService for Service { request: tonic::Request, ) -> Result, tonic::Status> { let request = request.into_inner(); - let topics: Result, _> = request - .topics - .split(',') - .map(|topic| { - let hex = decode(topic)?; - if hex.len() != 32 { - return Err(tonic::Status::invalid_argument( - "topic len must be 32 bytes", - )); - } - Ok(H256::from_slice(&hex)) - }) - .collect(); - let topics = topics?; + + let topics = parse_topics(request.topics)?; self.agg .get_event_abi(RawLog { data: decode(&request.data)?, topics, }) .await - .map(|abi| tonic::Response::new(GetEventAbiResponse { abi })) + .map(|abi| GetEventAbiResponse { abi }) .map_err(|e| tonic::Status::internal(e.to_string())) + .map(tonic::Response::new) + } + + async fn batch_get_event_abis( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let batch_request = request.into_inner(); + + let mut raw_logs = Vec::new(); + for request in batch_request.requests { + let topics = parse_topics(request.topics)?; + raw_logs.push(RawLog { + data: decode(&request.data)?, + topics, + }); + } + + let batch_abis = self + .agg + .batch_get_event_abi(raw_logs) + .await + .map_err(|e| tonic::Status::internal(e.to_string()))?; + + let mut responses = Vec::new(); + for abi in batch_abis { + responses.push(GetEventAbiResponse { abi }) + } + + Ok(tonic::Response::new(BatchGetEventAbisResponse { + responses, + })) } } + +fn parse_topics(topics: String) -> Result, tonic::Status> { + topics + .split(',') + .map(|topic| { + let hex = decode(topic)?; + if hex.len() != 32 { + return Err(tonic::Status::invalid_argument( + "topic len must be 32 bytes", + )); + } + Ok(H256::from_slice(&hex)) + }) + .collect() +} diff --git a/sig-provider/sig-provider-server/src/settings.rs b/sig-provider/sig-provider-server/src/settings.rs index 8a4fd14e8..f8c7cf862 100644 --- a/sig-provider/sig-provider-server/src/settings.rs +++ b/sig-provider/sig-provider-server/src/settings.rs @@ -1,82 +1,35 @@ use blockscout_service_launcher::{ - GrpcServerSettings, HttpServerSettings, JaegerSettings, MetricsSettings, ServerSettings, + launcher::{ConfigSettings, MetricsSettings, ServerSettings}, + tracing::{JaegerSettings, TracingSettings}, }; -use config::{Config, File}; -use serde::{de, Deserialize, Serialize}; -use std::{net::SocketAddr, str::FromStr}; +use serde::{Deserialize, Serialize}; -/// Wrapper under [`serde::de::IgnoredAny`] which implements -/// [`PartialEq`] and [`Eq`] for fields to be ignored. -#[derive(Copy, Clone, Debug, Default, Deserialize)] -struct IgnoredAny(de::IgnoredAny); - -impl PartialEq for IgnoredAny { - fn eq(&self, _other: &Self) -> bool { - // We ignore that values, so they should not impact the equality - true - } -} - -impl Eq for IgnoredAny {} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(default, deny_unknown_fields)] +#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct Settings { + #[serde(default)] pub server: ServerSettings, - pub sources: SourcesSettings, + #[serde(default)] pub metrics: MetricsSettings, + #[serde(default)] + pub tracing: TracingSettings, + #[serde(default)] pub jaeger: JaegerSettings, - // Is required as we deny unknown fields, but allow users provide - // path to config through PREFIX__CONFIG env variable. If removed, - // the setup would fail with `unknown field `config`, expected one of...` - #[serde(skip_serializing, rename = "config")] - config_path: IgnoredAny, + #[serde(default)] + pub sources: SourcesSettings, } -impl Default for Settings { - fn default() -> Self { - Self { - server: ServerSettings { - http: HttpServerSettings { - enabled: true, - addr: SocketAddr::from_str("0.0.0.0:8050").unwrap(), - }, - grpc: GrpcServerSettings { - enabled: true, - addr: SocketAddr::from_str("0.0.0.0:8051").unwrap(), - }, - }, - sources: Default::default(), - metrics: Default::default(), - jaeger: Default::default(), - config_path: Default::default(), - } - } +impl ConfigSettings for Settings { + const SERVICE_NAME: &'static str = "SIG_PROVIDER"; } -impl Settings { - pub fn new() -> anyhow::Result { - let config_path = std::env::var("SIG_PROVIDER__CONFIG"); - - let mut builder = Config::builder(); - if let Ok(config_path) = config_path { - builder = builder.add_source(File::with_name(&config_path)); - }; - // Use `__` so that it would be possible to address keys with underscores in names (e.g. `access_key`) - builder = - builder.add_source(config::Environment::with_prefix("SIG_PROVIDER").separator("__")); - - let settings: Settings = builder.build()?.try_deserialize()?; - - Ok(settings) - } -} -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] #[serde(default, deny_unknown_fields)] pub struct SourcesSettings { pub fourbyte: url::Url, pub sigeth: url::Url, + pub eth_bytecode_db: EthBytecodeDbSettings, } impl Default for SourcesSettings { @@ -84,6 +37,23 @@ impl Default for SourcesSettings { Self { fourbyte: url::Url::parse("https://www.4byte.directory/").unwrap(), sigeth: url::Url::parse("https://sig.eth.samczsun.com/").unwrap(), + eth_bytecode_db: Default::default(), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct EthBytecodeDbSettings { + pub enabled: bool, + pub url: url::Url, +} + +impl Default for EthBytecodeDbSettings { + fn default() -> Self { + Self { + enabled: true, + url: url::Url::parse("https://eth-bytecode-db.services.blockscout.com/").unwrap(), } } } diff --git a/sig-provider/sig-provider-server/tests/mock.rs b/sig-provider/sig-provider-server/tests/mock.rs index b538bf6d0..7eaca30f6 100644 --- a/sig-provider/sig-provider-server/tests/mock.rs +++ b/sig-provider/sig-provider-server/tests/mock.rs @@ -1,18 +1,93 @@ -use actix_web::App; -use httpmock::MockServer; +use blockscout_service_launcher::test_server; +use httpmock::{Mock, MockServer, Then, When}; use pretty_assertions::assert_eq; use serde_json::Value; -use sig_provider_server::{http_configure, new_service, SourcesSettings}; -use std::time::Duration; +use sig_provider_server::{EthBytecodeDbSettings, SourcesSettings}; +use std::{cell::RefCell, time::Duration}; + +async fn run_server(sources: &SourceMocks<'_>) -> url::Url { + let mut settings = sig_provider_server::Settings::default(); + let (server_settings, base) = test_server::get_test_server_settings(); + settings.server = server_settings; + settings.jaeger.enabled = false; + settings.tracing.enabled = false; + + settings.sources = SourcesSettings { + fourbyte: format!("http://127.0.0.1:{}/", sources.fourbyte.port()) + .parse() + .unwrap(), + sigeth: format!("http://127.0.0.1:{}/", sources.sigeth.port()) + .parse() + .unwrap(), + eth_bytecode_db: EthBytecodeDbSettings { + enabled: true, + url: format!("http://127.0.0.1:{}/", sources.eth_bytecode_db.port()) + .parse() + .unwrap(), + }, + }; + test_server::init_server(|| sig_provider_server::sig_provider(settings), &base).await; + base +} + +struct SourceMocks<'a> { + fourbyte: MockServer, + sigeth: MockServer, + eth_bytecode_db: MockServer, + + mocks: RefCell>>, +} + +impl<'a> SourceMocks<'a> { + pub fn new() -> Self { + Self { + fourbyte: MockServer::start(), + sigeth: MockServer::start(), + eth_bytecode_db: MockServer::start(), + + mocks: RefCell::new(Vec::new()), + } + } + + pub fn fourbyte_mock(&'a self, config_fn: F) + where + F: FnOnce(When, Then), + { + self.mocks.borrow_mut().push(self.fourbyte.mock(config_fn)); + } + + pub fn sigeth_mock(&'a self, config_fn: F) + where + F: FnOnce(When, Then), + { + self.mocks.borrow_mut().push(self.sigeth.mock(config_fn)); + } + + pub fn eth_bytecode_db_mock(&'a self, config_fn: F) + where + F: FnOnce(When, Then), + { + self.mocks + .borrow_mut() + .push(self.eth_bytecode_db.mock(config_fn)); + } + + pub fn assert(&self) { + for mock in self.mocks.borrow().iter() { + mock.assert(); + } + } +} #[tokio::test] async fn create() { let _ = tracing_subscriber::fmt::try_init(); + let mocks = SourceMocks::new(); + let fourbyte_request = serde_json::json!({"contract_abi":"[{\"constant\":false,\"inputs\":[],\"name\":\"f\",\"outputs\":[],\"type\":\"function\"},{\"inputs\":[],\"type\":\"constructor\"},{\"anonymous\":false,\"inputs\":[{\"name\":\"\",\"type\":\"string\",\"indexed\":true}],\"name\":\"E\",\"type\":\"event\"}]"}); let fourbyte_response = serde_json::json!({"num_processed":25,"num_imported":3,"num_duplicates":18,"num_ignored":4}); - let fourbyte = MockServer::start(); - let fourbyte_handle = fourbyte.mock(|when, then| { + mocks.fourbyte_mock(|when, then| { when.method(httpmock::Method::POST) .path("/api/v1/import-solidity/") .header("Content-type", "application/json") @@ -24,8 +99,7 @@ async fn create() { let sigeth_request = serde_json::json!({"type":"abi","data":[[{"constant":false,"inputs":[],"name":"f","outputs":[],"type":"function"},{"inputs":[],"type":"constructor"},{"anonymous":false,"inputs":[{"name":"","type":"string","indexed":true}],"name":"E","type":"event"}]]}); let sigeth_response = serde_json::json!({"ok":true,"result":{"event":{"imported":{},"duplicated":{"E(string)":"0x3e9992c940c54ea252d3a34557cc3d3014281525c43d694f89d5f3dfd820b07d"},"invalid":null},"function":{"imported":{},"duplicated":{"f()":"0x26121ff0"},"invalid":null}}}); - let sigeth = MockServer::start(); - let sigeth_handle = sigeth.mock(|when, then| { + mocks.sigeth_mock(|when, then| { when.method(httpmock::Method::POST) .path("/api/v1/import") .header("Content-type", "application/json") @@ -35,32 +109,16 @@ async fn create() { .json_body(sigeth_response); }); - let service = new_service(SourcesSettings { - fourbyte: format!("http://127.0.0.1:{}/", fourbyte.port()) - .parse() - .unwrap(), - sigeth: format!("http://127.0.0.1:{}/", sigeth.port()) - .parse() - .unwrap(), - }); - let app = actix_web::test::init_service( - App::new().configure(|config| http_configure(config, service.clone(), service.clone())), - ) - .await; + let base = run_server(&mocks).await; + let route = "/api/v1/signatures"; let request = serde_json::json!({"abi":"[{\"constant\":false,\"inputs\":[],\"name\":\"f\",\"outputs\":[],\"type\":\"function\"},{\"inputs\":[],\"type\":\"constructor\"},{\"anonymous\":false,\"inputs\":[{\"name\":\"\",\"type\":\"string\",\"indexed\":true}],\"name\":\"E\",\"type\":\"event\"}]"}); - let request = actix_web::test::TestRequest::default() - .method(http::Method::POST) - .uri("/api/v1/signatures") - .append_header(("Content-type", "application/json")) - .set_json(request) - .to_request(); - let response: serde_json::Value = actix_web::test::call_and_read_body_json(&app, request).await; + let response: serde_json::Value = test_server::send_post_request(&base, route, &request).await; // allow async handle to work tokio::time::sleep(Duration::from_millis(100)).await; - fourbyte_handle.assert(); - sigeth_handle.assert(); + mocks.assert(); + assert_eq!(serde_json::json!({}), response); } @@ -87,9 +145,10 @@ fn sort_json(mut v: Value) -> Value { async fn get_function() { let _ = tracing_subscriber::fmt::try_init(); + let mocks = SourceMocks::new(); + let fourbyte_response = serde_json::json!({"count":4,"next":null,"previous":null,"results":[{"id":844293,"created_at":"2022-08-26T12:22:13.363345Z","text_signature":"watch_tg_invmru_119a5a98(address,uint256,uint256)","hex_signature":"0x70a08231","bytes_signature":"p ‚1"},{"id":166551,"created_at":"2019-09-24T11:36:57.296021Z","text_signature":"passphrase_calculate_transfer(uint64,address)","hex_signature":"0x70a08231","bytes_signature":"p ‚1"},{"id":166550,"created_at":"2019-09-24T11:36:37.525020Z","text_signature":"branch_passphrase_public(uint256,bytes8)","hex_signature":"0x70a08231","bytes_signature":"p ‚1"},{"id":143,"created_at":"2016-07-09T03:58:27.545013Z","text_signature":"balanceOf(address)","hex_signature":"0x70a08231","bytes_signature":"p ‚1"}]}); - let fourbyte = MockServer::start(); - let fourbyte_handle = fourbyte.mock(|when, then| { + mocks.fourbyte_mock(|when, then| { when.method(httpmock::Method::GET) .path("/api/v1/signatures/") .query_param("hex_signature", "70a08231"); @@ -99,8 +158,7 @@ async fn get_function() { }); let sigeth_response = serde_json::json!({"ok":true,"result":{"event":{},"function":{"0x70a08231":[{"name":"passphrase_calculate_transfer(uint64,address)","filtered":true},{"name":"branch_passphrase_public(uint256,bytes8)","filtered":true},{"name":"balanceOf(address)","filtered":false}]}}}); - let sigeth = MockServer::start(); - let sigeth_handle = sigeth.mock(|when, then| { + mocks.sigeth_mock(|when, then| { when.method(httpmock::Method::GET) .path("/api/v1/signatures") .query_param("function", "0x70a08231") @@ -110,27 +168,12 @@ async fn get_function() { .json_body(sigeth_response); }); - let service = new_service(SourcesSettings { - fourbyte: format!("http://127.0.0.1:{}/", fourbyte.port()) - .parse() - .unwrap(), - sigeth: format!("http://127.0.0.1:{}/", sigeth.port()) - .parse() - .unwrap(), - }); - let app = actix_web::test::init_service( - App::new().configure(|config| http_configure(config, service.clone(), service.clone())), - ) - .await; + let base = run_server(&mocks).await; - let request = actix_web::test::TestRequest::default() - .method(http::Method::GET) - .uri("/api/v1/abi/function?txInput=0x70a0823100000000000000000000000000000000219ab540356cbb839cbe05303d7705fa") - .to_request(); - let response: serde_json::Value = actix_web::test::call_and_read_body_json(&app, request).await; + let route = "/api/v1/abi/function?txInput=0x70a0823100000000000000000000000000000000219ab540356cbb839cbe05303d7705fa"; + let response: serde_json::Value = test_server::send_get_request(&base, route).await; - fourbyte_handle.assert(); - sigeth_handle.assert(); + mocks.assert(); assert_eq!( sort_json( @@ -144,9 +187,10 @@ async fn get_function() { async fn get_event() { let _ = tracing_subscriber::fmt::try_init(); + let mocks = SourceMocks::new(); + let fourbyte_response = serde_json::json!({"count":1,"next":null,"previous":null,"results":[{"id":1,"created_at":"2020-11-30T22:38:00.801049Z","text_signature":"Transfer(address,address,uint256)","hex_signature":"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef","bytes_signature":"\u{1234}\u{4132}\u{1244}\u{1110}"}]}); - let fourbyte = MockServer::start(); - let fourbyte_handle = fourbyte.mock(|when, then| { + mocks.fourbyte_mock(|when, then| { when.method(httpmock::Method::GET) .path("/api/v1/event-signatures/") .query_param( @@ -159,8 +203,7 @@ async fn get_event() { }); let sigeth_response = serde_json::json!({"ok":true,"result":{"event":{"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef":[{"name":"Transfer(address,address,uint256)","filtered":false}]},"function":{}}}); - let sigeth = MockServer::start(); - let sigeth_handle = sigeth.mock(|when, then| { + mocks.sigeth_mock(|when, then| { when.method(httpmock::Method::GET) .path("/api/v1/signatures") .query_param( @@ -173,32 +216,95 @@ async fn get_event() { .json_body(sigeth_response); }); - let service = new_service(SourcesSettings { - fourbyte: format!("http://127.0.0.1:{}/", fourbyte.port()) - .parse() - .unwrap(), - sigeth: format!("http://127.0.0.1:{}/", sigeth.port()) - .parse() - .unwrap(), + let eth_bytecode_db_request = serde_json::json!({"selector": "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"}); + let eth_bytecode_db_response = serde_json::json!({"eventDescriptions":[{"type":"event","name":"Transfer","inputs":"[{\"indexed\":true,\"internalType\":\"address\",\"name\":\"from\",\"type\":\"address\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"to\",\"type\":\"address\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"amount\",\"type\":\"uint256\"}]"}]}); + mocks.eth_bytecode_db_mock(|when, then| { + when.method(httpmock::Method::POST) + .path("/api/v2/event-descriptions:search") + .header("Content-type", "application/json") + .json_body(eth_bytecode_db_request); + then.status(200) + .header("Content-type", "application/json") + .json_body(eth_bytecode_db_response); }); - let app = actix_web::test::init_service( - App::new().configure(|config| http_configure(config, service.clone(), service.clone())), - ) - .await; - let request = actix_web::test::TestRequest::default() - .method(http::Method::GET) - .uri("/api/v1/abi/event?topics=0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef,000000000000000000000000b8ace4d9bc469ddc8e788e636e817c299a1a8150,000000000000000000000000f76c5b19e86c256482f4aad1dae620a0c3ac0cd6&data=00000000000000000000000000000000000000000000000000000000006acfc0") - .to_request(); - let response: serde_json::Value = actix_web::test::call_and_read_body_json(&app, request).await; + let base = run_server(&mocks).await; - fourbyte_handle.assert(); - sigeth_handle.assert(); + let route = "/api/v1/abi/event?topics=0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef,000000000000000000000000b8ace4d9bc469ddc8e788e636e817c299a1a8150,000000000000000000000000f76c5b19e86c256482f4aad1dae620a0c3ac0cd6&data=00000000000000000000000000000000000000000000000000000000006acfc0"; + let response: serde_json::Value = test_server::send_get_request(&base, route).await; + + mocks.assert(); assert_eq!( - sort_json( - serde_json::json!([{"inputs":[{"components":[],"indexed":true,"name":"arg0","type":"address","value":"b8ace4d9bc469ddc8e788e636e817c299a1a8150"},{"components":[],"indexed":true,"name":"arg1","type":"address","value":"f76c5b19e86c256482f4aad1dae620a0c3ac0cd6"},{"components":[],"indexed":false,"name":"arg2","type":"uint256","value":"6acfc0"}],"name":"Transfer"}]) - ), - sort_json(response), + serde_json::json!([ + { + "inputs":[ + {"components":[],"indexed":true,"name":"from","type":"address","value":"b8ace4d9bc469ddc8e788e636e817c299a1a8150"}, + {"components":[],"indexed":true,"name":"to","type":"address","value":"f76c5b19e86c256482f4aad1dae620a0c3ac0cd6"}, + {"components":[],"indexed":false,"name":"amount","type":"uint256","value":"6acfc0"}], + "name":"Transfer" + }, + { + "inputs":[ + {"components":[],"indexed":true,"name":"arg0","type":"address","value":"b8ace4d9bc469ddc8e788e636e817c299a1a8150"}, + {"components":[],"indexed":true,"name":"arg1","type":"address","value":"f76c5b19e86c256482f4aad1dae620a0c3ac0cd6"}, + {"components":[],"indexed":false,"name":"arg2","type":"uint256","value":"6acfc0"}], + "name":"Transfer" + } + ]), + response, + ); +} + +#[tokio::test] +async fn batch_get_events() { + let _ = tracing_subscriber::fmt::try_init(); + + let mocks = SourceMocks::new(); + + { + let eth_bytecode_db_request = serde_json::json!({"selectors":[ + "0x6e9ed8cf1494d7312e7618c9411ab219f27a9840ed74dda9581992ca7575eb86", + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x83f86eb20c894914ecf65cefc94682009cdb5066a609e8428699fa87b19b5c57", + ]}); + let eth_bytecode_db_response = serde_json::json!({"responses": [ + {"eventDescriptions":[{"type":"event","name":"C","inputs":"[{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"c\",\"type\":\"uint256\"}]"}]}, + {"eventDescriptions":[]}, + {"eventDescriptions":[{"type":"event","name":"A","inputs":"[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"a\",\"type\":\"uint256\"},{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"b\",\"type\":\"uint256\"}]"},{"type":"event","name":"A","inputs":"[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"a2\",\"type\":\"uint256\"},{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"b2\",\"type\":\"uint256\"}]"}]}, + ]}); + mocks.eth_bytecode_db_mock(|when, then| { + when.method(httpmock::Method::POST) + .path("/api/v2/event-descriptions:batch-search") + .header("Content-type", "application/json") + .json_body(eth_bytecode_db_request); + then.status(200) + .header("Content-type", "application/json") + .json_body(eth_bytecode_db_response); + }); + } + + let base = run_server(&mocks).await; + + let route = "/api/v1/abi/events:batch-get"; + let request = serde_json::json!({"requests":[ + {"data":"0x0000000000000000000000000000000000000000000000000000000000000001","topics":"0x6e9ed8cf1494d7312e7618c9411ab219f27a9840ed74dda9581992ca7575eb86"}, + {"data":"0x00000000000000000000000000000c000000000000000000000000000006acfc","topics":"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef,000000000000000000000000b8ace4d9bc469ddc8e788e636e817c299a1a8150,000000000000000000000000f76c5b19e86c256482f4aad1dae620a0c3ac0cd6"}, + {"data":"0x","topics":"0x83f86eb20c894914ecf65cefc94682009cdb5066a609e8428699fa87b19b5c57,0x0000000000000000000000000000000000000000000000000000000000000001,0x0000000000000000000000000000000000000000000000000000000000000002"} + ]}); + let response: serde_json::Value = test_server::send_post_request(&base, route, &request).await; + + mocks.assert(); + + assert_eq!( + serde_json::json!([ + {"abi": [{"inputs":[{"components":[],"indexed":false,"name":"c","type":"uint256","value":"1"}],"name":"C"}] }, + {"abi": [] }, + {"abi": [ + {"inputs":[{"components":[],"indexed":true,"name":"a","type":"uint256","value":"1"},{"components":[],"indexed":true,"name":"b","type":"uint256","value":"2"}],"name":"A"}, + {"inputs":[{"components":[],"indexed":true,"name":"a2","type":"uint256","value":"1"},{"components":[],"indexed":true,"name":"b2","type":"uint256","value":"2"}],"name":"A"}, + ]}, + ]), + response, ); } diff --git a/sig-provider/sig-provider-server/tests/settings.rs b/sig-provider/sig-provider-server/tests/settings.rs index 07767d4c9..5411e21ee 100644 --- a/sig-provider/sig-provider-server/tests/settings.rs +++ b/sig-provider/sig-provider-server/tests/settings.rs @@ -1,10 +1,11 @@ +use blockscout_service_launcher::launcher::ConfigSettings; use pretty_assertions::assert_eq; use sig_provider_server::Settings; #[test] fn base_settings_are_default() { std::env::set_var("SIG_PROVIDER__CONFIG", "config/base.toml"); - let example = Settings::new().expect("Failed to parse config"); + let example = Settings::build().expect("Failed to parse config"); let default = Settings::default(); assert_eq!(default, example); } diff --git a/sig-provider/sig-provider/Cargo.toml b/sig-provider/sig-provider/Cargo.toml index 6a2677528..2d8c45306 100644 --- a/sig-provider/sig-provider/Cargo.toml +++ b/sig-provider/sig-provider/Cargo.toml @@ -5,7 +5,9 @@ edition = "2021" [dependencies] sig-provider-proto = { path = "../sig-provider-proto" } +alloy-json-abi = { git = "https://github.com/alloy-rs/core", rev = "398d7e2", features = ["serde_json"] } tokio = "1" +tokio-stream = "0.1" async-trait = "0.1" anyhow = "1.0" tracing = "0.1" @@ -17,7 +19,7 @@ reqwest-retry = "0.1" hex = "0.4" serde = { version = "1", features = ["derive"] } serde_json = "1.0" -ethabi = "17" +ethabi = "18.0.0" mockall = "0.11" itertools = "0.10" async-recursion = "1.0.0" diff --git a/sig-provider/sig-provider/src/aggregator.rs b/sig-provider/sig-provider/src/aggregator.rs index 26c33c0b2..2d00ed240 100644 --- a/sig-provider/sig-provider/src/aggregator.rs +++ b/sig-provider/sig-provider/src/aggregator.rs @@ -1,4 +1,5 @@ -use crate::SignatureSource; +use crate::{sources::CompleteSignatureSource, SignatureSource}; +use anyhow::Context; use ethabi::{Event, EventParam, ParamType, RawLog, Token}; use itertools::Itertools; use sig_provider_proto::blockscout::sig_provider::v1::{Abi, Argument}; @@ -6,10 +7,11 @@ use std::{collections::HashSet, sync::Arc}; pub struct SourceAggregator { sources: Vec>, + complete_sources: Vec>, } macro_rules! proxy { - ($sources:ident, $request:ident, $fn:ident) => {{ + ($sources:expr, $request:expr, $fn:ident) => {{ let tasks = $sources.iter().map(|source| source.$fn($request)); let responses: Vec<_> = futures::future::join_all(tasks) .await @@ -32,16 +34,32 @@ macro_rules! proxy { }}; } +macro_rules! get_event_signatures { + ($sources:expr, $request:expr) => {{ + let responses = proxy!($sources, $request, get_event_signatures); + crate::aggregator::SourceAggregator::merge_signatures(responses) + }}; +} + impl SourceAggregator { // You should provide sources in priority descending order (first - max priority) - pub fn new(sources: Vec>) -> SourceAggregator { - SourceAggregator { sources } + pub fn new( + sources: Vec>, + complete_sources: Vec>, + ) -> SourceAggregator { + SourceAggregator { + sources, + complete_sources, + } } - fn merge_signatures, II: IntoIterator>( + fn merge_signatures, II: IntoIterator>( sigs: II, - ) -> Vec { - let mut content: HashSet = HashSet::default(); + ) -> Vec + where + T: Clone + Eq + std::hash::Hash, + { + let mut content: HashSet = HashSet::default(); sigs.into_iter() .flatten() .filter(|sig| content.insert(sig.clone())) @@ -61,13 +79,6 @@ impl SourceAggregator { Ok(signatures) } - pub async fn get_event_signatures(&self, hex: &str) -> Result, anyhow::Error> { - let sources = &self.sources; - let responses = proxy!(sources, hex, get_event_signatures); - let signatures = Self::merge_signatures(responses); - Ok(signatures) - } - pub async fn get_function_abi(&self, tx_input: &[u8]) -> Result, anyhow::Error> { if tx_input.len() < 4 { anyhow::bail!("tx input len must be at least 4 bytes"); @@ -93,23 +104,113 @@ impl SourceAggregator { anyhow::bail!("log should contain at least one topic"); } let hex_sig = hex::encode(raw.topics[0].as_bytes()); - let sigs = self.get_event_signatures(&hex_sig).await?; - Ok(sigs - .into_iter() - .filter_map(|sig| { - let (name, args) = parse_signature(&sig)?; - let (values, indexed) = decode_log(name.to_string(), &args, raw.clone())?; - let mut inputs = parse_args("arg".into(), &args, &values); + + let complete_sigs = get_event_signatures!(&self.complete_sources, &hex_sig); + let sigs = get_event_signatures!(&self.sources, &hex_sig); + + process_event_signatures(&raw, complete_sigs, sigs).await + } + + pub async fn batch_get_event_abi( + &self, + raw_logs: Vec, + ) -> Result>, anyhow::Error> { + let mut hex_sigs = Vec::new(); + for raw in &raw_logs { + if raw.topics.is_empty() { + anyhow::bail!("log should contain at least one topic") + } + hex_sigs.push(hex::encode(raw.topics[0].as_bytes())); + } + + let complete_responses = proxy!( + &self.complete_sources, + &hex_sigs, + batch_get_event_signatures + ); + let responses = proxy!(&self.sources, &hex_sigs, batch_get_event_signatures); + + let mut results = Vec::new(); + for (index, raw_log) in raw_logs.iter().enumerate() { + let batch_complete_signatures: Vec<_> = complete_responses + .iter() + .map(|response| response.get(index).cloned().unwrap_or_default()) + .collect(); + let complete_signatures = SourceAggregator::merge_signatures(batch_complete_signatures); + + let batch_signatures: Vec<_> = responses + .iter() + .map(|response| response.get(index).cloned().unwrap_or_default()) + .collect(); + let signatures = SourceAggregator::merge_signatures(batch_signatures); + + let abis = process_event_signatures(raw_log, complete_signatures, signatures).await?; + results.push(abis) + } + + Ok(results) + } +} + +async fn process_event_signatures( + raw: &RawLog, + complete_signatures: Vec, + signatures: Vec, +) -> Result, anyhow::Error> { + let complete_abis: Vec<_> = complete_signatures.into_iter().filter_map(|alloy_event| { + let ethabi_event = try_from_alloy_event_to_ethabi_event(alloy_event.clone()) + .map_err(|err| tracing::error!("converting alloy_json_abi::Event into ethabi::Event failed for {alloy_event:?}; err={err:#}")).ok()?; + ethabi_event.parse_log_whole(raw.clone()).ok() + .map(|event| { + let mut names = Vec::new(); + let mut values = Vec::new(); + for param in event.params.into_iter() { + names.push(param.name); + values.push(param.value); + } + let indexed: Vec<_> = ethabi_event.inputs.iter().map(|param| param.indexed).collect(); + let args: Vec<_> = ethabi_event.inputs.into_iter().map(|param| param.kind).collect(); + let mut inputs = parse_args_with_names(&names, &args, &values); for (input, indexed) in inputs.iter_mut().zip(indexed) { input.indexed = Some(indexed); } - Some(Abi { - name: name.into(), + Abi { + name: ethabi_event.name, inputs, - }) + } }) - .collect()) - } + }).collect(); + + let abis: Vec<_> = signatures + .into_iter() + .filter_map(|sig| { + let (name, args) = parse_signature(&sig)?; + let (values, indexed) = decode_log(name.to_string(), &args, raw.clone())?; + let mut inputs = parse_args("arg".into(), &args, &values); + for (input, indexed) in inputs.iter_mut().zip(indexed) { + input.indexed = Some(indexed); + } + Some(Abi { + name: name.into(), + inputs, + }) + }) + .collect(); + + let mut seen_abis = HashSet::new(); + let result: Vec<_> = complete_abis + .into_iter() + .chain(abis) + .filter_map(|abi| { + if !seen_abis.contains(&abi) { + seen_abis.insert(abi.clone()); + Some(abi) + } else { + None + } + }) + .collect(); + Ok(result) } fn parse_signature(sig: &str) -> Option<(&str, Vec)> { @@ -195,16 +296,32 @@ fn parse_arg(name: String, param: &ParamType, value: &Token) -> Argument { } } -fn parse_args(pref: String, args: &[ParamType], values: &[Token]) -> Vec { - let inputs = args +fn parse_args_with_names(names: &[String], args: &[ParamType], values: &[Token]) -> Vec { + let inputs = names .iter() + .zip(args.iter()) .zip(values.iter()) - .enumerate() - .map(|(index, (arg, value))| parse_arg(format!("{pref}{index}"), arg, value)) + .map(|((name, arg), value)| parse_arg(name.clone(), arg, value)) .collect(); inputs } +fn parse_args(pref: String, args: &[ParamType], values: &[Token]) -> Vec { + let names = (0..args.len()) + .map(|index| format!("{pref}{index}")) + .collect::>(); + parse_args_with_names(&names, args, values) +} + +fn try_from_alloy_event_to_ethabi_event( + alloy_event: alloy_json_abi::Event, +) -> Result { + serde_json::from_value( + serde_json::to_value(alloy_event).context("serializing alloy_json_abi::Event")?, + ) + .context("deserializing ethabi::Event") +} + #[cfg(test)] mod tests { use crate::sources::MockSignatureSource; @@ -290,7 +407,7 @@ mod tests { }); let source = Arc::new(source); - let agg = Arc::new(SourceAggregator::new(vec![source.clone()])); + let agg = Arc::new(SourceAggregator::new(vec![source.clone()], vec![])); let function = agg .get_function_abi(&hex::decode(input).unwrap()) @@ -341,7 +458,7 @@ mod tests { }); let source = Arc::new(source); - let agg = Arc::new(SourceAggregator::new(vec![source.clone()])); + let agg = Arc::new(SourceAggregator::new(vec![source.clone()], vec![])); let function = agg .get_function_abi(&hex::decode(input).unwrap()) @@ -590,7 +707,7 @@ mod tests { .returning(|_| Ok(vec![sig.into()])); let source = Arc::new(source); - let agg = Arc::new(SourceAggregator::new(vec![source.clone()])); + let agg = Arc::new(SourceAggregator::new(vec![source.clone()], vec![])); let event = agg.get_event_abi(input).await.unwrap(); assert_eq!(abi, event[0]); @@ -649,7 +766,7 @@ mod tests { .returning(|_| Ok(vec![sig.into()])); let source = Arc::new(source); - let agg = Arc::new(SourceAggregator::new(vec![source.clone()])); + let agg = Arc::new(SourceAggregator::new(vec![source.clone()], vec![])); let event = agg.get_event_abi(input).await.unwrap(); assert_eq!(abi, event[0]); @@ -723,7 +840,7 @@ mod tests { .returning(|_| Ok(vec![sig.into()])); let source = Arc::new(source); - let agg = Arc::new(SourceAggregator::new(vec![source.clone()])); + let agg = Arc::new(SourceAggregator::new(vec![source.clone()], vec![])); let event = agg.get_event_abi(input).await.unwrap(); assert_eq!(abi, event[0]); diff --git a/sig-provider/sig-provider/src/lib.rs b/sig-provider/sig-provider/src/lib.rs index f59fb5e22..9d9fda399 100644 --- a/sig-provider/sig-provider/src/lib.rs +++ b/sig-provider/sig-provider/src/lib.rs @@ -2,4 +2,4 @@ mod aggregator; mod sources; pub use aggregator::SourceAggregator; -pub use sources::{fourbyte, sigeth, SignatureSource}; +pub use sources::{eth_bytecode_db, fourbyte, sigeth, CompleteSignatureSource, SignatureSource}; diff --git a/sig-provider/sig-provider/src/sources/eth_bytecode_db.rs b/sig-provider/sig-provider/src/sources/eth_bytecode_db.rs new file mode 100644 index 000000000..72df5ca56 --- /dev/null +++ b/sig-provider/sig-provider/src/sources/eth_bytecode_db.rs @@ -0,0 +1,165 @@ +use crate::{ + eth_bytecode_db::json::{ + BatchSearchEventDescriptionResponse, BatchSearchEventDescriptionsRequest, + SearchEventDescriptionResponse, SearchEventDescriptionsRequest, + }, + sources::CompleteSignatureSource, +}; +use itertools::Itertools; +use reqwest_middleware::ClientWithMiddleware; + +pub struct Source { + host: url::Url, + client: ClientWithMiddleware, +} + +impl Source { + pub fn new(host: url::Url) -> Source { + Source { + host, + client: super::new_client(), + } + } + + async fn send_post_request( + &self, + path: &str, + request: &Request, + ) -> Result + where + Request: serde::Serialize, + Response: serde::de::DeserializeOwned, + { + let response = self + .client + .post(self.host.join(path).unwrap()) + .json(request) + .send() + .await + .map_err(anyhow::Error::msg)?; + if response.status().is_success() { + Ok(response.json().await?) + } else { + Err(anyhow::anyhow!( + "invalid status code got as a result: {}", + response.status(), + )) + } + } +} + +#[async_trait::async_trait] +impl CompleteSignatureSource for Source { + async fn get_event_signatures( + &self, + hex: &str, + ) -> Result, anyhow::Error> { + let route = "/api/v2/event-descriptions:search"; + let request = SearchEventDescriptionsRequest { + selector: super::hash(hex), + }; + Ok(self + .send_post_request::<_, SearchEventDescriptionResponse>(route, &request) + .await? + .event_descriptions + .into_iter() + .map(alloy_json_abi::Event::try_from) + .collect::>()?) + } + + async fn batch_get_event_signatures( + &self, + hex: &[String], + ) -> Result>, anyhow::Error> { + const BATCH_LIMIT: usize = 100; + + let route = "/api/v2/event-descriptions:batch-search"; + + let chunk_requests: Vec<_> = hex + .iter() + .chunks(BATCH_LIMIT) + .into_iter() + .map(|hex| BatchSearchEventDescriptionsRequest { + selectors: hex + .into_iter() + .map(|hex| super::hash(hex).to_string()) + .collect::>(), + }) + .collect(); + + let mut responses = Vec::new(); + for request in chunk_requests { + let chunk_responses = self + .send_post_request::<_, BatchSearchEventDescriptionResponse>(route, &request) + .await? + .responses; + responses.extend(chunk_responses) + } + + let mut results = Vec::new(); + for response in responses { + results.push( + response + .event_descriptions + .into_iter() + .map(alloy_json_abi::Event::try_from) + .collect::, _>>()?, + ); + } + + Ok(results) + } + + fn source(&self) -> String { + self.host.to_string() + } +} + +mod json { + use anyhow::Context; + use serde::{Deserialize, Serialize}; + + #[derive(Clone, Debug, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct SearchEventDescriptionsRequest { + pub selector: String, + } + + #[derive(Clone, Debug, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct SearchEventDescriptionResponse { + pub event_descriptions: Vec, + } + + #[derive(Clone, Debug, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct BatchSearchEventDescriptionsRequest { + pub selectors: Vec, + } + + #[derive(Clone, Debug, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct BatchSearchEventDescriptionResponse { + pub responses: Vec, + } + + #[derive(Clone, Debug, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct EventDescription { + name: String, + inputs: String, + } + + impl TryFrom for alloy_json_abi::Event { + type Error = anyhow::Error; + fn try_from(value: EventDescription) -> Result { + let inputs: Vec = serde_json::from_str(&value.inputs) + .context("deserializing event_description inputs")?; + Ok(Self { + name: value.name, + inputs, + anonymous: false, + }) + } + } +} diff --git a/sig-provider/sig-provider/src/sources/mod.rs b/sig-provider/sig-provider/src/sources/mod.rs index 8d75b53f5..18ec26631 100644 --- a/sig-provider/sig-provider/src/sources/mod.rs +++ b/sig-provider/sig-provider/src/sources/mod.rs @@ -1,3 +1,4 @@ +pub mod eth_bytecode_db; pub mod fourbyte; pub mod sigeth; @@ -7,6 +8,14 @@ use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; use std::time::Duration; +fn hash(hex: &str) -> String { + if hex.starts_with("0x") { + hex.to_owned() + } else { + "0x".to_owned() + hex + } +} + #[automock] #[async_trait] pub trait SignatureSource { @@ -16,6 +25,32 @@ pub trait SignatureSource { // Resulting signatures should be sorted in priority descending order (first - max priority) async fn get_event_signatures(&self, hex: &str) -> Result, anyhow::Error>; + async fn batch_get_event_signatures( + &self, + hex: &[String], + ) -> Result>, anyhow::Error> { + Ok(vec![vec![]; hex.len()]) + } + + // for errors + fn source(&self) -> String; +} + +#[automock] +#[async_trait] +pub trait CompleteSignatureSource { + async fn get_event_signatures( + &self, + hex: &str, + ) -> Result, anyhow::Error>; + + async fn batch_get_event_signatures( + &self, + hex: &[String], + ) -> Result>, anyhow::Error> { + Ok(vec![vec![]; hex.len()]) + } + // for errors fn source(&self) -> String; } diff --git a/sig-provider/sig-provider/src/sources/sigeth.rs b/sig-provider/sig-provider/src/sources/sigeth.rs index 264fc5019..eaf9173b3 100644 --- a/sig-provider/sig-provider/src/sources/sigeth.rs +++ b/sig-provider/sig-provider/src/sources/sigeth.rs @@ -14,14 +14,6 @@ impl Source { } } - fn hash(hex: &str) -> String { - if hex.starts_with("0x") { - hex.to_owned() - } else { - "0x".to_owned() + hex - } - } - async fn fetch(&self, path: &str) -> Result { let response = self .client @@ -65,7 +57,7 @@ impl SignatureSource for Source { } async fn get_function_signatures(&self, hex: &str) -> Result, anyhow::Error> { - let hash = Self::hash(hex); + let hash = super::hash(hex); let resp = self .fetch(&format!("/api/v1/signatures?function={hash}&all")) .await?; @@ -74,7 +66,7 @@ impl SignatureSource for Source { } async fn get_event_signatures(&self, hex: &str) -> Result, anyhow::Error> { - let hash = Self::hash(hex); + let hash = super::hash(hex); let resp = self .fetch(&format!("/api/v1/signatures?event={hash}&all")) .await?;