From 5ff71cfa92e84323e9d2f1b8acbffdddcfeb0f07 Mon Sep 17 00:00:00 2001 From: Ryan Tate Date: Wed, 21 Sep 2022 20:07:14 -0700 Subject: [PATCH] add integrations test harness Signed-off-by: Ryan Tate --- .github/workflows/ci.yaml | 4 +- Cargo.lock | 1034 ++++++++++++++++- Cargo.toml | 1 + Makefile | 9 + erc20-market-cap/Makefile | 2 +- erc20-market-cap/src/pb.rs | 4 +- erc20-market-cap/substreams.yaml | 2 +- erc20-price/Cargo.toml | 2 +- erc721/build.rs | 4 +- integrations/.cargo/config.toml | 2 + integrations/Cargo.toml | 18 + integrations/Makefile | 3 + integrations/build.rs | 58 + integrations/src/errors.rs | 17 + integrations/src/lib.rs | 9 + integrations/src/module.rs | 88 ++ integrations/src/pb/messari.erc20_price.v1.rs | 54 + integrations/src/tests.rs | 81 ++ integrations/substreams.yaml | 21 + substreams-helper/build.rs | 1 + uniswap-v2/build.rs | 4 +- uniswap-v2/src/pb.rs | 4 +- 22 files changed, 1394 insertions(+), 28 deletions(-) create mode 100644 integrations/.cargo/config.toml create mode 100644 integrations/Cargo.toml create mode 100644 integrations/Makefile create mode 100644 integrations/build.rs create mode 100644 integrations/src/errors.rs create mode 100644 integrations/src/lib.rs create mode 100644 integrations/src/module.rs create mode 100644 integrations/src/pb/messari.erc20_price.v1.rs create mode 100644 integrations/src/tests.rs create mode 100644 integrations/substreams.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1381caf0..4f2673a7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -41,7 +41,9 @@ jobs: ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ - target/ + **/target/ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Build run: make build-all + - name: Integrations + run: make integrations-linux diff --git a/Cargo.lock b/Cargo.lock index d8891303..5d585fc1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,12 +23,95 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +[[package]] +name = "async-stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "axum" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e3356844c4d6a6d6467b8da2cffb4a2820be256f50a3a386c9d152bab31043" +dependencies = [ + "async-trait", + "axum-core", + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde", + "sync_wrapper", + "tokio", + "tower", + "tower-http", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9f0c0a60006f2a293d82d571f635042a72edf927539b7685bd62d361963839b" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + [[package]] name = "bigdecimal" version = "0.3.0" @@ -67,6 +150,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" + [[package]] name = "byte-slice-cast" version = "1.2.1" @@ -127,6 +216,22 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + [[package]] name = "crunchy" version = "0.2.2" @@ -159,9 +264,18 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +[[package]] +name = "encoding_rs" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "erc20-price-substreams" -version = "0.0.1" +version = "0.1.0" dependencies = [ "anyhow", "ethabi", @@ -258,12 +372,90 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + [[package]] name = "funty" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "futures-channel" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf" + +[[package]] +name = "futures-io" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68" + +[[package]] +name = "futures-sink" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56" + +[[package]] +name = "futures-task" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1" + +[[package]] +name = "futures-util" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90" +dependencies = [ + "futures-core", + "futures-io", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "generic-array" version = "0.14.6" @@ -285,6 +477,25 @@ dependencies = [ "wasi", ] +[[package]] +name = "h2" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca32592cf21ac7ccab1825cd87f6c9b3d9022c44d086172ed0966bec8af30be" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -297,6 +508,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hex" version = "0.4.3" @@ -309,6 +529,105 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" +[[package]] +name = "http" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "hyper" +version = "0.14.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "impl-codec" version = "0.6.0" @@ -366,6 +685,12 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "ipnet" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" + [[package]] name = "itertools" version = "0.10.4" @@ -381,6 +706,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" +[[package]] +name = "js-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "keccak" version = "0.1.2" @@ -408,6 +742,12 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "matchit" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" + [[package]] name = "memchr" version = "2.5.0" @@ -420,12 +760,48 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8452105ba047068f40ff7093dd1d9da90898e63dd61736462e9cdda6a90ad3c3" +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mio" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys", +] + [[package]] name = "multimap" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "native-tls" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "num-bigint" version = "0.4.3" @@ -456,12 +832,67 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "once_cell" version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f7254b99e31cad77da24b08ebf628882739a608578bb1bcdfc1f9c21260d7c0" +[[package]] +name = "openssl" +version = "0.10.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "pad" version = "0.1.6" @@ -497,6 +928,12 @@ dependencies = [ "syn", ] +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + [[package]] name = "petgraph" version = "0.6.2" @@ -508,32 +945,70 @@ dependencies = [ ] [[package]] -name = "ppv-lite86" -version = "0.2.16" +name = "pin-project" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +dependencies = [ + "pin-project-internal", +] [[package]] -name = "prettyplease" -version = "0.1.19" +name = "pin-project-internal" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a49e86d2c26a24059894a3afa13fd17d063419b05dfb83f06d9c3566060c3f5a" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2", + "quote", "syn", ] [[package]] -name = "primitive-types" -version = "0.11.1" +name = "pin-project-lite" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a" -dependencies = [ - "fixed-hash", - "impl-codec", - "impl-rlp", - "impl-serde", - "uint", +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" + +[[package]] +name = "ppv-lite86" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" + +[[package]] +name = "prettyplease" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a49e86d2c26a24059894a3afa13fd17d063419b05dfb83f06d9c3566060c3f5a" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "primitive-types" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a" +dependencies = [ + "fixed-hash", + "impl-codec", + "impl-rlp", + "impl-serde", + "uint", ] [[package]] @@ -744,6 +1219,43 @@ dependencies = [ "winapi", ] +[[package]] +name = "reqwest" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75aa69a3f06bbcc66ede33af2af253c6f7a86b1ca0033f60c580a27074fbf92" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "rlp" version = "0.5.1" @@ -766,6 +1278,39 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +[[package]] +name = "schannel" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +dependencies = [ + "lazy_static", + "windows-sys", +] + +[[package]] +name = "security-framework" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.144" @@ -797,6 +1342,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha3" version = "0.10.5" @@ -807,6 +1364,25 @@ dependencies = [ "keccak", ] +[[package]] +name = "slab" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] + +[[package]] +name = "socket2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -946,6 +1522,21 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "substreams-integrations" +version = "0.1.0" +dependencies = [ + "prost 0.11.0", + "prost-types 0.11.1", + "reqwest", + "substreams", + "thiserror", + "tokio", + "tokio-stream", + "tonic", + "tonic-build", +] + [[package]] name = "substreams-macro" version = "0.0.20" @@ -987,6 +1578,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" + [[package]] name = "tap" version = "1.0.1" @@ -1036,6 +1633,96 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "1.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0020c875007ad96677dcc890298f4b942882c5d4eb7cc8f439fc3bf813dc9c95" +dependencies = [ + "autocfg", + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "once_cell", + "pin-project-lite", + "socket2", + "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-macros" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6edf2d6bc038a43d31353570e27270603f4648d18f5ed10c0e179abe43255af" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + [[package]] name = "toml" version = "0.5.9" @@ -1045,6 +1732,151 @@ dependencies = [ "serde", ] +[[package]] +name = "tonic" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11cd56bdb54ef93935a6a79dbd1d91f1ebd4c64150fd61654031fd6b8b775c91" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost 0.11.0", + "prost-derive 0.11.0", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", + "tracing-futures", +] + +[[package]] +name = "tonic-build" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fbcd2800e34e743b9ae795867d5f77b535d3a3be69fd731e39145719752df8c" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build 0.11.1", + "quote", + "syn", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c530c8675c1dbf98facee631536fa116b5fb6382d7dd6dc1b118d970eafe3ba" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" +dependencies = [ + "cfg-if 1.0.0", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" +dependencies = [ + "once_cell", +] + +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "typenum" version = "1.15.0" @@ -1063,30 +1895,148 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "unicode-bidi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + [[package]] name = "unicode-ident" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd" +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-width" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" + +[[package]] +name = "web-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wee_alloc" version = "0.4.5" @@ -1132,6 +2082,58 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + [[package]] name = "wyz" version = "0.5.0" diff --git a/Cargo.toml b/Cargo.toml index 553b1fe4..10f4d804 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "erc20-price", "erc20-market-cap", "erc721", + "integrations", "substreams-helper", "uniswap-v2" ] diff --git a/Makefile b/Makefile index 9df23fad..c7084b5b 100644 --- a/Makefile +++ b/Makefile @@ -5,3 +5,12 @@ build-all: make -C erc20-price build make -C erc721 build make -C compound-v2 build + make -C integrations build + +.PHONY: integrations-darwin +integrations-darwin: + cargo test --package substreams-integrations --target aarch64-apple-darwin -- --nocapture + +.PHONY: integrations-linux +integrations-linux: + cargo test --package substreams-integrations --target x86_64-unknown-linux-gnu -- --nocapture \ No newline at end of file diff --git a/erc20-market-cap/Makefile b/erc20-market-cap/Makefile index 4c7fc2b2..e9ccc172 100644 --- a/erc20-market-cap/Makefile +++ b/erc20-market-cap/Makefile @@ -10,4 +10,4 @@ build: .PHONY: example example: - substreams run -e api-dev.streamingfast.io:443 substreams.yaml map_market_cap -s 13000700 -t +1 + substreams run -e api-dev.streamingfast.io:443 substreams.yaml map_market_cap -s 13000000 -t +1 diff --git a/erc20-market-cap/src/pb.rs b/erc20-market-cap/src/pb.rs index 981ea8c3..f142c82a 100644 --- a/erc20-market-cap/src/pb.rs +++ b/erc20-market-cap/src/pb.rs @@ -1,2 +1,2 @@ -pub mod erc20_market_cap; -pub mod erc20_price; \ No newline at end of file +pub mod erc20_price; +pub mod erc20_market_cap; \ No newline at end of file diff --git a/erc20-market-cap/substreams.yaml b/erc20-market-cap/substreams.yaml index 8520fd53..3c0f730f 100644 --- a/erc20-market-cap/substreams.yaml +++ b/erc20-market-cap/substreams.yaml @@ -21,7 +21,7 @@ binaries: modules: - name: map_market_cap kind: map - initialBlock: 13000700 + initialBlock: 13000000 inputs: - map: erc20Price:map_price output: diff --git a/erc20-price/Cargo.toml b/erc20-price/Cargo.toml index 520427a8..a0e0b63e 100644 --- a/erc20-price/Cargo.toml +++ b/erc20-price/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "erc20-price-substreams" -version = "0.0.1" +version = "0.1.0" description = "Price all ERC20 tokens." edition = "2021" repository = "https://github.com/messari/substreams/erc20-price" diff --git a/erc721/build.rs b/erc721/build.rs index 68bc205f..fb321d31 100644 --- a/erc721/build.rs +++ b/erc721/build.rs @@ -2,8 +2,8 @@ use anyhow::{Ok, Result}; use substreams_common::codegen; fn main() -> Result<(), anyhow::Error> { - // println!("cargo:rerun-if-changed=proto"); - // println!("cargo:rerun-if-changed=abi"); + println!("cargo:rerun-if-changed=proto"); + println!("cargo:rerun-if-changed=abi"); codegen::generate(None)?; Ok(()) diff --git a/integrations/.cargo/config.toml b/integrations/.cargo/config.toml new file mode 100644 index 00000000..ab370428 --- /dev/null +++ b/integrations/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +target = "aarch64-apple-darwin" \ No newline at end of file diff --git a/integrations/Cargo.toml b/integrations/Cargo.toml new file mode 100644 index 00000000..841cdc30 --- /dev/null +++ b/integrations/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "substreams-integrations" +version = "0.1.0" +edition = "2021" + +[dependencies] +prost = "0.11.0" +prost-types = "0.11.1" +substreams = "0.0.20" +thiserror = "1.0.35" +tokio = { version = "1", features = ["macros", "rt"] } +tokio-stream = "0.1.10" +tonic = "0.8.1" + +[build-dependencies] +reqwest = { version = "0.11", features = ["json", "blocking"] } +tokio = { version = "1", features = ["macros", "rt"] } +tonic-build = "0.8.0" diff --git a/integrations/Makefile b/integrations/Makefile new file mode 100644 index 00000000..2388f607 --- /dev/null +++ b/integrations/Makefile @@ -0,0 +1,3 @@ +.PHONY: build +build: + cargo build \ No newline at end of file diff --git a/integrations/build.rs b/integrations/build.rs new file mode 100644 index 00000000..c7eeabca --- /dev/null +++ b/integrations/build.rs @@ -0,0 +1,58 @@ +use std::fs; +use std::path::Path; + +fn main() { + download_and_compile_substream_v1(); +} + +fn download_and_compile_substream_v1() { + const SUBSTREAMS_GITHUB_REV: &str = "https://raw.githubusercontent.com/streamingfast/substreams/73ec39f882638be99960b59feab8e23f2ea76c50"; + const SUBSTREAMS_PROTO_SRC: &str = "/proto/sf/substreams/v1/substreams.proto"; + const CLOCK_PROTO_SRC: &str = "/proto/sf/substreams/v1/clock.proto"; + const MODULE_PROTO_SRC: &str = "/proto/sf/substreams/v1/modules.proto"; + + const DEFAULT_TARGET_DIR: &str = "./target/sf/substreams/v1"; + + let out_dir = Path::new(DEFAULT_TARGET_DIR); + + fs::create_dir_all(out_dir).ok(); + + let substreams_proto = out_dir.join("substreams.proto"); + let clock_proto = out_dir.join("clock.proto"); + let module_proto = out_dir.join("modules.proto"); + + let substreams_proto_url = format!("{}/{}", SUBSTREAMS_GITHUB_REV, SUBSTREAMS_PROTO_SRC); + let clock_proto_url = format!("{}/{}", SUBSTREAMS_GITHUB_REV, CLOCK_PROTO_SRC); + let module_proto_url = format!("{}/{}", SUBSTREAMS_GITHUB_REV, MODULE_PROTO_SRC); + + let mut substreams_proto_file = fs::File::create(&substreams_proto).unwrap(); + let mut clock_proto_file = fs::File::create(&clock_proto).unwrap(); + let mut module_proto_file = fs::File::create(&module_proto).unwrap(); + + reqwest::blocking::get(&substreams_proto_url) + .unwrap() + .copy_to(&mut substreams_proto_file) + .unwrap(); + reqwest::blocking::get(&clock_proto_url) + .unwrap() + .copy_to(&mut clock_proto_file) + .unwrap(); + reqwest::blocking::get(&module_proto_url) + .unwrap() + .copy_to(&mut module_proto_file) + .unwrap(); + + tonic_build::configure() + .build_client(true) + .build_server(true) + .out_dir(out_dir) + .compile( + &[ + substreams_proto.as_path(), + clock_proto.as_path(), + module_proto.as_path(), + ], + &["./target"], + ) + .unwrap(); +} diff --git a/integrations/src/errors.rs b/integrations/src/errors.rs new file mode 100644 index 00000000..03186a55 --- /dev/null +++ b/integrations/src/errors.rs @@ -0,0 +1,17 @@ +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error(transparent)] + Io(#[from] std::io::Error), + + #[error("Missing module info: {0}")] + MissingModule(String), + + #[error("Substreams output is empty")] + EmptyOutput, + + #[error("Substreams output does not match cached output")] + OutputMismatch, + + #[error("Substreams error: {0}")] + Substream(String), +} diff --git a/integrations/src/lib.rs b/integrations/src/lib.rs new file mode 100644 index 00000000..b1ff6667 --- /dev/null +++ b/integrations/src/lib.rs @@ -0,0 +1,9 @@ +pub mod errors; +pub mod module; + +#[rustfmt::skip] +#[path = "../target/sf/substreams/v1/sf.substreams.v1.rs"] +pub mod substreams_client; + +#[cfg(test)] +mod tests; diff --git a/integrations/src/module.rs b/integrations/src/module.rs new file mode 100644 index 00000000..defc2d8c --- /dev/null +++ b/integrations/src/module.rs @@ -0,0 +1,88 @@ +use crate::errors::Error; + +const DEFAULT_START_BLOCK: u64 = 13e6 as u64; +const DEFAULT_NUM_BLOCKS: usize = 10; + +pub struct Module { + pub method: String, + pub path: String, + pub cache_file: String, + pub start_block: Option, + pub num_blocks: Option, +} + +impl Module { + pub fn new(method: &str, path: &str, cache_file: &str) -> Self { + Self { + method: method.to_string(), + path: path.to_string(), + cache_file: cache_file.to_string(), + start_block: None, + num_blocks: None, + } + } + + pub fn cached_output(&self) -> Result, Error> { + let cache = std::fs::read_to_string(&self.cache_file).ok(); + Ok(cache) + } + + pub fn check_cached_output(&self, output: String) -> Result<(), Error> { + let cache = self.cached_output()?; + + if output.is_empty() { + return Err(Error::EmptyOutput); + } else if cache.is_none() || cache.clone().unwrap().is_empty() { + self.cache_output(output)?; + } else if cache.unwrap() != output { + return Err(Error::OutputMismatch); + } + + Ok(()) + } + + pub fn cache_output(&self, output: String) -> Result<(), Error> { + if output.is_empty() { + return Err(Error::EmptyOutput); + } + + std::fs::write(&self.cache_file, output)?; + Ok(()) + } + + // Runs substream module using the `substreams` cli. + // NOTE: `substreams` cli must be installed to run this command. + pub fn run(&self) -> Result { + use std::process::Command; + + // generate pb files under src/pb + let output = Command::new("substreams") + .args(&[ + "run", + "-e", + "api-dev.streamingfast.io:443", + &self.path, + &self.method, + "-s", + self.start_block + .unwrap_or(DEFAULT_START_BLOCK) + .to_string() + .as_str(), + "-t", + format!("+{}", self.num_blocks.unwrap_or(DEFAULT_NUM_BLOCKS)).as_str(), + ]) + .output()?; + + if !output.stderr.is_empty() { + return Err(Error::Substream(String::from_utf8(output.stderr).unwrap())); + } + + let content = String::from_utf8_lossy(&output.stdout); + + Ok(content.to_string()) + } + + pub fn test(&self) -> Result<(), Error> { + self.check_cached_output(self.run()?) + } +} diff --git a/integrations/src/pb/messari.erc20_price.v1.rs b/integrations/src/pb/messari.erc20_price.v1.rs new file mode 100644 index 00000000..bb4e4ca0 --- /dev/null +++ b/integrations/src/pb/messari.erc20_price.v1.rs @@ -0,0 +1,54 @@ +// @generated +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Erc20Price { + #[prost(uint64, tag="1")] + pub block_number: u64, + #[prost(string, tag="2")] + pub price_usd: ::prost::alloc::string::String, + #[prost(bytes="vec", tag="3")] + pub token_address: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Erc20Prices { + #[prost(message, repeated, tag="1")] + pub items: ::prost::alloc::vec::Vec, +} +/// Encoded file descriptor set for the `messari.erc20_price.v1` package +pub const FILE_DESCRIPTOR_SET: &[u8] = &[ + 0x0a, 0xa8, 0x04, 0x0a, 0x11, 0x65, 0x72, 0x63, 0x32, 0x30, 0x5f, 0x70, 0x72, 0x69, 0x63, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x16, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x72, 0x69, 0x2e, + 0x65, 0x72, 0x63, 0x32, 0x30, 0x5f, 0x70, 0x72, 0x69, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x22, 0x71, + 0x0a, 0x0a, 0x45, 0x52, 0x43, 0x32, 0x30, 0x50, 0x72, 0x69, 0x63, 0x65, 0x12, 0x21, 0x0a, 0x0c, + 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x0b, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, + 0x1b, 0x0a, 0x09, 0x70, 0x72, 0x69, 0x63, 0x65, 0x5f, 0x75, 0x73, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x69, 0x63, 0x65, 0x55, 0x73, 0x64, 0x12, 0x23, 0x0a, 0x0d, + 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, + 0x73, 0x22, 0x47, 0x0a, 0x0b, 0x45, 0x52, 0x43, 0x32, 0x30, 0x50, 0x72, 0x69, 0x63, 0x65, 0x73, + 0x12, 0x38, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x22, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x72, 0x69, 0x2e, 0x65, 0x72, 0x63, 0x32, 0x30, 0x5f, + 0x70, 0x72, 0x69, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x52, 0x43, 0x32, 0x30, 0x50, 0x72, + 0x69, 0x63, 0x65, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x4a, 0xb6, 0x02, 0x0a, 0x06, 0x12, + 0x04, 0x00, 0x00, 0x0c, 0x01, 0x0a, 0x08, 0x0a, 0x01, 0x0c, 0x12, 0x03, 0x00, 0x00, 0x12, 0x0a, + 0x08, 0x0a, 0x01, 0x02, 0x12, 0x03, 0x02, 0x00, 0x1f, 0x0a, 0x0a, 0x0a, 0x02, 0x04, 0x00, 0x12, + 0x04, 0x04, 0x00, 0x08, 0x01, 0x0a, 0x0a, 0x0a, 0x03, 0x04, 0x00, 0x01, 0x12, 0x03, 0x04, 0x08, + 0x12, 0x0a, 0x0b, 0x0a, 0x04, 0x04, 0x00, 0x02, 0x00, 0x12, 0x03, 0x05, 0x02, 0x1a, 0x0a, 0x0c, + 0x0a, 0x05, 0x04, 0x00, 0x02, 0x00, 0x05, 0x12, 0x03, 0x05, 0x02, 0x08, 0x0a, 0x0c, 0x0a, 0x05, + 0x04, 0x00, 0x02, 0x00, 0x01, 0x12, 0x03, 0x05, 0x09, 0x15, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, + 0x02, 0x00, 0x03, 0x12, 0x03, 0x05, 0x18, 0x19, 0x0a, 0x0b, 0x0a, 0x04, 0x04, 0x00, 0x02, 0x01, + 0x12, 0x03, 0x06, 0x02, 0x17, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x01, 0x05, 0x12, 0x03, + 0x06, 0x02, 0x08, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x01, 0x01, 0x12, 0x03, 0x06, 0x09, + 0x12, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x01, 0x03, 0x12, 0x03, 0x06, 0x15, 0x16, 0x0a, + 0x0b, 0x0a, 0x04, 0x04, 0x00, 0x02, 0x02, 0x12, 0x03, 0x07, 0x02, 0x1a, 0x0a, 0x0c, 0x0a, 0x05, + 0x04, 0x00, 0x02, 0x02, 0x05, 0x12, 0x03, 0x07, 0x02, 0x07, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, + 0x02, 0x02, 0x01, 0x12, 0x03, 0x07, 0x08, 0x15, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x02, + 0x03, 0x12, 0x03, 0x07, 0x18, 0x19, 0x0a, 0x0a, 0x0a, 0x02, 0x04, 0x01, 0x12, 0x04, 0x0a, 0x00, + 0x0c, 0x01, 0x0a, 0x0a, 0x0a, 0x03, 0x04, 0x01, 0x01, 0x12, 0x03, 0x0a, 0x08, 0x13, 0x0a, 0x0b, + 0x0a, 0x04, 0x04, 0x01, 0x02, 0x00, 0x12, 0x03, 0x0b, 0x02, 0x20, 0x0a, 0x0c, 0x0a, 0x05, 0x04, + 0x01, 0x02, 0x00, 0x04, 0x12, 0x03, 0x0b, 0x02, 0x0a, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x01, 0x02, + 0x00, 0x06, 0x12, 0x03, 0x0b, 0x0b, 0x15, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x01, 0x02, 0x00, 0x01, + 0x12, 0x03, 0x0b, 0x16, 0x1b, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x01, 0x02, 0x00, 0x03, 0x12, 0x03, + 0x0b, 0x1e, 0x1f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +]; +// @@protoc_insertion_point(module) \ No newline at end of file diff --git a/integrations/src/tests.rs b/integrations/src/tests.rs new file mode 100644 index 00000000..6478d455 --- /dev/null +++ b/integrations/src/tests.rs @@ -0,0 +1,81 @@ +use crate::module::Module; + +#[test] +fn test_erc20_price_module_map_price() -> Result<(), Box> { + let module = Module::new( + "map_price", + "../erc20-price/substreams.yaml", + "./target/erc20-price-cache.txt", + ); + module.test()?; + + Ok(()) +} + +#[test] +fn test_erc20_market_cap_module_map_market_cap() -> Result<(), Box> { + let mut module = Module::new( + "map_market_cap", + "../erc20-market-cap/substreams.yaml", + "./target/erc20-market-cap-cache.txt", + ); + module.num_blocks = Some(1); + module.test()?; + + Ok(()) +} + +#[test] +fn test_erc721_module_block_to_transfers() -> Result<(), Box> { + let module = Module::new( + "block_to_transfers", + "../erc721/substreams.yaml", + "./target/erc721-cache.txt", + ); + module.test()?; + + Ok(()) +} + +#[test] +fn test_uniswap_v2_module_store_pools() -> Result<(), Box> { + let mut module = Module::new( + "map_pair_created_event,map_pools,store_pools", + "../uniswap-v2/substreams.yaml", + "./target/uniswap-v2-cache.txt", + ); + module.start_block = Some(10008355); + module.num_blocks = Some(200); + module.test()?; + + Ok(()) +} + +// NOTE: Receiving GOAWAY error with grpc clientl; +// May be due to improper API key; +// TODO: Contact SF to understand how to use the gRPC `blocks` +// endpoint. +#[ignore] +#[tokio::test] +async fn test_substreams_client() -> Result<(), Box> { + use crate::substreams_client::{stream_client::StreamClient, Request}; + use tokio_stream::StreamExt; + + const DEFAULT_SUBSTREAMS_ENDPOINT: &str = "https://api-dev.streamingfast.io:443"; + + let mut client = StreamClient::connect(DEFAULT_SUBSTREAMS_ENDPOINT).await?; + let mut request = Request::default(); + request.start_block_num = 13e5 as i64; + request.stop_block_num = (request.start_block_num + 10) as u64; + request.output_modules = vec!["sf.ethereum.v1".to_string()]; + let mut stream = client + .blocks(tonic::Request::new(request)) + .await? + .into_inner(); + + while let Some(output) = stream.next().await { + println!("{:?}", output?); + } + + Ok(()) +} diff --git a/integrations/substreams.yaml b/integrations/substreams.yaml new file mode 100644 index 00000000..f17ff784 --- /dev/null +++ b/integrations/substreams.yaml @@ -0,0 +1,21 @@ +specVersion: v0.1.0 +package: + name: "substreams_integrations" + version: v0.1.0 + +imports: + erc20Price: ./erc20-price-substreams-v0.1.0.spkg + +binaries: + default: + type: wasm/rust-v1 + file: ../target/wasm32-unknown-unknown/release/substreams_integration.wasm + +modules: + - name: map_market_cap + kind: map + initialBlock: 13000700 + inputs: + - map: erc20Price:map_price + output: + type: proto:messari.erc20_market_cap.v1.Erc20MarketCaps diff --git a/substreams-helper/build.rs b/substreams-helper/build.rs index fd709441..7bbeace3 100644 --- a/substreams-helper/build.rs +++ b/substreams-helper/build.rs @@ -2,6 +2,7 @@ use anyhow::{Ok, Result}; use substreams_common::codegen; fn main() -> Result<(), anyhow::Error> { + println!("cargo:rerun-if-changed=abi"); codegen::generate_abi(None)?; Ok(()) diff --git a/uniswap-v2/build.rs b/uniswap-v2/build.rs index 68bc205f..fb321d31 100644 --- a/uniswap-v2/build.rs +++ b/uniswap-v2/build.rs @@ -2,8 +2,8 @@ use anyhow::{Ok, Result}; use substreams_common::codegen; fn main() -> Result<(), anyhow::Error> { - // println!("cargo:rerun-if-changed=proto"); - // println!("cargo:rerun-if-changed=abi"); + println!("cargo:rerun-if-changed=proto"); + println!("cargo:rerun-if-changed=abi"); codegen::generate(None)?; Ok(()) diff --git a/uniswap-v2/src/pb.rs b/uniswap-v2/src/pb.rs index 34edaa73..278b1fd7 100644 --- a/uniswap-v2/src/pb.rs +++ b/uniswap-v2/src/pb.rs @@ -1,3 +1,3 @@ pub mod dex_amm; -pub mod uniswap; -pub mod common; \ No newline at end of file +pub mod common; +pub mod uniswap; \ No newline at end of file