diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dadbecc..f33fffa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,13 @@ name: Rust on: push: - branches: [ main ] + branches: + - main + - 0.7.x # we're going to maintain 0.7.x version until the adoption of 0.8.x will be completed pull_request: - branches: [ main ] + branches: + - main + - 0.7.x # we're going to maintain 0.7.x version until the adoption of 0.8.x will be completed env: CARGO_TERM_COLOR: always @@ -53,85 +57,22 @@ jobs: run: | cargo clippy - test: - runs-on: ubuntu-20.04 - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - profile: minimal - components: clippy - - name: Run tests - run: | - cargo test - - publish: - runs-on: ubuntu-20.04 - needs: [rustfmt, rustclippy] - if: github.ref == 'refs/heads/main' - + release-plz: + name: release-plz + runs-on: ubuntu-latest + needs: [check, rustclippy, rustfmt] + if: ${{ github.event_name != 'pull_request' }} # Specify the branch condition steps: - - name: Checkout Repository - uses: actions/checkout@v2 - with: - # fetch tags for cargo ws publish - # might be a simple `fetch-tags: true` option soon, see https://github.com/actions/checkout/pull/579 - fetch-depth: 0 - - - name: Setup - run: | - git config user.name github-actions - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - cargo install --git https://github.com/miraclx/cargo-workspaces --rev b2d49b9e575e29fd2395352e4d0df47def025039 cargo-workspaces - export GIT_PREVIOUS_TAG=$(git describe --tags --abbrev=0) - echo "GIT_PREVIOUS_TAG=${GIT_PREVIOUS_TAG}" >> $GITHUB_ENV - echo "[ pre run] current latest git tag is \"${GIT_PREVIOUS_TAG}\"" - - name: Publish to crates.io and tag the commit - id: tag-and-publish - env: - CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} - run: | - cargo ws publish --all --yes --exact \ - --skip-published --no-git-commit --allow-dirty \ - --tag-existing --tag-prefix 'v' \ - --tag-msg 'crates.io snapshot' --tag-msg '%{https://crates.io/crates/%n/%v}' \ - --no-individual-tags --no-git-push - export GIT_LATEST_TAG=$(git describe --tags --abbrev=0) - echo "GIT_LATEST_TAG=${GIT_LATEST_TAG}" >> $GITHUB_ENV - echo "[post run] current latest git tag is \"${GIT_LATEST_TAG}\"" - echo "::set-output name=tagged::$( [[ "$GIT_LATEST_TAG" == "$GIT_PREVIOUS_TAG" ]] && echo 0 || echo 1 )" - # returning multi-line outputs gets truncated to include only the first line - # we have to escape the newline chars, runner auto unescapes them later - # https://github.community/t/set-output-truncates-multiline-strings/16852/3 - GIT_TAG_MESSAGE="$(git tag -l --format='%(body)' ${GIT_LATEST_TAG})" - GIT_TAG_MESSAGE="${GIT_TAG_MESSAGE//'%'/'%25'}" - GIT_TAG_MESSAGE="${GIT_TAG_MESSAGE//$'\n'/'%0A'}" - GIT_TAG_MESSAGE="${GIT_TAG_MESSAGE//$'\r'/'%0D'}" - echo "::set-output name=git_tag_message::${GIT_TAG_MESSAGE}" - - name: Push tags to GitHub (if any) - if: steps.tag-and-publish.outputs.tagged == 1 - run: git push --tags - - - name: Extract release notes - if: steps.tag-and-publish.outputs.tagged == 1 - id: extract-release-notes - uses: ffurrer2/extract-release-notes@c24866884b7a0d2fd2095be2e406b6f260479da8 - - - name: Create release - if: steps.tag-and-publish.outputs.tagged == 1 - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ env.GIT_LATEST_TAG }} - release_name: ${{ env.GIT_LATEST_TAG }} - body: | - ## What's changed? - - ${{ steps.extract-release-notes.outputs.release_notes }} - **Crate Link**: ${{ steps.tag-and-publish.outputs.git_tag_message }} - **Full Changelog**: https://github.com/${{ github.repository }}/compare/${{ env.GIT_PREVIOUS_TAG }}...${{ env.GIT_LATEST_TAG }} + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + token: ${{ secrets.CUSTOM_GITHUB_TOKEN }} + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: Run release-plz + uses: MarcoIeni/release-plz-action@v0.5 + env: + # https://marcoieni.github.io/release-plz/github/trigger.html + GITHUB_TOKEN: ${{ secrets.CUSTOM_GITHUB_TOKEN }} + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index e3dd3a7..8d6c8ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,29 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased](https://github.com/near/near-lake-framework/compare/v0.7.2...HEAD) +- Simpler start boilerplate, simpler structures to deal with! + +### Breaking changes + +This version introduces a different much simplified concept of Lake Framework usage. Thus it brings breaking changes. + +We introduce `near-lake-primitives` crate with simplified primitive structures (e.g `Block`, `Transaction`, `StateChange`, etc.) which is heavily used by Lake Framework since now. + +And some other changes: + +- `LakeConfig` is renamed to be just `Lake`. It is done because since this update `Lake` is accepting the **indexing function** from a user and runs the streamer implicitly. Thus shortening and simplifying the start boilerplate to something like this: + ```rust + fn main() -> anyhow::Result<()> { + // Lake Framework start boilerplate + near_lake_framework::LakeBuilder::default() + .mainnet() + .start_block_height(80504433) + .build()? + .run(handle_block) // user-defined asynchronous function that handles each block + } + ``` + + Please note your main function isn't required to be asynchronous anymore! It is now handled by Lake Framework under the hood. ## [0.7.2](https://github.com/near/near-lake-framework/compare/v0.7.1...0.7.2) @@ -36,7 +59,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added proper error handling in a few places - Updated the dependencies version of AWS crates -## [0.6.0](https://github.com/near/near-lake-framework/compare/v0.5.2...0.6.0) +## [0.6.0](https://github.com/near/near-lake-framework/compare/v0.5.2...v0.6.0) - Upgrade underlying dependency `near-indexer-primitives` to versions between 0.15 and 0.16 @@ -51,7 +74,7 @@ a base64-encoded String that now became raw `Vec`: **Refer to this [`nearcore` commit](https://github.com/near/nearcore/commit/8e9be9fff4d520993c81b0e3738c0f223a9538c0) to find all the changes of this kind.** -## [0.5.2](https://github.com/near/near-lake-framework/compare/v0.5.1...0.5.2) +## [0.5.2](https://github.com/near/near-lake-framework/compare/v0.5.1...v0.5.2) - Fixed the bug that caused a lag by 100 blocks that was introduced in 0.5.1 diff --git a/Cargo.toml b/Cargo.toml index 2660d08..c15e293 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,40 +1,18 @@ -[package] -name = "near-lake-framework" -version = "0.0.0" # managed by cargo-workspaces, see below +[workspace] +members = [ + "lake-framework", + "lake-primitives", + "lake-parent-transaction-cache", + "lake-context-derive", +] + +# cargo-workspaces +[workspace.package] +version = "0.8.0" license = "MIT OR Apache-2.0" repository = "https://github.com/near/near-lake-framework" description = "Library to connect to the NEAR Lake S3 and stream the data" categories = ["asynchronous", "api-bindings", "network-programming"] keywords = ["near", "near-lake", "near-indexer"] authors = ["Near Inc "] -edition = "2021" -rust-version = "1.58.1" - -# cargo-workspaces -[workspace.metadata.workspaces] -version = "0.7.2" - -[dependencies] -anyhow = "1.0.51" -aws-config = "0.53.0" -aws-types = "0.53.0" -aws-credential-types = "0.53.0" -aws-sdk-s3 = "0.23.0" -async-stream = "0.3.3" -async-trait = "0.1.64" -derive_builder = "0.11.2" -futures = "0.3.23" -serde = { version = "1", features = ["derive"] } -serde_json = "1.0.75" -thiserror = "1.0.38" -tokio = { version = "1.28.2", features = ["sync", "time", "rt", "macros"] } -tokio-stream = { version = "0.1" } -tracing = "0.1.13" - -near-indexer-primitives = "0.17" - -[dev-dependencies] -aws-smithy-http = "0.53.0" - -[lib] -doctest = false +rust-version = "1.69.0" diff --git a/lake-context-derive/Cargo.toml b/lake-context-derive/Cargo.toml new file mode 100644 index 0000000..eda295d --- /dev/null +++ b/lake-context-derive/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "lake-context-derive" +version.workspace = true +# version = "0.0.0" # managed by cargo-workspaces +edition = "2021" + +[lib] +proc-macro = true + +[dependencies] +syn = "2.0" +quote = "1.0" \ No newline at end of file diff --git a/lake-context-derive/README.md b/lake-context-derive/README.md new file mode 100644 index 0000000..8f28e0e --- /dev/null +++ b/lake-context-derive/README.md @@ -0,0 +1,51 @@ +# Lake Context Derive + +Lake Context Derive is a Rust crate that provides a derive macro for easy and convenient implementation of the `near_lake_framework::LakeContextExt` trait. This trait has two functions: `execute_before_run` and `execute_after_run` that are executed before and after the user-provided indexer function respectively. + +## Usage + +The Lake Context Derive macro can be utilized by annotating the context struct with `#[derive(LakeContext)]`. This trait implementation will then facilitate the combination of different contexts. For instance, to use a `ParentTransactionCache` with some additional data, one would define a context like: + +```no_run +use lake_parent_transaction_cache::ParentTransactionCache; + +#[derive(LakeContext)] +struct MyContext { + db_connection_string: String, + parent_tx_cache: ParentTransactionCache, +} +``` + +### Instantiation + +You can create an instance of your context as follows: + +```no_run +let my_context = MyContext { + db_connection_string: String::from("postgres://user:pass@host/db"), + parent_tx_cache: ParentTransactionCache::default().build().unwrap(), +}; +``` + +### User Indexer Function + +This will simplify your indexer function signature. It now needs only the context as an additional parameter: + +```no_run +async fn handle_block( + mut block: Block, + ctx: &MyContext, +) -> anyhow::Result<()> { + // body +} +``` + +The Lake Context Derive will look for all fields in the struct that implement `LakeContextExt`, and will append their trait methods to the top-level calls. For `execute_before_run`, it is done in ascending order, and for `execute_after_run` in descending order. + +## Purpose + +The purpose of the Lake Context Derive crate is to alleviate some of the common pain points in context development and usage in Rust. By encapsulating and standardizing the handling of these function calls, we aim to create a more accessible and user-friendly approach to context implementation. + +## Collaboration + +We hope that this tool will be useful for the Rust community and look forward to seeing how it can be used in a range of different projects. We encourage community contributions, whether that's through sharing your own unique context implementations or by providing feedback and suggestions for how we can continue to improve the Lake Context Derive. diff --git a/lake-context-derive/src/lib.rs b/lake-context-derive/src/lib.rs new file mode 100644 index 0000000..57da168 --- /dev/null +++ b/lake-context-derive/src/lib.rs @@ -0,0 +1,80 @@ +#![doc = include_str!("../README.md")] +use proc_macro::TokenStream; +use quote::quote; + +#[proc_macro_derive(LakeContext)] +pub fn lake_context_derive(input: TokenStream) -> TokenStream { + let input = syn::parse_macro_input!(input as syn::DeriveInput); + + // Used in the quasi-quotation below as `#name`. + let name = input.ident; + + // Build the trait impl. + // Iterate over all fields and for each field generate a call to `execute_before_run`. + // if the field is a an impl of LakeContext, then call `execute_before_run` on the struct. + + let fields = if let syn::Data::Struct(syn::DataStruct { + fields: syn::Fields::Named(syn::FieldsNamed { named, .. }), + .. + }) = &input.data + { + named + } else { + unimplemented!(); + }; + + let calls_before_run = fields + .iter() + .filter(|f| { + let ty = &f.ty; + if let syn::Type::Path(syn::TypePath { path, .. }) = ty { + if let Some(ident) = path.get_ident() { + ident == "LakeContext" + } else { + false + } + } else { + false + } + }) + .map(|f| { + let name = &f.ident; + quote! { self.#name.execute_before_run(block); } + }); + + let calls_after_run = fields + .iter() + .rev() + .filter(|f| { + let ty = &f.ty; + if let syn::Type::Path(syn::TypePath { path, .. }) = ty { + if let Some(ident) = path.get_ident() { + ident == "LakeContext" + } else { + false + } + } else { + false + } + }) + .map(|f| { + let name = &f.ident; + quote! { self.#name.execute_after_run(); } + }); + + let expanded = quote! { + // The generated impl. + impl near_lake_framework::LakeContextExt for #name { + fn execute_before_run(&self, block: &mut near_lake_primitives::block::Block) { + #( #calls_before_run )* + } + + fn execute_after_run(&self) { + #( #calls_after_run )* + } + } + }; + + // Hand the output tokens back to the compiler. + proc_macro::TokenStream::from(expanded) +} diff --git a/lake-framework/Cargo.toml b/lake-framework/Cargo.toml new file mode 100644 index 0000000..ff4a14e --- /dev/null +++ b/lake-framework/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "near-lake-framework" +version.workspace = true +edition = "2021" + +[dependencies] +aws-config = "0.53.0" +aws-types = "0.53.0" +aws-credential-types = "0.53.0" +aws-sdk-s3 = "0.23.0" +async-stream = "0.3.3" +async-trait = "0.1.64" +derive_builder = "0.11.2" +futures = "0.3.23" +serde = { version = "1", features = ["derive"] } +serde_json = "1.0.75" +thiserror = "1.0.38" +tokio = { version = "1.1", features = ["sync", "time", "rt-multi-thread"] } +tokio-stream = { version = "0.1" } +tracing = "0.1.13" + +near-lake-primitives = { path = "../lake-primitives" } +lake-context-derive = { path = "../lake-context-derive" } + +[dev-dependencies] +aws-smithy-http = "0.53.0" +# use by examples +anyhow = "1.0.51" + +# used by nft_indexer example +regex = "1.5.4" +once_cell = "1.8.0" + +# used in the doc examples +diesel = { version = "2", features= ["postgres_backend", "postgres"] } + +# used by with_context_parent_tx_cache example +lake-parent-transaction-cache = { path = "../lake-parent-transaction-cache" } diff --git a/lake-framework/README.md b/lake-framework/README.md new file mode 100644 index 0000000..0dc293b --- /dev/null +++ b/lake-framework/README.md @@ -0,0 +1,313 @@ +# NEAR Lake Framework + +NEAR Lake Framework is a small library companion to [NEAR Lake](https://github.com/near/near-lake). It allows you to build +your own indexer that subscribes to the stream of blocks from the NEAR Lake data source and create your own logic to process +the NEAR Protocol data. + +## Example + +```no_run +fn main() -> anyhow::Result<()> { + near_lake_framework::LakeBuilder::default() + .testnet() + .start_block_height(112205773) + .build()? + .run(handle_block)?; + Ok(()) +} + +// The handler function to take the `Block` +// and print the block height +async fn handle_block( + block: near_lake_primitives::block::Block, +) -> anyhow::Result<()> { + eprintln!( + "Block #{}", + block.block_height(), + ); +# Ok(()) +} +``` + +### Pass the context to the function + +```no_run +#[derive(near_lake_framework::LakeContext)] +struct MyContext { + my_field: String +} + +fn main() -> anyhow::Result<()> { + + let context = MyContext { + my_field: "My value".to_string(), + }; + + near_lake_framework::LakeBuilder::default() + .testnet() + .start_block_height(112205773) + .build()? + .run_with_context(handle_block, &context)?; + + Ok(()) +} + +// The handler function to take the `Block` +// and print the block height +async fn handle_block( + block: near_lake_primitives::block::Block, + context: &MyContext, +) -> anyhow::Result<()> { + eprintln!( + "Block #{} / {}", + block.block_height(), + context.my_field, + ); +# Ok(()) +} +``` + +## Parent Transaction for the Receipt Context + +It is an old problem that the NEAR Protocol doesn't provide the parent transaction hash in the receipt. This is a problem for the indexer that needs to know the parent transaction hash to build the transaction tree. We've got you covered with the [`lake-parent-transaction-cache`](../lake-parent-transaction-cache/) crate that provides a cache for the parent transaction hashes. + +```no_run +use near_lake_framework::near_lake_primitives; +use near_lake_primitives::CryptoHash; +use lake_parent_transaction_cache::{ParentTransactionCache, ParentTransactionCacheBuilder}; +use near_lake_primitives::actions::ActionMetaDataExt; + +fn main() -> anyhow::Result<()> { + let parent_transaction_cache_ctx = ParentTransactionCacheBuilder::default() + .build()?; + // Lake Framework start boilerplate + near_lake_framework::LakeBuilder::default() + .mainnet() + .start_block_height(88444526) + .build()? + // developer-defined async function that handles each block + .run_with_context(print_function_call_tx_hash, &parent_transaction_cache_ctx)?; + Ok(()) +} + +async fn print_function_call_tx_hash( + mut block: near_lake_primitives::block::Block, + ctx: &ParentTransactionCache, +) -> anyhow::Result<()> { + // Cache has been updated before this function is called. + let block_height = block.block_height(); + let actions: Vec<( + &near_lake_primitives::actions::FunctionCall, + Option, + )> = block + .actions() + .filter_map(|action| action.as_function_call()) + .map(|action| { + ( + action, + ctx.get_parent_transaction_hash(&action.receipt_id()), + ) + }) + .collect(); + + if !actions.is_empty() { + // Here's the usage of the context. + println!("Block #{:?}\n{:#?}", block_height, actions); + } + + Ok(()) +} +``` + +## Tutorials: + + - + - [Migrating to NEAR Lake Framework](https://near-indexers.io/tutorials/lake/migrating-to-near-lake-framework) from [NEAR Indexer Framework](https://near-indexers.io/docs/projects/near-indexer-framework) + +### More examples + +You might want to have a look at the always up-to-date examples in [`examples`](https://github.com/near/near-lake-framework-rs/tree/main/lake-framework/examples) folder. + +Other examples that we try to keep up-to-date but we might fail sometimes: + + - simple example of a data printer built on top of NEAR Lake Framework + - another simple example of the indexer built on top of NEAR Lake Framework for a tutorial purpose + + - an example of the indexer built on top of NEAR Lake Framework that watches for transactions related to specified account(s) + - a community-made project that uses NEAR Lake Framework + +## How to use + +### AWS S3 Credentials + +In order to be able to get objects from the AWS S3 bucket you need to provide the AWS credentials. + +#### Passing credentials to the config builder + +```rust +use near_lake_framework::LakeBuilder; + +# fn main() { +let credentials = aws_credential_types::Credentials::new( + "AKIAIOSFODNN7EXAMPLE", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + None, + None, + "custom_credentials", +); +let s3_config = aws_sdk_s3::Config::builder() + .credentials_provider(credentials) + .build(); + +let lake = LakeBuilder::default() + .s3_config(s3_config) + .s3_bucket_name("near-lake-data-custom") + .s3_region_name("eu-central-1") + .start_block_height(1) + .build() + .expect("Failed to build LakeConfig"); +# } + ``` + +**You should never hardcode your credentials, it is insecure. Use the described method to pass the credentials you read from CLI arguments** + +#### File-based AWS credentials +AWS default profile configuration with aws configure looks similar to the following: + +`~/.aws/credentials` +```text +[default] +aws_access_key_id=AKIAIOSFODNN7EXAMPLE +aws_secret_access_key=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +``` + +[AWS docs: Configuration and credential file settings](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html) + +### Environmental variables + +Alternatively, you can provide your AWS credentials via environment variables with constant names: + +```text +$ export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE +$ AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +$ AWS_DEFAULT_REGION=eu-central-1 +``` + +### Dependencies + +Add the following dependencies to your `Cargo.toml` + +```toml +... +[dependencies] +futures = "0.3.5" +itertools = "0.10.3" +tokio = { version = "1.1", features = ["sync", "time", "macros", "rt-multi-thread"] } +tokio-stream = { version = "0.1" } + +# NEAR Lake Framework +near-lake-framework = "0.8.0" +``` + +### Custom S3 storage + +In case you want to run your own [near-lake](https://github.com/near/near-lake) instance and store data in some S3 compatible storage ([Minio](https://min.io/) or [Localstack](https://localstack.cloud/) as example) + You can owerride default S3 API endpoint by using `s3_endpoint` option + + - run minio + +```bash +$ mkdir -p /data/near-lake-custom && minio server /data +``` + + - pass custom `aws_sdk_s3::config::Config` to the [LakeBuilder] + +``` +use near_lake_framework::LakeBuilder; + +# #[tokio::main] +# async fn main() { +let aws_config = aws_config::from_env().load().await; +let mut s3_conf = aws_sdk_s3::config::Builder::from(&aws_config) + .endpoint_url("http://0.0.0.0:9000") + .build(); + +let lake = LakeBuilder::default() + .s3_config(s3_conf) + .s3_bucket_name("near-lake-data-custom") + .s3_region_name("eu-central-1") + .start_block_height(1) + .build() + .expect("Failed to build LakeConfig"); +# } +``` + +## Configuration + +Everything should be configured before the start of your indexer application via `LakeConfigBuilder` struct. + +Available parameters: + + * [`start_block_height(value: u64)`](LakeConfigBuilder::start_block_height) - block height to start the stream from + * *optional* [`s3_bucket_name(value: impl Into)`](LakeConfigBuilder::s3_bucket_name) - provide the AWS S3 bucket name (you need to provide it if you use custom S3-compatible service, otherwise you can use [LakeConfigBuilder::mainnet] and [LakeConfigBuilder::testnet]) + * *optional* [`LakeConfigBuilder::s3_region_name(value: impl Into)`](LakeConfigBuilder::s3_region_name) - provide the AWS S3 region name (if you need to set a custom one) + * *optional* [`LakeConfigBuilder::s3_config(value: aws_sdk_s3::config::Config`](LakeConfigBuilder::s3_config) - provide custom AWS SDK S3 Config + +## Cost estimates (Updated Mar 10, 2022 with more precise calculations) + +**TL;DR** approximately $20 per month (for AWS S3 access, paid directly to AWS) for the reading of fresh blocks + +### Historical indexing + +| Blocks | GET | LIST | Subtotal GET | Subtotal LIST | Total $ | +|---|---|---|---|---|---| +| 1000 | 5000 | 4 | 0.00215 | 0.0000216 | $0.00 | +| 86,400 | 432000 | 345.6 | 0.18576 | 0.00186624 | $0.19 | +| 2,592,000 | 12960000 | 10368 | 5.5728 | 0.0559872 | $5.63 | +| 77,021,059 | 385105295 | 308084.236 | 165.5952769 | 1.663654874 | $167.26 | + +**Note:** ~77m of blocks is the number of blocks on the moment I was calculating. + +**84,400 blocks is approximate number of blocks per day** (1 block per second * 60 seconds * 60 minutes * 24 hours) + +**2,592,000 blocks is approximate number of blocks per months** (86,400 blocks per day * 30 days) + +### Tip of the network indexing + +| Blocks | GET | LIST | Subtotal GET | Subtotal LIST | Total $ | +|---|---|---|---|---|---| +| 1000 | 5000 | 1000 | 0.00215 | 0.0054 | $0.01 | +| 86,400 | 432000 | 86,400 | 0.18576 | 0.46656 | $0.65 | +| 2,592,000 | 12960000 | 2,592,000 | 5.5728 | 13.9968 | $19.57 | +| 77,021,059 | 385105295 | 77,021,059 | 165.5952769 | 415.9137186 | $581.51 | + +Explanation: + +Assuming NEAR Protocol produces accurately 1 block per second (which is really not, the average block production time is 1.3s). A full day consists of 86400 seconds, that's the max number of blocks that can be produced. + +According to the [Amazon S3 prices](https://aws.amazon.com/s3/pricing/?nc1=h_ls) `list` requests are charged for $0.0054 per 1000 requests and `get` is charged for $0.00043 per 1000 requests. + +Calculations (assuming we are following the tip of the network all the time): + +```text +86400 blocks per day * 5 requests for each block / 1000 requests * $0.0004 per 1k requests = $0.19 * 30 days = $5.7 +``` +**Note:** 5 requests for each block means we have 4 shards (1 file for common block data and 4 separate files for each shard) + +And a number of `list` requests we need to perform for 30 days: + +```text +86400 blocks per day / 1000 requests * $0.005 per 1k list requests = $0.47 * 30 days = $14.1 + +$5.7 + $14.1 = $19.8 +``` + +The price depends on the number of shards + +## Future plans + +We use Milestones with clearly defined acceptance criteria: + + * [x] [MVP](https://github.com/near/near-lake-framework/milestone/1) + * [ ] [0.8 High-level update](https://github.com/near/near-lake-framework-rs/milestone/3) + * [ ] [1.0](https://github.com/near/near-lake-framework/milestone/2) diff --git a/lake-framework/blocks/000000879765/block.json b/lake-framework/blocks/000000879765/block.json new file mode 100644 index 0000000..1581ebb --- /dev/null +++ b/lake-framework/blocks/000000879765/block.json @@ -0,0 +1,63 @@ +{ + "author": "test.near", + "header": { + "height": 879765, + "prev_height": 879764, + "epoch_id": "Hp4sw9ZGSceYadnvh7NpYJVVK7rcdir48jfrsxvwKQu9", + "next_epoch_id": "4h5mecoLYVFeZxAMAX3Mq3GQfEnuvSAPPo9kEpr4rGUL", + "hash": "95K8Je1iAVqieVU8ZuGgSdbvYs8T9rL6ER1XnRekMGbj", + "prev_hash": "9Da84RTsubZPcLxzK1K6JkCnDnMn4DxaSRzJPtnYJXUM", + "prev_state_root": "6zDM1UGLsZ7HnyUofDrTF73gv5vk2N614ViDkXBkq4ej", + "chunk_receipts_root": "9ETNjrt6MkwTgSVMMbpukfxRshSD1avBUUa4R4NuqwHv", + "chunk_headers_root": "4otZ2Zj1wANZweh33kWETr3VbF3HwW9zWET4YRYTo2pL", + "chunk_tx_root": "9rdfzfYzJMZyaj2yMvjget2ZsPNbZhKqY1qUXc1urDfu", + "outcome_root": "7tkzFg8RHBmMw1ncRJZCCZAizgq4rwCftTKYLce8RU8t", + "chunks_included": 1, + "challenges_root": "11111111111111111111111111111111", + "timestamp": 1676913656724153000, + "timestamp_nanosec": "1676913656724153000", + "random_value": "Au7bq9XzGAhDm2wb4PxbXQnTngzVTcWYa76Govx6n7NK", + "validator_proposals": [], + "chunk_mask": [ + true + ], + "gas_price": "100000000", + "block_ordinal": 879714, + "rent_paid": "0", + "validator_reward": "0", + "total_supply": "2085303629225498163419972383984892", + "challenges_result": [], + "last_final_block": "BS9QJenf3N9pKy8PZ5xRuowZi9X9T4sSDDu4i3i5UJZe", + "last_ds_final_block": "9Da84RTsubZPcLxzK1K6JkCnDnMn4DxaSRzJPtnYJXUM", + "next_bp_hash": "EtsYQonaJ7n5nRt32XJC5dBxxBxh7a9UVApykmmt8fCQ", + "block_merkle_root": "CqRoDd8BR4su7Z8vSfvg45HrugZnwbMbnXHRTWYQkWfZ", + "epoch_sync_data_hash": null, + "approvals": [ + "ed25519:3RBQ4PnfBbnDn8WnCScQJH9asjkicuhZZo36aa6FVa2Lbnj531NLiBkTmj8rhg5vfsarmYLgQmcMcXRuJ4jkzKns" + ], + "signature": "ed25519:2dWsY1QadJyNaVkyga5Wcj9DFRizAyFc9STjyN5Mtxc59ZzNYqML6qQTgtLeCYkpCy1h7kG34jcALTpEDQpkBoKQ", + "latest_protocol_version": 59 + }, + "chunks": [ + { + "chunk_hash": "7Ewp1AnL6o29UXLW2up9miQBdSaKxCnfRyhMGt9G4epN", + "prev_block_hash": "9Da84RTsubZPcLxzK1K6JkCnDnMn4DxaSRzJPtnYJXUM", + "outcome_root": "11111111111111111111111111111111", + "prev_state_root": "2ViDp7rmam77VmhY5C9KW92a6mgUTCKQ3Scz8tFyH13z", + "encoded_merkle_root": "44MrDjQzt1jU5PGUYY69THZ4g3SsfQiNiKKorey3GVtq", + "encoded_length": 364, + "height_created": 879765, + "height_included": 879765, + "shard_id": 0, + "gas_used": 0, + "gas_limit": 1000000000000000, + "rent_paid": "0", + "validator_reward": "0", + "balance_burnt": "0", + "outgoing_receipts_root": "H4Rd6SGeEBTbxkitsCdzfu9xL9HtZ2eHoPCQXUeZ6bW4", + "tx_root": "GKd8Evs3JdahRpS8q14q6RzzkodzFiSQPcH4yJxs4ZjG", + "validator_proposals": [], + "signature": "ed25519:2qev3mWQdYLi9aPwCnFHt22GFxhuGTGfnaz3msGcduUdXeycTQDBkY4EyQzpph4frXCybuYHE6g4GFxD2HVmWbJY" + } + ] +} diff --git a/lake-framework/blocks/000000879765/shard_0.json b/lake-framework/blocks/000000879765/shard_0.json new file mode 100644 index 0000000..2178ac2 --- /dev/null +++ b/lake-framework/blocks/000000879765/shard_0.json @@ -0,0 +1,239 @@ +{ + "shard_id": 0, + "chunk": { + "author": "test.near", + "header": { + "chunk_hash": "7Ewp1AnL6o29UXLW2up9miQBdSaKxCnfRyhMGt9G4epN", + "prev_block_hash": "9Da84RTsubZPcLxzK1K6JkCnDnMn4DxaSRzJPtnYJXUM", + "outcome_root": "11111111111111111111111111111111", + "prev_state_root": "2ViDp7rmam77VmhY5C9KW92a6mgUTCKQ3Scz8tFyH13z", + "encoded_merkle_root": "44MrDjQzt1jU5PGUYY69THZ4g3SsfQiNiKKorey3GVtq", + "encoded_length": 364, + "height_created": 879765, + "height_included": 0, + "shard_id": 0, + "gas_used": 0, + "gas_limit": 1000000000000000, + "rent_paid": "0", + "validator_reward": "0", + "balance_burnt": "0", + "outgoing_receipts_root": "H4Rd6SGeEBTbxkitsCdzfu9xL9HtZ2eHoPCQXUeZ6bW4", + "tx_root": "GKd8Evs3JdahRpS8q14q6RzzkodzFiSQPcH4yJxs4ZjG", + "validator_proposals": [], + "signature": "ed25519:2qev3mWQdYLi9aPwCnFHt22GFxhuGTGfnaz3msGcduUdXeycTQDBkY4EyQzpph4frXCybuYHE6g4GFxD2HVmWbJY" + }, + "transactions": [ + { + "transaction": { + "signer_id": "test.near", + "public_key": "ed25519:8Rn4FJeeRYcrLbcrAQNFVgvbZ2FCEQjgydbXwqBwF1ib", + "nonce": 39, + "receiver_id": "test.near", + "actions": [ + { + "Delegate": { + "delegate_action": { + "sender_id": "test.near", + "receiver_id": "test.near", + "actions": [ + { + "AddKey": { + "public_key": "ed25519:CnQMksXTTtn81WdDujsEMQgKUMkFvDJaAjDeDLTxVrsg", + "access_key": { + "nonce": 0, + "permission": "FullAccess" + } + } + } + ], + "nonce": 879546, + "max_block_height": 100, + "public_key": "ed25519:8Rn4FJeeRYcrLbcrAQNFVgvbZ2FCEQjgydbXwqBwF1ib" + }, + "signature": "ed25519:25uGrsJNU3fVgUpPad3rGJRy2XQum8gJxLRjKFCbd7gymXwUxQ9r3tuyBCD6To7SX5oSJ2ScJZejwqK1ju8WdZfS" + } + } + ], + "signature": "ed25519:3vKF31u2naSjow1uQEfkoWy834fu9xhk66oBfTAYL3XVtJVAf1FREt7owJzwyRrN5F4mtd1rkvv1iTPTL86Szb2j", + "hash": "EZnJpyJDnkwnadB1V8PqjVMx7oe2zLhUMtJ8v6EUh1NQ" + }, + "outcome": { + "execution_outcome": { + "proof": [ + { + "hash": "7kPZTTVYJHvUg4g3S7SFErkKs18Ex1kN4rESnZwtJb2U", + "direction": "Right" + } + ], + "block_hash": "95K8Je1iAVqieVU8ZuGgSdbvYs8T9rL6ER1XnRekMGbj", + "id": "EZnJpyJDnkwnadB1V8PqjVMx7oe2zLhUMtJ8v6EUh1NQ", + "outcome": { + "logs": [], + "receipt_ids": [ + "AQDQ9G4QpK7x2inV3GieVEbqeoCGF9nmvrViQ2UgEXDQ" + ], + "gas_burnt": 409824625000, + "tokens_burnt": "40982462500000000000", + "executor_id": "test.near", + "status": { + "SuccessReceiptId": "AQDQ9G4QpK7x2inV3GieVEbqeoCGF9nmvrViQ2UgEXDQ" + }, + "metadata": { + "version": 1, + "gas_profile": null + } + } + }, + "receipt": null + } + } + ], + "receipts": [ + { + "predecessor_id": "test.near", + "receiver_id": "test.near", + "receipt_id": "AQDQ9G4QpK7x2inV3GieVEbqeoCGF9nmvrViQ2UgEXDQ", + "receipt": { + "Action": { + "signer_id": "test.near", + "signer_public_key": "ed25519:8Rn4FJeeRYcrLbcrAQNFVgvbZ2FCEQjgydbXwqBwF1ib", + "gas_price": "100000000", + "output_data_receivers": [], + "input_data_ids": [], + "actions": [ + { + "Delegate": { + "delegate_action": { + "sender_id": "test.near", + "receiver_id": "test.near", + "actions": [ + { + "AddKey": { + "public_key": "ed25519:CnQMksXTTtn81WdDujsEMQgKUMkFvDJaAjDeDLTxVrsg", + "access_key": { + "nonce": 0, + "permission": "FullAccess" + } + } + } + ], + "nonce": 879546, + "max_block_height": 100, + "public_key": "ed25519:8Rn4FJeeRYcrLbcrAQNFVgvbZ2FCEQjgydbXwqBwF1ib" + }, + "signature": "ed25519:25uGrsJNU3fVgUpPad3rGJRy2XQum8gJxLRjKFCbd7gymXwUxQ9r3tuyBCD6To7SX5oSJ2ScJZejwqK1ju8WdZfS" + } + } + ] + } + } + } + ] + }, + "receipt_execution_outcomes": [ + { + "execution_outcome": { + "proof": [ + { + "hash": "6vBgNYcwx6pcESfrw5YRBRamatBH8red3GEt3s3ntefm", + "direction": "Left" + } + ], + "block_hash": "95K8Je1iAVqieVU8ZuGgSdbvYs8T9rL6ER1XnRekMGbj", + "id": "AQDQ9G4QpK7x2inV3GieVEbqeoCGF9nmvrViQ2UgEXDQ", + "outcome": { + "logs": [], + "receipt_ids": [ + "5rc8UEhD4hmNQ3pJJM5Xc3VHeLXpCQqkA3ep8ag4aaDA" + ], + "gas_burnt": 308059500000, + "tokens_burnt": "30805950000000000000", + "executor_id": "test.near", + "status": { + "Failure": { + "ActionError": { + "index": 0, + "kind": "DelegateActionExpired" + } + } + }, + "metadata": { + "version": 3, + "gas_profile": [] + } + } + }, + "receipt": { + "predecessor_id": "test.near", + "receiver_id": "test.near", + "receipt_id": "AQDQ9G4QpK7x2inV3GieVEbqeoCGF9nmvrViQ2UgEXDQ", + "receipt": { + "Action": { + "signer_id": "test.near", + "signer_public_key": "ed25519:8Rn4FJeeRYcrLbcrAQNFVgvbZ2FCEQjgydbXwqBwF1ib", + "gas_price": "100000000", + "output_data_receivers": [], + "input_data_ids": [], + "actions": [ + { + "Delegate": { + "delegate_action": { + "sender_id": "test.near", + "receiver_id": "test.near", + "actions": [ + { + "AddKey": { + "public_key": "ed25519:CnQMksXTTtn81WdDujsEMQgKUMkFvDJaAjDeDLTxVrsg", + "access_key": { + "nonce": 0, + "permission": "FullAccess" + } + } + } + ], + "nonce": 879546, + "max_block_height": 100, + "public_key": "ed25519:8Rn4FJeeRYcrLbcrAQNFVgvbZ2FCEQjgydbXwqBwF1ib" + }, + "signature": "ed25519:25uGrsJNU3fVgUpPad3rGJRy2XQum8gJxLRjKFCbd7gymXwUxQ9r3tuyBCD6To7SX5oSJ2ScJZejwqK1ju8WdZfS" + } + } + ] + } + } + } + } + ], + "state_changes": [ + { + "cause": { + "type": "transaction_processing", + "tx_hash": "EZnJpyJDnkwnadB1V8PqjVMx7oe2zLhUMtJ8v6EUh1NQ" + }, + "type": "account_update", + "change": { + "account_id": "test.near", + "amount": "999999549946933447300000000000000", + "locked": "81773107345435833494396250588347", + "code_hash": "11111111111111111111111111111111", + "storage_usage": 182, + "storage_paid_at": 0 + } + }, + { + "cause": { + "type": "transaction_processing", + "tx_hash": "EZnJpyJDnkwnadB1V8PqjVMx7oe2zLhUMtJ8v6EUh1NQ" + }, + "type": "access_key_update", + "change": { + "account_id": "test.near", + "public_key": "ed25519:8Rn4FJeeRYcrLbcrAQNFVgvbZ2FCEQjgydbXwqBwF1ib", + "access_key": { + "nonce": 39, + "permission": "FullAccess" + } + } + } + ] +} diff --git a/lake-framework/examples/actions.rs b/lake-framework/examples/actions.rs new file mode 100644 index 0000000..227ca39 --- /dev/null +++ b/lake-framework/examples/actions.rs @@ -0,0 +1,37 @@ +//! This example shows how to filter actions in a block. +//! It it a more real-life example than the simple example. +//! It is going to follow the NEAR Social contract and print all function calls to it. +use near_lake_framework::near_lake_primitives; +// We need to import this trait to use the `as_function_call` method. +use near_lake_primitives::actions::ActionMetaDataExt; + +const CONTRACT_ID: &str = "social.near"; + +fn main() -> anyhow::Result<()> { + eprintln!("Starting..."); + // Lake Framework start boilerplate + near_lake_framework::LakeBuilder::default() + .mainnet() + .start_block_height(88444526) + .build()? + // developer-defined async function that handles each block + .run(print_function_calls_to_my_account)?; + Ok(()) +} + +async fn print_function_calls_to_my_account( + mut block: near_lake_primitives::block::Block, +) -> anyhow::Result<()> { + let block_height = block.block_height(); + let actions: Vec<&near_lake_primitives::actions::FunctionCall> = block + .actions() + .filter(|action| action.receiver_id().as_str() == CONTRACT_ID) + .filter_map(|action| action.as_function_call()) + .collect(); + + if !actions.is_empty() { + println!("Block #{:?}\n{:#?}", block_height, actions); + } + + Ok(()) +} diff --git a/lake-framework/examples/nft_indexer.rs b/lake-framework/examples/nft_indexer.rs new file mode 100644 index 0000000..7b8b45a --- /dev/null +++ b/lake-framework/examples/nft_indexer.rs @@ -0,0 +1,195 @@ +//! This is a more complex real-life example of how to use the NEAR Lake Framework. +//! +//! It is going to follow the network and watch for the Events according to the +//! [Events Format][1]. It will monitor for nft_mint events from the known +//! marketplaces, such as Mintbase and Paras, and index them to print in the terminal. +//! +//! [1]: https://nomicon.io/Standards/EventsFormat +use near_lake_framework::near_lake_primitives; +use regex::Regex; + +use once_cell::sync::Lazy; + +static MINTBASE_STORE_REGEXP: Lazy = + Lazy::new(|| Regex::new(r"^*.mintbase\d+.near$").unwrap()); + +fn main() -> anyhow::Result<()> { + eprintln!("Starting..."); + // Lake Framework start boilerplate + near_lake_framework::LakeBuilder::default() + .testnet() + .start_block_height(112205773) + .build()? + .run(handle_block)?; // developer-defined async function that handles each block + Ok(()) +} + +async fn handle_block(mut block: near_lake_primitives::block::Block) -> anyhow::Result<()> { + // Indexing lines START + let nfts: Vec = block + .events() // fetching all the events that occurred in the block + .filter(|event| event.standard() == "nep171") + .filter(|event| event.event() == "nft_mint") // filter them by "nft_mint" event only + .filter_map(|event| parse_event(event)) + .collect(); + // Indexing lines END + + if !nfts.is_empty() { + println!("We caught freshly minted NFTs!\n{:#?}", nfts); + } + Ok(()) +} + +// ================================================================ +// The following lines define structures and methods that support +// the goal of indexing NFT MINT events and printing links to newly +// created NFTs. +// These lines are not related to the NEAR Lake Framework. +// This logic is developer-defined and tailored to their indexing needs. +// ================================================================ + +/// Parses the given event to extract NFT data for known Marketplaces (Mintbase and Paras). +/// +/// The function parses the event data to extract the owner and link to the NFT, then filters out any +/// Marketplaces or contracts that it doesn't know how to parse. The resulting NFT data is returned +/// as an `Option`. Note that the logic used in this function is specific to the needs +/// of this application and does not relate to the Lake Framework. +/// +/// # Arguments +/// +/// * `event` - The event to parse for NFT data. +/// +/// # Returns +/// +/// An `Option` containing the extracted NFT data, or `None` if the event data could not +/// be parsed. +fn parse_event(event: &near_lake_primitives::events::Event) -> Option { + let marketplace = { + if MINTBASE_STORE_REGEXP.is_match(event.related_receipt_receiver_id().as_str()) { + Marketplace::Mintbase + } else if event.related_receipt_receiver_id().as_str() == "x.paras.near" { + Marketplace::Paras + } else { + Marketplace::Unknown + } + }; + + if let Some(event_data) = event.data() { + if let Some(nfts) = marketplace + .convert_event_data_to_nfts(event_data.clone(), event.related_receipt_receiver_id()) + { + Some(NFTReceipt { + receipt_id: event.related_receipt_id().to_string(), + marketplace_name: marketplace.name(), + nfts, + }) + } else { + None + } + } else { + None + } +} + +enum Marketplace { + Mintbase, + Paras, + Unknown, +} + +impl Marketplace { + fn name(&self) -> String { + match self { + Self::Mintbase => "Mintbase".to_string(), + Self::Paras => "Paras".to_string(), + Self::Unknown => "Unknown".to_string(), + } + } + fn convert_event_data_to_nfts( + &self, + event_data: serde_json::Value, + receiver_id: &near_lake_primitives::near_primitives::types::AccountId, + ) -> Option> { + match self { + Self::Mintbase => Some(self.mintbase(event_data, receiver_id)), + Self::Paras => Some(self.paras(event_data, receiver_id)), + Self::Unknown => None, + } + } + + fn paras( + &self, + event_data: serde_json::Value, + receiver_id: &near_lake_primitives::near_primitives::types::AccountId, + ) -> Vec { + let paras_event_data = serde_json::from_value::>(event_data) + .expect("Failed to parse NftMintLog"); + + paras_event_data + .iter() + .map(|nft_mint_log| NFT { + owner: nft_mint_log.owner_id.clone(), + links: nft_mint_log + .token_ids + .iter() + .map(|token_id| { + format!( + "https://paras.id/token/{}::{}/{}", + receiver_id.to_string(), + token_id.split(":").collect::>()[0], + token_id, + ) + }) + .collect(), + }) + .collect() + } + + fn mintbase( + &self, + event_data: serde_json::Value, + receiver_id: &near_lake_primitives::near_primitives::types::AccountId, + ) -> Vec { + let mintbase_event_data = serde_json::from_value::>(event_data) + .expect("Failed to parse NftMintLog"); + + mintbase_event_data + .iter() + .map(|nft_mint_log| NFT { + owner: nft_mint_log.owner_id.clone(), + links: vec![format!( + "https://mintbase.io/contract/{}/token/{}", + receiver_id.to_string(), + nft_mint_log.token_ids[0] + )], + }) + .collect() + } +} + +// We are allowing the dead_code lint because not all fields of the structures are used +// However, they are printed to the terminal for debugging purposes. +#[allow(dead_code)] +#[derive(Debug)] +struct NFTReceipt { + receipt_id: String, + marketplace_name: String, + nfts: Vec, +} + +// We are allowing the dead_code lint because not all fields of the structures are used +// However, they are printed to the terminal for debugging purposes. +#[allow(dead_code)] +#[derive(Debug)] +struct NFT { + owner: String, + links: Vec, +} + +#[derive(Debug, serde::Deserialize)] +struct NftMintLog { + owner_id: String, + token_ids: Vec, + // There is also a `memo` field, but it is not used in this example + // memo: Option, +} diff --git a/lake-framework/examples/simple.rs b/lake-framework/examples/simple.rs new file mode 100644 index 0000000..c9d59bd --- /dev/null +++ b/lake-framework/examples/simple.rs @@ -0,0 +1,21 @@ +//! A simple example of how to use the Lake Framework +//! This indexer will listen to the NEAR blockchain and print the block height of each block + +use near_lake_framework::near_lake_primitives; + +fn main() -> anyhow::Result<()> { + eprintln!("Starting..."); + // Lake Framework start boilerplate + near_lake_framework::LakeBuilder::default() + .testnet() + .start_block_height(112205773) + .build()? + .run(handle_block)?; // developer-defined async function that handles each block + Ok(()) +} + +async fn handle_block(block: near_lake_primitives::block::Block) -> anyhow::Result<()> { + println!("Block {:?}", block.block_height()); + + Ok(()) +} diff --git a/lake-framework/examples/with_context.rs b/lake-framework/examples/with_context.rs new file mode 100644 index 0000000..9de6150 --- /dev/null +++ b/lake-framework/examples/with_context.rs @@ -0,0 +1,79 @@ +//! This example show how to use a context with Lake Framework. +//! It is going to follow the NEAR Social contract and the block height along +//! with a number of calls to the contract. +use near_lake_framework::{near_lake_primitives, LakeContext}; +use std::io::Write; +// We need to import this trait to use the `as_function_call` method. +use near_lake_primitives::actions::ActionMetaDataExt; + +const CONTRACT_ID: &str = "social.near"; + +// This is the context we're going to use. +// Lake::run_with_context requires the context to implement the LakeContext trait. +// That trait requires to implement two methods `execute_before_run` and `execute_after_run`. +// However, we don't actually need them in our cause of using the context. +// That's why we're using the derive macro to implement the trait for us. +// The macro will generate the default implementation of the methods. Those methods are empty. +// By doing so, we don't need to implement the trait manually and can use the context as is. +#[derive(Clone, LakeContext)] +struct FileContext { + path: std::path::PathBuf, +} + +impl FileContext { + fn new(path: impl Into) -> Self { + Self { path: path.into() } + } + + // append to the file + pub fn write(&self, value: &str) -> anyhow::Result<()> { + let mut file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&self.path)?; + file.write_all(value.as_bytes())?; + Ok(()) + } +} + +fn main() -> anyhow::Result<()> { + println!("Starting..."); + // Create the context + let context = FileContext::new("./output.txt"); + // Lake Framework start boilerplate + near_lake_framework::LakeBuilder::default() + .mainnet() + .start_block_height(88444526) + .build()? + // developer-defined async function that handles each block + .run_with_context(print_function_calls_to_my_account, &context)?; + Ok(()) +} + +async fn print_function_calls_to_my_account( + mut block: near_lake_primitives::block::Block, + ctx: &FileContext, +) -> anyhow::Result<()> { + let block_height = block.block_height(); + let actions: Vec<&near_lake_primitives::actions::FunctionCall> = block + .actions() + .filter(|action| action.receiver_id().as_str() == CONTRACT_ID) + .filter_map(|action| action.as_function_call()) + .collect(); + + if !actions.is_empty() { + // Here's the usage of the context. + ctx.write( + format!( + "Block #{} - {} calls to {}\n", + block_height, + actions.len(), + CONTRACT_ID + ) + .as_str(), + )?; + println!("Block #{:?}\n{:#?}", block_height, actions); + } + + Ok(()) +} diff --git a/lake-framework/examples/with_context_parent_tx_cache.rs b/lake-framework/examples/with_context_parent_tx_cache.rs new file mode 100644 index 0000000..f6a23fa --- /dev/null +++ b/lake-framework/examples/with_context_parent_tx_cache.rs @@ -0,0 +1,62 @@ +//! This example show how to use a context ParentTransactionCache with the Lake Framework. +//! It is going to follow the NEAR Social contract and cache the parent Transaction for the Receipts. +//! Thus we would be able to capture the Transaction where the change to the contract state has started. +//! **WARNING**: ParentTransactionCache captures all the transactions in the block. +//! That's why we filter it by only one account we're watching here. +use near_lake_framework::near_lake_primitives; +use near_lake_primitives::CryptoHash; +// We need to import this trait to use the `as_function_call` method. +use lake_parent_transaction_cache::{ParentTransactionCache, ParentTransactionCacheBuilder}; +use near_lake_primitives::actions::ActionMetaDataExt; + +const CONTRACT_ID: &str = "social.near"; + +fn main() -> anyhow::Result<()> { + println!("Starting..."); + // Building the ParentTransactionCache context. + // The way of instantiation of the context depends on the implementation developers choose. + // ParentTransactionCache follows the Builder pattern. + // This will create the context with the default size of the cache (100_000) + // and a filter for the account we're watching. + // It will omit caching all the transactions that are not related to the account. + let parent_transaction_cache_ctx = ParentTransactionCacheBuilder::default() + .for_account(String::from(CONTRACT_ID).try_into()?) + .build()?; + // Lake Framework start boilerplate + near_lake_framework::LakeBuilder::default() + .mainnet() + .start_block_height(88444526) + .build()? + // developer-defined async function that handles each block + .run_with_context(print_function_call_tx_hash, &parent_transaction_cache_ctx)?; + Ok(()) +} + +async fn print_function_call_tx_hash( + mut block: near_lake_primitives::block::Block, + ctx: &ParentTransactionCache, +) -> anyhow::Result<()> { + // Cache has been updated before this function is called. + let block_height = block.block_height(); + let actions: Vec<( + &near_lake_primitives::actions::FunctionCall, + Option, + )> = block + .actions() + .filter(|action| action.receiver_id().as_str() == CONTRACT_ID) + .filter_map(|action| action.as_function_call()) + .map(|action| { + ( + action, + ctx.get_parent_transaction_hash(&action.receipt_id()), + ) + }) + .collect(); + + if !actions.is_empty() { + // Here's the usage of the context. + println!("Block #{:?}\n{:#?}", block_height, actions); + } + + Ok(()) +} diff --git a/lake-framework/src/lib.rs b/lake-framework/src/lib.rs new file mode 100644 index 0000000..185da91 --- /dev/null +++ b/lake-framework/src/lib.rs @@ -0,0 +1,125 @@ +#![doc = include_str!("../README.md")] +#[macro_use] +extern crate derive_builder; + +use futures::{Future, StreamExt}; + +pub use lake_context_derive::LakeContext; +pub use near_lake_primitives::{self, near_indexer_primitives}; + +pub use aws_credential_types::Credentials; +pub use types::{Lake, LakeBuilder, LakeContextExt, LakeError}; + +mod s3_fetchers; +mod streamer; +pub(crate) mod types; + +pub(crate) const LAKE_FRAMEWORK: &str = "near_lake_framework"; + +impl types::Lake { + /// Creates `mpsc::channel` and returns the `receiver` to read the stream of `StreamerMessage` + ///```no_run + /// # use near_lake_framework::{LakeContext}; + /// + /// #[derive(LakeContext)] + /// struct MyContext { + /// my_field: String, + /// } + /// + ///# fn main() -> anyhow::Result<()> { + /// + /// let context = MyContext { + /// my_field: "my_value".to_string(), + /// }; + /// + /// near_lake_framework::LakeBuilder::default() + /// .testnet() + /// .start_block_height(112205773) + /// .build()? + /// .run_with_context(handle_block, &context)?; + /// Ok(()) + ///# } + /// + /// # async fn handle_block(_block: near_lake_primitives::block::Block, context: &MyContext) -> anyhow::Result<()> { Ok(()) } + ///``` + pub fn run_with_context<'context, C: LakeContextExt, E, Fut>( + self, + f: impl Fn(near_lake_primitives::block::Block, &'context C) -> Fut, + context: &'context C, + ) -> Result<(), LakeError> + where + Fut: Future>, + E: Into>, + { + let runtime = tokio::runtime::Runtime::new() + .map_err(|err| LakeError::RuntimeStartError { error: err })?; + + runtime.block_on(async move { + // capture the concurrency value before it moves into the streamer + let concurrency = self.concurrency; + + // instantiate the NEAR Lake Framework Stream + let (sender, stream) = streamer::streamer(self); + + // read the stream events and pass them to a handler function with + // concurrency 1 + let mut handlers = tokio_stream::wrappers::ReceiverStream::new(stream) + .map(|streamer_message| async { + let mut block: near_lake_primitives::block::Block = streamer_message.into(); + + context.execute_before_run(&mut block); + + let user_indexer_function_execution_result = f(block, context).await; + + context.execute_after_run(); + + user_indexer_function_execution_result + }) + .buffer_unordered(concurrency); + + while let Some(_handle_message) = handlers.next().await {} + drop(handlers); // close the channel so the sender will stop + + // propagate errors from the sender + match sender.await { + Ok(Ok(())) => Ok(()), + Ok(Err(err)) => Err(err), + Err(err) => Err(err.into()), // JoinError + } + }) + } + + /// Creates `mpsc::channel` and returns the `receiver` to read the stream of `StreamerMessage` + ///```no_run + ///# fn main() -> anyhow::Result<()> { + /// near_lake_framework::LakeBuilder::default() + /// .testnet() + /// .start_block_height(112205773) + /// .build()? + /// .run(handle_block)?; + /// Ok(()) + ///# } + /// + /// # async fn handle_block(_block: near_lake_primitives::block::Block) -> anyhow::Result<()> { Ok(()) } + ///``` + pub fn run( + self, + f: impl Fn(near_lake_primitives::block::Block) -> Fut, + ) -> Result<(), LakeError> + where + Fut: Future>, + E: Into>, + { + struct EmptyContext {} + + impl LakeContextExt for EmptyContext { + fn execute_before_run(&self, _block: &mut near_lake_primitives::block::Block) {} + + fn execute_after_run(&self) {} + } + + let context = EmptyContext {}; + + self.run_with_context(|block, _context| f(block), &context) + } +} diff --git a/src/s3_fetchers.rs b/lake-framework/src/s3_fetchers.rs similarity index 93% rename from src/s3_fetchers.rs rename to lake-framework/src/s3_fetchers.rs index 8e428b5..14bdfc6 100644 --- a/src/s3_fetchers.rs +++ b/lake-framework/src/s3_fetchers.rs @@ -77,10 +77,7 @@ pub(crate) async fn list_block_heights( lake_s3_client: &impl S3Client, s3_bucket_name: &str, start_from_block_height: crate::types::BlockHeight, -) -> Result< - Vec, - crate::types::LakeError, -> { +) -> Result, crate::types::LakeError> { tracing::debug!( target: crate::LAKE_FRAMEWORK, "Fetching block heights from S3, after #{}...", @@ -117,10 +114,7 @@ pub(crate) async fn fetch_streamer_message( lake_s3_client: &impl S3Client, s3_bucket_name: &str, block_height: crate::types::BlockHeight, -) -> Result< - near_indexer_primitives::StreamerMessage, - crate::types::LakeError, -> { +) -> Result { let block_view = { let body_bytes = loop { match lake_s3_client @@ -151,7 +145,9 @@ pub(crate) async fn fetch_streamer_message( }; }; - serde_json::from_slice::(body_bytes.as_ref())? + serde_json::from_slice::( + body_bytes.as_ref(), + )? }; let fetch_shards_futures = (0..block_view.chunks.len() as u64) @@ -163,7 +159,7 @@ pub(crate) async fn fetch_streamer_message( let shards = futures::future::try_join_all(fetch_shards_futures).await?; - Ok(near_indexer_primitives::StreamerMessage { + Ok(near_lake_primitives::StreamerMessage { block: block_view, shards, }) @@ -175,10 +171,7 @@ async fn fetch_shard_or_retry( s3_bucket_name: &str, block_height: crate::types::BlockHeight, shard_id: u64, -) -> Result< - near_indexer_primitives::IndexerShard, - crate::types::LakeError, -> { +) -> Result { let body_bytes = loop { match lake_s3_client .get_object( @@ -216,9 +209,7 @@ async fn fetch_shard_or_retry( } }; - Ok(serde_json::from_slice::< - near_indexer_primitives::IndexerShard, - >(body_bytes.as_ref())?) + Ok(serde_json::from_slice::(body_bytes.as_ref())?) } #[cfg(test)] diff --git a/lake-framework/src/streamer.rs b/lake-framework/src/streamer.rs new file mode 100644 index 0000000..b9de22a --- /dev/null +++ b/lake-framework/src/streamer.rs @@ -0,0 +1,291 @@ +use aws_sdk_s3::Client; + +use futures::stream::StreamExt; +use tokio::sync::mpsc; +use tokio::sync::mpsc::error::SendError; + +use near_lake_primitives::near_indexer_primitives; + +use crate::{s3_fetchers, types}; + +/// Creates [mpsc::Receiver] and +/// [mpsc::Sender] spawns the streamer +/// process that writes [near_idnexer_primitives::StreamerMessage] to the given `mpsc::channel` +/// returns both `sender` and `receiver` +pub(crate) fn streamer( + config: crate::Lake, +) -> ( + tokio::task::JoinHandle>, + mpsc::Receiver, +) { + let (sender, receiver) = mpsc::channel(config.blocks_preload_pool_size); + (tokio::spawn(start(sender, config)), receiver) +} + +fn stream_block_heights<'a: 'b, 'b>( + lake_s3_client: &'a s3_fetchers::LakeS3Client, + s3_bucket_name: &'a str, + mut start_from_block_height: crate::types::BlockHeight, +) -> impl futures::Stream + 'b { + async_stream::stream! { + loop { + tracing::debug!(target: crate::LAKE_FRAMEWORK, "Fetching a list of blocks from S3..."); + match s3_fetchers::list_block_heights( + lake_s3_client, + s3_bucket_name, + start_from_block_height, + ) + .await { + Ok(block_heights) => { + if block_heights.is_empty() { + tracing::debug!( + target: crate::LAKE_FRAMEWORK, + "There are no newer block heights than {} in bucket {}. Fetching again in 2s...", + start_from_block_height, + s3_bucket_name, + ); + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + continue; + } + tracing::debug!( + target: crate::LAKE_FRAMEWORK, + "Received {} newer block heights", + block_heights.len() + ); + + start_from_block_height = *block_heights.last().unwrap() + 1; + for block_height in block_heights { + tracing::debug!(target: crate::LAKE_FRAMEWORK, "Yielding {} block height...", block_height); + yield block_height; + } + } + Err(err) => { + tracing::warn!( + target: crate::LAKE_FRAMEWORK, + "Failed to get block heights from bucket {}: {}. Retrying in 1s...", + s3_bucket_name, + err, + ); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } + } + } + } +} + +// The only consumer of the BlockHeights Streamer +async fn prefetch_block_heights_into_pool( + pending_block_heights: &mut std::pin::Pin< + &mut impl tokio_stream::Stream, + >, + limit: usize, + await_for_at_least_one: bool, +) -> Result, crate::types::LakeError> { + let mut block_heights = Vec::with_capacity(limit); + for remaining_limit in (0..limit).rev() { + tracing::debug!(target: crate::LAKE_FRAMEWORK, "Polling for the next block height without awaiting... (up to {} block heights are going to be fetched)", remaining_limit); + match futures::poll!(pending_block_heights.next()) { + std::task::Poll::Ready(Some(block_height)) => { + block_heights.push(block_height); + } + std::task::Poll::Pending => { + if await_for_at_least_one && block_heights.is_empty() { + tracing::debug!(target: crate::LAKE_FRAMEWORK, "There were no block heights available immediatelly, and the prefetching blocks queue is empty, so we need to await for at least a single block height to be available before proceeding..."); + match pending_block_heights.next().await { + Some(block_height) => { + block_heights.push(block_height); + } + None => { + return Err(crate::types::LakeError::InternalError { + error_message: "This state should be unreachable as the block heights stream should be infinite.".to_string() + }); + } + } + continue; + } + tracing::debug!(target: crate::LAKE_FRAMEWORK, "There were no block heights available immediatelly, so we should not block here and keep processing the blocks."); + break; + } + std::task::Poll::Ready(None) => { + return Err( + crate::types::LakeError::InternalError { + error_message: "This state should be unreachable as the block heights stream should be infinite.".to_string() + } + ); + } + } + } + Ok(block_heights) +} + +#[allow(unused_labels)] // we use loop labels for code-readability +pub(crate) async fn start( + streamer_message_sink: mpsc::Sender, + config: crate::Lake, +) -> Result<(), crate::types::LakeError> { + let mut start_from_block_height = config.start_block_height; + + let s3_client = if let Some(config) = config.s3_config { + Client::from_conf(config) + } else { + let aws_config = aws_config::from_env().load().await; + let s3_config = aws_sdk_s3::config::Builder::from(&aws_config) + .region(aws_types::region::Region::new(config.s3_region_name)) + .build(); + Client::from_conf(s3_config) + }; + let lake_s3_client = s3_fetchers::LakeS3Client::new(s3_client.clone()); + + let mut last_processed_block_hash: Option = None; + + 'main: loop { + // In the beginning of the 'main' loop we create a Block Heights stream + // and prefetch the initial data in that pool. + // Later the 'stream' loop might exit to this 'main' one to repeat the procedure. + // This happens because we assume Lake Indexer that writes to the S3 Bucket might + // in some cases, write N+1 block before it finishes writing the N block. + // We require to stream blocks consistently, so we need to try to load the block again. + + let pending_block_heights = stream_block_heights( + &lake_s3_client, + &config.s3_bucket_name, + start_from_block_height, + ); + tokio::pin!(pending_block_heights); + + let mut streamer_messages_futures = futures::stream::FuturesOrdered::new(); + tracing::debug!( + target: crate::LAKE_FRAMEWORK, + "Prefetching up to {} blocks...", + config.blocks_preload_pool_size + ); + + streamer_messages_futures.extend( + prefetch_block_heights_into_pool( + &mut pending_block_heights, + config.blocks_preload_pool_size, + true, + ) + .await? + .into_iter() + .map(|block_height| { + s3_fetchers::fetch_streamer_message( + &lake_s3_client, + &config.s3_bucket_name, + block_height, + ) + }), + ); + + tracing::debug!( + target: crate::LAKE_FRAMEWORK, + "Awaiting for the first prefetched block..." + ); + 'stream: while let Some(streamer_message_result) = streamer_messages_futures.next().await { + let streamer_message = streamer_message_result.map_err(|err| { + tracing::error!( + target: crate::LAKE_FRAMEWORK, + "Failed to fetch StreamerMessage with error: \n{:#?}", + err, + ); + err + })?; + + tracing::debug!( + target: crate::LAKE_FRAMEWORK, + "Received block #{} ({})", + streamer_message.block.header.height, + streamer_message.block.header.hash + ); + // check if we have `last_processed_block_hash` (might be None only on start) + if let Some(prev_block_hash) = last_processed_block_hash { + // compare last_processed_block_hash` with `block.header.prev_hash` of the current + // block (ensure we don't miss anything from S3) + // retrieve the data from S3 if prev_hashes don't match and repeat the main loop step + if prev_block_hash != streamer_message.block.header.prev_hash { + tracing::warn!( + target: crate::LAKE_FRAMEWORK, + "`prev_hash` does not match, refetching the data from S3 in 200ms", + ); + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + break 'stream; + } + } + + // store current block info as `last_processed_block_*` for next iteration + last_processed_block_hash = Some(streamer_message.block.header.hash); + start_from_block_height = streamer_message.block.header.height + 1; + + tracing::debug!( + target: crate::LAKE_FRAMEWORK, + "Prefetching up to {} blocks... (there are {} blocks in the prefetching pool)", + config.blocks_preload_pool_size, + streamer_messages_futures.len(), + ); + tracing::debug!( + target: crate::LAKE_FRAMEWORK, + "Streaming block #{} ({})", + streamer_message.block.header.height, + streamer_message.block.header.hash + ); + let blocks_preload_pool_current_len = streamer_messages_futures.len(); + + let prefetched_block_heights_future = prefetch_block_heights_into_pool( + &mut pending_block_heights, + config + .blocks_preload_pool_size + .saturating_sub(blocks_preload_pool_current_len), + blocks_preload_pool_current_len == 0, + ); + + let streamer_message_sink_send_future = streamer_message_sink.send(streamer_message); + + let (prefetch_res, send_res): ( + Result, crate::types::LakeError>, + Result<_, SendError>, + ) = futures::join!( + prefetched_block_heights_future, + streamer_message_sink_send_future, + ); + + if let Err(SendError(err)) = send_res { + tracing::debug!( + target: crate::LAKE_FRAMEWORK, + "Failed to send StreamerMessage (#{:0>12}) to the channel. Channel is closed, exiting \n{:?}", + start_from_block_height - 1, + err, + ); + return Ok(()); + } + + streamer_messages_futures.extend( + prefetch_res + .map_err(|err| { + tracing::error!( + target: crate::LAKE_FRAMEWORK, + "Failed to prefetch block heights to the prefetching pool with error: \n{:#?}", + err + ); + err + })? + .into_iter() + .map(|block_height| { + s3_fetchers::fetch_streamer_message( + &lake_s3_client, + &config.s3_bucket_name, + block_height, + ) + } + )); + } + + tracing::warn!( + target: crate::LAKE_FRAMEWORK, + "Exited from the 'stream' loop. It may happen in two cases:\n + 1. Blocks has ended (impossible, might be an error on the Lake Buckets),\n + 2. Received a Block which prev_hash doesn't match the previously streamed block.\n + Will attempt to restart the stream from block #{}", + start_from_block_height, + ); + } +} diff --git a/lake-framework/src/types.rs b/lake-framework/src/types.rs new file mode 100644 index 0000000..2ec2046 --- /dev/null +++ b/lake-framework/src/types.rs @@ -0,0 +1,324 @@ +/// Type alias represents the block height +pub type BlockHeight = u64; + +/// Configuration struct for NEAR Lake Framework +/// NB! Consider using [`LakeBuilder`] +/// Building the `Lake` example: +/// ``` +/// use near_lake_framework::LakeBuilder; +/// +/// # fn main() { +/// let lake = LakeBuilder::default() +/// .testnet() +/// .start_block_height(82422587) +/// .build() +/// .expect("Failed to build Lake"); +/// # } +/// ``` +#[derive(Default, Builder, Debug)] +#[builder(pattern = "owned")] +pub struct Lake { + /// AWS S3 Bucket name + #[builder(setter(into))] + pub(crate) s3_bucket_name: String, + /// AWS S3 Region name + #[builder(setter(into))] + pub(crate) s3_region_name: String, + /// Defines the block height to start indexing from + pub(crate) start_block_height: u64, + /// Custom aws_sdk_s3::config::Config + /// ## Use-case: custom endpoint + /// You might want to stream data from the custom S3-compatible source () . In order to do that you'd need to pass `aws_sdk_s3::config::Config` configured + /// ``` + /// use near_lake_framework::LakeBuilder; + /// + /// # #[tokio::main] + /// # async fn main() { + /// let aws_config = aws_config::from_env().load().await; + /// let mut s3_conf = aws_sdk_s3::config::Builder::from(&aws_config) + /// .endpoint_url("http://0.0.0.0:9000") + /// .build(); + /// + /// let lake = LakeBuilder::default() + /// .s3_config(s3_conf) + /// .s3_bucket_name("near-lake-data-custom") + /// .s3_region_name("eu-central-1") + /// .start_block_height(1) + /// .build() + /// .expect("Failed to build Lake"); + /// # } + /// ``` + #[builder(setter(strip_option), default)] + pub(crate) s3_config: Option, + /// Defines how many *block heights* Lake Framework will try to preload into memory to avoid S3 `List` requests. + /// Default: 100 + /// + /// *Note*: This value is not the number of blocks to preload, but the number of block heights. + /// Also, this value doesn't affect your indexer much if it follows the tip of the network. + /// This parameter is useful for historical indexing. + #[builder(default = "100")] + pub(crate) blocks_preload_pool_size: usize, + /// Number of concurrent blocks to process. Default: 1 + /// **WARNING**: Increase this value only if your block handling logic doesn't have to rely on previous blocks and can be processed in parallel + #[builder(default = "1")] + pub(crate) concurrency: usize, +} + +impl LakeBuilder { + /// Shortcut to set up [LakeBuilder::s3_bucket_name] for mainnet + /// ``` + /// use near_lake_framework::LakeBuilder; + /// + /// # fn main() { + /// let lake = LakeBuilder::default() + /// .mainnet() + /// .start_block_height(65231161) + /// .build() + /// .expect("Failed to build Lake"); + /// # } + /// ``` + pub fn mainnet(mut self) -> Self { + self.s3_bucket_name = Some("near-lake-data-mainnet".to_string()); + self.s3_region_name = Some("eu-central-1".to_string()); + self + } + + /// Shortcut to set up [LakeBuilder::s3_bucket_name] for testnet + /// ``` + /// use near_lake_framework::LakeBuilder; + /// + /// # fn main() { + /// let lake = LakeBuilder::default() + /// .testnet() + /// .start_block_height(82422587) + /// .build() + /// .expect("Failed to build Lake"); + /// # } + /// ``` + pub fn testnet(mut self) -> Self { + self.s3_bucket_name = Some("near-lake-data-testnet".to_string()); + self.s3_region_name = Some("eu-central-1".to_string()); + self + } + + /// Shortcut to set up [LakeBuilder::s3_bucket_name] for betanet + /// ``` + /// use near_lake_framework::LakeBuilder; + /// + /// # fn main() { + /// let lake = LakeBuilder::default() + /// .betanet() + /// .start_block_height(82422587) + /// .build() + /// .expect("Failed to build Lake"); + /// # } + /// ``` + pub fn betanet(mut self) -> Self { + self.s3_bucket_name = Some("near-lake-data-betanet".to_string()); + self.s3_region_name = Some("us-east-1".to_string()); + self + } +} + +#[allow(clippy::enum_variant_names)] +#[derive(thiserror::Error, Debug)] +pub enum LakeError { + #[error("Failed to parse structure from JSON: {error_message}")] + ParseError { + #[from] + error_message: serde_json::Error, + }, + #[error("AWS S3 error")] + AwsGetObjectError { + #[from] + error: aws_sdk_s3::types::SdkError, + }, + #[error("AWS S3 error")] + AwsLisObjectsV2Error { + #[from] + error: aws_sdk_s3::types::SdkError, + }, + #[error("Failed to convert integer")] + IntConversionError { + #[from] + error: std::num::TryFromIntError, + }, + #[error("Join error")] + JoinError { + #[from] + error: tokio::task::JoinError, + }, + #[error("Failed to start runtime")] + RuntimeStartError { + #[from] + error: std::io::Error, + }, + #[error("Internal error: {error_message}")] + InternalError { error_message: String }, +} + +/// ### The concept of Context for the Lake Framework +/// The main idea of the Lake Framework is to provide a simple way to index data from the NEAR blockchain. +/// The framework is designed to be as flexible as possible, so it doesn't provide any specific logic for indexing. +/// Instead, it provides a way to implement your own logic. One of the main concepts of the framework is the Context. +/// The Context is a struct that implements the [LakeContext] trait. It is used to pass data between the framework and your logic. +/// The Context is created once and then passed to the framework. The framework will call the [LakeContext::execute_before_run] +/// method before the indexing process starts and [LakeContext::execute_after_run] after the indexing process is finished. +/// The Context is useful for passing data between blocks. For example, you can use it to store the last block timestamp and use it in the next block. +/// +/// Also the Context is necessary to pass the "global" data to the indexing process. For example, you can use it to pass the database connection pool. +/// +/// ### Examples +/// +/// #### Simple Context examples (explicit) +/// **WARNING**: This example demonsrates how Context works explicitly. In the real-world application you would do less boilerplate. See further examples. +/// In this example we will create a simple Context that prints the block height before the processing the block. +/// ```no_run +/// use near_lake_framework::LakeContextExt; // note Lake Framework exports this trait with a suffix Ext in the name +/// struct PrinterContext; +/// +/// impl LakeContextExt for PrinterContext { +/// fn execute_before_run(&self, block: &mut near_lake_primitives::block::Block) { +/// println!("Processing block {}", block.header().height()); +/// } +/// fn execute_after_run(&self) {} +/// } +/// ``` +/// As you can see we will be printing `Processing block {block_height}` before processing the block. And we will do nothing after +/// the indexing process is finished. +/// +/// The next example is showing how to provide some value to the indexing process. +/// ```no_run +/// use near_lake_framework::LakeContextExt; // note Lake Framework exports this trait with a suffix Ext in the name +/// use near_lake_framework::LakeBuilder; +/// # use diesel::Connection; +/// +/// struct ApplicationDataContext { +/// pub db_pool: diesel::pg::PgConnection, +/// } +/// +/// // We need our context to do nothing before and after the indexing process. +/// // The only purpose is to provide the database connection pool to the indexing process. +/// impl LakeContextExt for ApplicationDataContext { +/// fn execute_before_run(&self, block: &mut near_lake_primitives::block::Block) {} +/// fn execute_after_run(&self) {} +/// } +/// +/// fn main() { +/// let db_pool = diesel::PgConnection::establish("postgres://localhost:5432") +/// .expect("Failed to connect to database"); +/// let context = ApplicationDataContext { db_pool }; +/// +/// let result = LakeBuilder::default() +/// .testnet() +/// .start_block_height(82422587) +/// .build() +/// .unwrap() +/// .run_with_context(indexing_function, &context); +/// } +/// +/// async fn indexing_function( +/// block: near_lake_primitives::block::Block, +/// context: &ApplicationDataContext, +/// ) -> Result<(), near_lake_framework::LakeError> { +/// // Now we can use the database connection pool +/// let db_pool = &context.db_pool; +/// ///... +/// Ok(()) +/// } +/// ``` +/// +/// #### Simple Context example (real-world) +/// The last example from the previous section is a bit verbose. In the real-world application you would do less boilerplate. +/// The main purpose of that example was to show you what's happening under the hood. However, for your convenience, the Lake Framework +/// provides a trait [LakeContextExt] that implements the [LakeContext] trait for you. So you can use it to create a simple Context. +/// +/// ```ignore +/// use near_lake_framework::LakeContext; // This is a derive macro +/// use near_lake_framework::LakeBuilder; +/// +/// #[derive(LakeContext)] +/// /// struct ApplicationDataContext { +/// pub db_pool: diesel::pg::PgConnection, +/// } +/// +/// // Here we got rid of the boilerplate code that we had in the previous example to impl the LakeContext trait. +/// +/// fn main() { +/// let db_pool = diesel::pg::PgConnection::establish("postgres://postgres:password@localhost:5432/database") +/// .unwrap_or_else(|_| panic!("Error connecting to database")) +/// +/// let context = ApplicationDataContext { db_pool }; +/// +/// let result = LakeBuilder::default() +/// .testnet() +/// .start_block_height(82422587) +/// .build() +/// .unwrap() +/// .run_with_context(indexing_function, &context); +/// } +/// +/// async fn indexing_function( +/// block: near_lake_primitives::block::Block, +/// context: &ApplicationDataContext, +/// ) -> Result<(), near_lake_framework::LakeError> { +/// // Now we can use the database connection pool +/// let db_pool = &context.db_pool; +/// // ... +/// Ok(()) +/// } +/// ``` +/// +/// It might look like not a big deal to get rid of the boilerplate code. However, it is very useful when you have a lot of Contexts or when you +/// use a ready-to-use Context from the community. +/// +/// #### Advanced Context example +/// In this example we will extend a previous one with the `ParentTransactionCache` context Lake Framework team has created and shared with everybody. +/// +/// ```no_run +/// use near_lake_framework::LakeContext; // This is a derive macro +/// use lake_parent_transaction_cache::{ParentTransactionCache, ParentTransactionCacheBuilder}; // This is a ready-to-use Context from the community that impls LakeContext trait +/// use near_lake_framework::LakeBuilder; +/// # use diesel::Connection; +/// +/// #[derive(LakeContext)] +/// struct ApplicationDataContext { +/// pub db_pool: diesel::pg::PgConnection, +/// pub parent_transaction_cache: ParentTransactionCache, +/// } +/// +/// fn main() { +/// let db_pool = diesel::PgConnection::establish("postgres://postgres:password@localhost:5432/database") +/// .unwrap_or_else(|_| panic!("Error connecting to database")); +/// let parent_transaction_cache = ParentTransactionCacheBuilder::default().build().unwrap(); +/// +/// let context = ApplicationDataContext { db_pool, parent_transaction_cache }; +/// +/// let result = LakeBuilder::default() +/// .testnet() +/// .start_block_height(82422587) +/// .build() +/// .unwrap() +/// .run_with_context(indexing_function, &context); +/// } +/// +/// async fn indexing_function( +/// block: near_lake_primitives::block::Block, +/// context: &ApplicationDataContext, +/// ) -> Result<(), near_lake_framework::LakeError> { +/// // Now we can use the database connection pool +/// let db_pool = &context.db_pool; +/// dbg!(&context.parent_transaction_cache); +/// Ok(()) +/// } +/// ``` +/// As you can see we have extended our context with the `ParentTransactionCache` context. And we can use it in our indexing function. +/// The `ParentTransactionCache` defines the `execute_before_run` and `execute_after_run` methods. So when we call `run_with_context` method +/// the Lake Framework will call `execute_before_run` and `execute_after_run` methods for us. +/// And we didn't need to implement them in our `ApplicationDataContext` struct because `LakeContext` derive macro did it for us automatically. +pub trait LakeContextExt { + /// This method will be called before the indexing process is started. + fn execute_before_run(&self, block: &mut near_lake_primitives::block::Block); + /// This method will be called after the indexing process is finished. + fn execute_after_run(&self); +} diff --git a/lake-parent-transaction-cache/Cargo.toml b/lake-parent-transaction-cache/Cargo.toml new file mode 100644 index 0000000..e7d2170 --- /dev/null +++ b/lake-parent-transaction-cache/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "lake-parent-transaction-cache" +version.workspace = true +edition = "2021" + +[dependencies] +cached = "0.43.0" +derive_builder = "0.12.0" + +near-lake-framework = { path = "../lake-framework" } + +[dev-dependencies] +anyhow = "1.0.44" \ No newline at end of file diff --git a/lake-parent-transaction-cache/README.md b/lake-parent-transaction-cache/README.md new file mode 100644 index 0000000..c5886c4 --- /dev/null +++ b/lake-parent-transaction-cache/README.md @@ -0,0 +1,120 @@ +# Lake Parent Transaction Cache (Context) + +Lake Parent Transaction Cache is a ready-to-use context for the Lake Framework in Rust. It provides a cache for keeping the relation between transactions and receipts in cache. + +## Example Usage + +```no_run +use lake_parent_transaction_cache::{ParentTransactionCache, ParentTransactionCacheBuilder}; +# use near_lake_framework::LakeBuilder; +# use near_lake_framework::near_lake_primitives::{block::Block, actions::ActionMetaDataExt}; + +# fn main() { +let parent_transaction_cache_ctx = ParentTransactionCacheBuilder::default() + .build() + .expect("Failed to build the ParentTransactionCache context"); + +LakeBuilder::default() + .mainnet() + .start_block_height(80504433) + .build() + .expect("Failed to build the Lake Framework") + .run_with_context(handle_block, &parent_transaction_cache_ctx) + .expect("Failed to run the Lake Framework"); +# } + +async fn handle_block( + mut block: Block, + ctx: &ParentTransactionCache, +) -> anyhow::Result<()> { + for action in block.actions() { + println!( + "Action receipt ID: {:?} | Parent TX hash: {:?}", + action.receipt_id(), + ctx.get_parent_transaction_hash(&action.receipt_id()) + ); + } + Ok(()) +} +``` + +## Getting Started + +To use the Lake Parent Transaction Cache context in your Rust project, follow these steps: + +1. Add the following dependencies to your `Cargo.toml` file: + +```toml +[dependencies] +lake_parent_transaction_cache = "" +``` + +2. Import the necessary modules in your code: + +```ignore +use lake_parent_transaction_cache::ParentTransactionCache; +use near_lake_primitives::actions::ActionMetaDataExt; +``` + +3. Create an instance of the `ParentTransactionCache` context: + +```no_run +# use lake_parent_transaction_cache::ParentTransactionCacheBuilder; +let parent_transaction_cache_ctx = ParentTransactionCacheBuilder::default(); +``` + +4. Configure the Lake Framework and run it with the created context: + +```ignore +near_lake_framework::LakeBuilder::default() + .mainnet() + .start_block_height() + .build()? + .run_with_context(, &parent_transaction_cache_ctx)?; +``` + +Replace `` with the starting block height you want to use. Replace `` with the function you want to use to index the blocks. + +## Advanced Usage + +### Cache size + +We use [SizedCache](https://docs.rs/cached/0.43.0/cached/stores/struct.SizedCache.html) under the hood. So we can configure the cache size by using the `cache_size` method: + +```no_run +# use lake_parent_transaction_cache::ParentTransactionCacheBuilder; +let parent_transaction_cache_ctx = ParentTransactionCacheBuilder::default() + .cache_size(100_000); +``` + +By default the cache size is 100,000. + +### Watch for specific accounts + +By default `ParentTransactionCache` context will cache the relation between Transaction and Receipt for every Transaction in the block. But you can configure it to watch for specific accounts only: + +#### You can pass a Vec of AccountId + +```no_run +# use lake_parent_transaction_cache::ParentTransactionCacheBuilder; +use near_lake_framework::near_primitives::types::AccountId; + +let accounts_to_watch: Vec = vec![ + String::from("alice.near).try_into().unwrap(), + String::from("bob.near).try_into().unwrap(), +]; +let parent_transaction_cache_ctx = ParentTransactionCacheBuilder::default() + .for_accounts(accounts_to_watch); +``` + +#### You can pass accounts to watch one by one using `for_account` method + +```no_run +# use lake_parent_transaction_cache::ParentTransactionCacheBuilder; +use near_lake_framework::near_primitives::types::AccountId; + +let parent_transaction_cache_ctx = ParentTransactionCacheBuilder::default() + .for_account(String::from("alice.near).try_into().unwrap()) + .for_account(String::from("bob.near).try_into().unwrap()); +``` + diff --git a/lake-parent-transaction-cache/src/lib.rs b/lake-parent-transaction-cache/src/lib.rs new file mode 100644 index 0000000..8002228 --- /dev/null +++ b/lake-parent-transaction-cache/src/lib.rs @@ -0,0 +1,107 @@ +#![doc = include_str!("../README.md")] +#[macro_use] +extern crate derive_builder; + +use cached::{Cached, SizedCache}; +use near_lake_framework::{ + near_indexer_primitives::{near_primitives::types::AccountId, CryptoHash}, + near_lake_primitives::{actions::ActionMetaDataExt, block::Block}, + LakeContextExt, +}; + +pub type ReceiptId = CryptoHash; +pub type TransactionHash = CryptoHash; +type Cache = SizedCache; + +#[derive(Debug, Builder)] +#[builder(pattern = "owned")] +pub struct ParentTransactionCache { + #[builder( + setter(custom = true, name = "cache_size"), + default = "std::sync::RwLock::new(Cache::with_size(100_000))" + )] + cache: std::sync::RwLock, + #[builder(setter(custom = true, name = "for_accounts"))] + account_ids: Vec, +} + +impl ParentTransactionCacheBuilder { + /// Sets the size of the cache. Default is 100_000. + pub fn cache_size(mut self, value: usize) -> Self { + self.cache = Some(std::sync::RwLock::new(Cache::with_size(value))); + self + } + + /// Stores the Vec of [AccountId](near_lake_framework::near_indexer_primitives::near_primitives::types::AccountId) to cache transactions for. + /// If not set, the cache will be created for all the Transactions in the block. + /// If set the cache will be created only for the transactions that have the + /// sender or receiver in the list of accounts. + /// **Warning**: This method overrides the previous value. + pub fn for_accounts(mut self, accounts_id: Vec) -> Self { + self.account_ids = Some(accounts_id); + self + } + + /// Adds an account to the watching list for the parent transaction cache. + /// Similarly to the method [for_accounts](#method.for_accounts) this method will + /// create the cache only for the transactions that have the sender or receiver + /// in the list of accounts. + /// **Warning**: This method appends to the previous value. + pub fn for_account(mut self, account_id: AccountId) -> Self { + if let Some(mut accounts_id) = self.account_ids.take() { + accounts_id.push(account_id); + self.account_ids = Some(accounts_id); + } else { + self.account_ids = Some(vec![account_id]); + } + self + } +} + +impl LakeContextExt for ParentTransactionCache { + /// The process to scan the [near_lake_primitives::Block](near_lake_framework::near_lake_primitives::block::Block) and update the cache + /// with the new transactions and first expected receipts. + /// The cache is used to find the parent transaction hash for a given receipt id. + fn execute_before_run(&self, block: &mut Block) { + // Fill up the cache with new transactions and first expected receipts + // We will try to skip the transactions related to the accounts we're not watching for. + // Based on `accounts_id` + for tx in block.transactions().filter(move |tx| { + self.account_ids.is_empty() + || self.account_ids.contains(tx.signer_id()) + || self.account_ids.contains(tx.receiver_id()) + }) { + let tx_hash = tx.transaction_hash(); + tx.actions_included() + .map(|action| action.metadata().receipt_id()) + .for_each(|receipt_id| { + let mut cache = self.cache.write().unwrap(); + cache.cache_set(receipt_id, tx_hash); + }); + } + for receipt in block.receipts() { + let receipt_id = receipt.receipt_id(); + let mut cache = self.cache.write().unwrap(); + let parent_tx_hash = cache.cache_remove(&receipt_id); + + if let Some(parent_tx_hash) = parent_tx_hash { + cache.cache_set(receipt_id, parent_tx_hash); + } + } + } + + /// We don't need to do anything after the run. + fn execute_after_run(&self) {} +} + +impl ParentTransactionCache { + /// Returns the parent transaction hash for a given receipt id. + /// If the receipt id is not found in the cache, it returns None. + /// If the receipt id is found in the cache, it returns the parent transaction hash. + pub fn get_parent_transaction_hash(&self, receipt_id: &ReceiptId) -> Option { + // **Note**: [cached::SizedCache] updates metadata on every cache access. That's why + // we need to use a write lock here. + let mut cache = self.cache.write().unwrap(); + cache.cache_get(receipt_id).cloned() + } +} diff --git a/lake-primitives/Cargo.toml b/lake-primitives/Cargo.toml new file mode 100644 index 0000000..c5c776e --- /dev/null +++ b/lake-primitives/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "near-lake-primitives" +version.workspace = true +edition = "2021" + +[dependencies] +anyhow = "1.0.51" +near-crypto = "0.17.0" +near-primitives-core = "0.17.0" +near-primitives = "0.17.0" +near-indexer-primitives = "0.17.0" +paste = "1.0.12" +serde = { version = "1", features = ["derive"] } +serde_json = "1.0.75" +thiserror = "1.0.38" diff --git a/lake-primitives/README.md b/lake-primitives/README.md new file mode 100644 index 0000000..65f50c1 --- /dev/null +++ b/lake-primitives/README.md @@ -0,0 +1,27 @@ +```markdown +# NEAR Lake Primitives + +NEAR Lake Primitives is a Rust crate that provides a set of high-level primitives specifically designed for the NEAR Lake Framework. It is part of the effort to streamline and facilitate the development of blockchain applications within the NEAR ecosystem. + +## Features + +The crate offers a range of fundamental primitives, or basic components, that developers can use to build more complex structures and processes within the NEAR Lake Framework. These primitives are optimized for efficient computation and robust interoperability. + +## Usage + +To use the NEAR Lake Primitives in your Rust project, add the following line to your `Cargo.toml` file: + +```toml +[dependencies] +near_lake_primitives = "0.8.0" +``` + +You can then import the crate in your code as follows: + +```rust +use near_lake_primitives; +``` + +## Examples + +TBD - Please provide examples here. \ No newline at end of file diff --git a/lake-primitives/src/lib.rs b/lake-primitives/src/lib.rs new file mode 100644 index 0000000..ecc96fa --- /dev/null +++ b/lake-primitives/src/lib.rs @@ -0,0 +1,16 @@ +pub use near_indexer_primitives::{ + self, near_primitives, types::AccountId, CryptoHash, IndexerShard, StreamerMessage, +}; + +pub use types::{ + actions::{self, Action}, + block::{self, Block, BlockHeader}, + delegate_actions::{self, DelegateAction}, + events::{self, Event, EventsTrait, RawEvent}, + receipts::{self, Receipt, ReceiptKind}, + state_changes::{self, StateChange, StateChangeCause, StateChangeValue}, + transactions::{self, Transaction}, + ReceiptId, +}; + +mod types; diff --git a/lake-primitives/src/types/actions.rs b/lake-primitives/src/types/actions.rs new file mode 100644 index 0000000..426d9f6 --- /dev/null +++ b/lake-primitives/src/types/actions.rs @@ -0,0 +1,321 @@ +use near_crypto::{PublicKey, Signature}; +use near_indexer_primitives::{ + types::{AccountId, Balance, Gas}, + views, CryptoHash, +}; + +use crate::types::delegate_actions; +pub use delegate_actions::{ + DelegateAction, DelegateAddKey, DelegateCreateAccount, DelegateDeleteAccount, + DelegateDeleteKey, DelegateDeployContract, DelegateFunctionCall, DelegateStake, + DelegateTransfer, +}; + +/// Represents the metadata of the action. +/// This is the information that is common to all actions. +#[derive(Debug, Clone)] +pub struct ActionMetadata { + pub(crate) receipt_id: CryptoHash, + pub(crate) predecessor_id: AccountId, + pub(crate) receiver_id: AccountId, + pub(crate) signer_id: AccountId, + pub(crate) signer_public_key: PublicKey, +} + +impl ActionMetadata { + /// Returns the [CryptoHash] id of the corresponding Receipt. + pub fn receipt_id(&self) -> CryptoHash { + self.receipt_id + } + + /// Returns the [AccountId] of the predecessor of the action. + pub fn predecessor_id(&self) -> AccountId { + self.predecessor_id.clone() + } + + /// Returns the [AccountId] of the receiver of the action. + pub fn receiver_id(&self) -> AccountId { + self.receiver_id.clone() + } + + /// Returns the [AccountId] of the signer of the action. + pub fn signer_id(&self) -> AccountId { + self.signer_id.clone() + } + + /// Returns the [PublicKey] of the signer of the action. + pub fn signer_public_key(&self) -> PublicKey { + self.signer_public_key.clone() + } +} + +pub trait ActionMetaDataExt { + fn metadata(&self) -> &ActionMetadata; + + fn receipt_id(&self) -> CryptoHash { + self.metadata().receipt_id() + } + fn predecessor_id(&self) -> AccountId { + self.metadata().predecessor_id() + } + fn receiver_id(&self) -> AccountId { + self.metadata().receiver_id() + } + fn signer_id(&self) -> AccountId { + self.metadata().signer_id() + } + fn signer_public_key(&self) -> PublicKey { + self.metadata().signer_public_key() + } +} + +/// High-level representation of the `Action`. +/// +/// Action is "registered" in the [Transaction](super::transactions::Transaction) to be performed on the blockchain. +/// There is a predefined set of actions that can be performed on the blockchain. +/// +/// #### Important notes on Action enum +/// +/// Please, note that each enum variant is a wrapper around the corresponding action struct. Also, we have special methods +/// for each action type that attempts to convert the action to the corresponding struct. For example, if you have an action +/// of type `Action::Transfer`, you can call `action.as_transfer()` to get the `Transfer` struct. If the action is not of +/// the corresponding type, the method will return `None`. This is done to simplify the usage of the `Action` enum. +#[derive(Debug, Clone)] +pub enum Action { + CreateAccount(CreateAccount), + DeployContract(DeployContract), + FunctionCall(FunctionCall), + Transfer(Transfer), + Stake(Stake), + AddKey(AddKey), + DeleteKey(DeleteKey), + DeleteAccount(DeleteAccount), + Delegate(Delegate), +} + +impl ActionMetaDataExt for Action { + fn metadata(&self) -> &ActionMetadata { + match self { + Self::CreateAccount(action) => action.metadata(), + Self::DeployContract(action) => action.metadata(), + Self::FunctionCall(action) => action.metadata(), + Self::Transfer(action) => action.metadata(), + Self::Stake(action) => action.metadata(), + Self::AddKey(action) => action.metadata(), + Self::DeleteKey(action) => action.metadata(), + Self::DeleteAccount(action) => action.metadata(), + Self::Delegate(action) => action.metadata(), + } + } +} + +macro_rules! impl_as_action_for { + ($action_type:ident) => { + paste::paste! { + pub fn [< as_ $action_type:snake:lower >](&self) -> Option<&$action_type> { + match self { + Self::$action_type(action) => Some(action), + _ => None, + } + } + } + }; +} + +impl Action { + impl_as_action_for!(CreateAccount); + impl_as_action_for!(DeployContract); + impl_as_action_for!(FunctionCall); + impl_as_action_for!(Transfer); + impl_as_action_for!(Stake); + impl_as_action_for!(AddKey); + impl_as_action_for!(DeleteKey); + impl_as_action_for!(DeleteAccount); + impl_as_action_for!(Delegate); +} + +// Macro to implement ActionMetaDataExt trait for each Action variant. +macro_rules! impl_action_metadata_ext { + ($action:ident) => { + impl ActionMetaDataExt for $action { + fn metadata(&self) -> &ActionMetadata { + &self.metadata + } + } + }; +} + +impl_action_metadata_ext!(CreateAccount); +impl_action_metadata_ext!(DeployContract); +impl_action_metadata_ext!(FunctionCall); +impl_action_metadata_ext!(Transfer); +impl_action_metadata_ext!(Stake); +impl_action_metadata_ext!(AddKey); +impl_action_metadata_ext!(DeleteKey); +impl_action_metadata_ext!(DeleteAccount); +impl_action_metadata_ext!(Delegate); + +/// Structure representing the `CreateAccount` action. +/// This is a special action that is used to create a new account on the blockchain. It doesn't contain any +/// additional data. The `receiver_id` from the metadata is the name of the account that is created by this action. +#[derive(Debug, Clone)] +pub struct CreateAccount { + pub(crate) metadata: ActionMetadata, +} + +/// Structure representing the `DeployContract` action. +#[derive(Debug, Clone)] +pub struct DeployContract { + pub(crate) metadata: ActionMetadata, + pub(crate) code: Vec, +} + +impl DeployContract { + /// Returns the contract code bytes. + pub fn code(&self) -> &[u8] { + &self.code + } +} + +/// Structure representing the `FunctionCall` action. +#[derive(Debug, Clone)] +pub struct FunctionCall { + pub(crate) metadata: ActionMetadata, + pub(crate) method_name: String, + pub(crate) args: Vec, + pub(crate) gas: Gas, + pub(crate) deposit: Balance, +} + +impl FunctionCall { + /// Returns the method name this FunctionCall calls. + pub fn method_name(&self) -> &str { + &self.method_name + } + + /// Returns the arguments bytes. + pub fn args(&self) -> &[u8] { + &self.args + } + + /// Returns the gas attached to this FunctionCall. + pub fn gas(&self) -> Gas { + self.gas + } + + /// Returns the deposit attached to this FunctionCall. + pub fn deposit(&self) -> Balance { + self.deposit + } +} + +/// Structure representing the `Transfer` action. +#[derive(Debug, Clone)] +pub struct Transfer { + pub(crate) metadata: ActionMetadata, + pub(crate) deposit: Balance, +} + +impl Transfer { + /// Returns the deposit attached to this Transfer. + pub fn deposit(&self) -> Balance { + self.deposit + } +} + +/// Structure representing the `Stake` action. +#[derive(Debug, Clone)] +pub struct Stake { + pub(crate) metadata: ActionMetadata, + pub(crate) stake: Balance, + pub(crate) public_key: PublicKey, +} + +impl Stake { + /// Returns the stake attached to this Stake. + pub fn stake(&self) -> Balance { + self.stake + } + + /// Returns the public key attached to this Stake. + pub fn public_key(&self) -> &PublicKey { + &self.public_key + } +} + +/// Structure representing the `AddKey` action. +#[derive(Debug, Clone)] +pub struct AddKey { + pub(crate) metadata: ActionMetadata, + pub(crate) public_key: PublicKey, + pub(crate) access_key: views::AccessKeyView, +} + +impl AddKey { + /// Returns the [PublicKey] added with this AddKey. + pub fn public_key(&self) -> &PublicKey { + &self.public_key + } + + /// Returns the [AccessKey](views::AccessKeyView) to the PublicKey being added with this AddKey. + pub fn access_key(&self) -> &views::AccessKeyView { + &self.access_key + } +} + +/// Structure representing the `DeleteKey` action. +#[derive(Debug, Clone)] +pub struct DeleteKey { + pub(crate) metadata: ActionMetadata, + pub(crate) public_key: PublicKey, +} + +impl DeleteKey { + /// Returns the [PublicKey] deleted with this DeleteKey. + pub fn public_key(&self) -> &PublicKey { + &self.public_key + } +} + +/// Structure representing the `DeleteAccount` action. +#[derive(Debug, Clone)] +pub struct DeleteAccount { + pub(crate) metadata: ActionMetadata, + pub(crate) beneficiary_id: AccountId, +} + +impl DeleteAccount { + /// Returns the beneficiary account ID of this DeleteAccount. + pub fn beneficiary_id(&self) -> &AccountId { + &self.beneficiary_id + } +} + +/// Structure representing the `Delegate` action. +/// This is related to the Meta-Transactions [NEP-366](https://github.com/near/NEPs/blob/master/neps/nep-0366.md). +/// +/// This action is used to delegate the right to sign transactions on behalf of the signer to another account. +/// The signer is the account that is signing the transaction that contains this action. +/// The receiver is the account that will be able to sign transactions on behalf of the signer. +/// The `delegate_action` is the action that the receiver will be able to sign on behalf of the signer. +/// The `signature` is the signature of the signer on the hash of the `delegate_action`. +/// +/// The `delegate_action` can be any action, except for another `Delegate` action. Thus not allowing the nesting of `Delegate` actions. +#[derive(Debug, Clone)] +pub struct Delegate { + pub(crate) metadata: ActionMetadata, + pub(crate) delegate_action: Vec, + pub(crate) signature: Signature, +} + +impl Delegate { + /// Returns the delegate action that the receiver will be able to sign on behalf of the signer. + pub fn delegate_action(&self) -> &[delegate_actions::DelegateAction] { + &self.delegate_action + } + + /// Returns the signature of the signer on the hash of the `delegate_action`. + pub fn signature(&self) -> &Signature { + &self.signature + } +} diff --git a/lake-primitives/src/types/block.rs b/lake-primitives/src/types/block.rs new file mode 100644 index 0000000..0fede21 --- /dev/null +++ b/lake-primitives/src/types/block.rs @@ -0,0 +1,357 @@ +use std::collections::HashMap; + +use super::actions::{self, ActionMetaDataExt}; +use super::events::{self, EventsTrait}; +use super::receipts::{self}; +use super::state_changes; +use super::transactions; +use crate::near_indexer_primitives::{types::AccountId, views, CryptoHash, StreamerMessage}; + +/// A structure that represents an entire block in the NEAR blockchain. +/// It is a high-level structure that is built on top of the low-level [StreamerMessage] structure. +/// +/// The access to all the data is provided through the getters. Thus we can refactor the structure yet keep the API stable and backward compatible. +/// +/// With a high-level update we are trying to replace the usage of the low-level [StreamerMessage] with this one. +/// +/// #### Important notes on the Block +/// - All the entities located on different shards were merged into one single list without differentiation. +/// The statement from **NEAR** is that **sharding is going to be dynamic and seamless for the users**, that’s why we’ve decided indexer +/// developers don’t want to care about shards either. +/// - Original [near_indexer_primitives::StreamerMessage] represents the blockchain data in *a most fair manner**. Although, it used to be +/// a pain in the neck for indexer developers, we’ve decided to act as a painkiller here. +/// - [Block] is not the fairest name for this structure either. **NEAR Protocol** is a sharded blockchain, so its block is actually an +/// ephemeral structure that represents a collection of *real blocks* called Chunks in **NEAR Protocol**. We’ve been simplifying things here though, +/// so here is a result of the simplification. +#[derive(Debug)] +pub struct Block { + streamer_message: StreamerMessage, + executed_receipts: Vec, + postponed_receipts: Vec, + transactions: Vec, + actions: Vec, + events: HashMap>, + state_changes: Vec, +} + +impl Block { + /// Return a reference to the original StreamerMessage of the block. This is the low-level structure. + /// + /// While introducing the high-level structures, methods, and helpers, we do want to keep the low-level “door” open + /// for advanced developers or edge cases which we haven’t accidentally covered, or just don’t have the capacity to cover. + /// + /// That’s why every instance of the Block will hold the original StreamerMessage for developers. + /// Think of it as backward compatibility if you prefer. + pub fn streamer_message(&self) -> &StreamerMessage { + &self.streamer_message + } + + /// Returns the block hash. It is a shortcut to get the data from the block header. + pub fn block_hash(&self) -> CryptoHash { + self.header().hash() + } + + /// Returns the previous block hash. It is a shortcut to get the data from the block header. + pub fn prev_block_hash(&self) -> CryptoHash { + self.header().prev_hash() + } + + /// Returns the block height. It is a shortcut to get the data from the block header. + pub fn block_height(&self) -> u64 { + self.header().height() + } + + /// Returns a [BlockHeader] structure of the block + /// + ///See [BlockHeader] structure sections for details. + pub fn header(&self) -> BlockHeader { + (&self.streamer_message).into() + } + + /// Returns an iterator over the [Receipt](crate::receipts::Receipt)s executed in this [Block]. + /// + /// This field is a representation of `StreamerMessage.shard[N].receipt_execution_outcomes` + /// + /// A reminder that `receipt_execution_outcomes` has a type [near_indexer_primitives::IndexerExecutionOutcomeWithReceipt] which is an + /// ephemeral structure from `near-indexer-primitives` that hold a [near_primitives::views::ExecutionOutcomeView] + /// along with the corresponding [near_primitives::views::ReceiptView]. + pub fn receipts(&mut self) -> impl Iterator { + if self.executed_receipts.is_empty() { + self.executed_receipts = self + .streamer_message + .shards + .iter() + .flat_map(|shard| shard.receipt_execution_outcomes.iter()) + .map(Into::into) + .collect(); + } + self.executed_receipts.iter() + } + + /// Returns an iterator of [Receipt](crate::receipts::Receipt) included yet not executed in the [Block]. + /// + /// [Receipts](crate::receipts::Receipt) included on the chain but not executed yet are called "postponed", + /// they are represented by the same structure [Receipt](crate::receipts::Receipt). + pub fn postponed_receipts(&mut self) -> impl Iterator { + if self.postponed_receipts.is_empty() { + let executed_receipts_ids: Vec<_> = self + .receipts() + .map(|receipt| receipt.receipt_id()) + .collect(); + self.postponed_receipts = self + .streamer_message + .shards + .iter() + .filter_map(|shard| shard.chunk.as_ref().map(|chunk| chunk.receipts.iter())) + .flatten() + // exclude receipts that are already executed + .filter(|receipt| !executed_receipts_ids.contains(&receipt.receipt_id)) + .map(Into::into) + .collect(); + } + self.postponed_receipts.iter() + } + + /// Returns an iterator of the [Transactions](crate::transactions::Transaction) included in the [Block]. + /// + /// **Heads up!** Some indexer developers care about [Transaction](crate::transactions::Transaction)s for the knowledge where + /// the action chain has begun. Other indexer developers care about it because of the habits + /// from other blockchains like Ethereum where a transaction is a main asset. In case of NEAR + /// [Receipts](crate::receipts::Receipt) are more important. + pub fn transactions(&mut self) -> impl Iterator { + if self.transactions.is_empty() { + self.transactions = self + .streamer_message + .shards + .iter() + .filter_map(|shard| shard.chunk.as_ref().map(|chunk| chunk.transactions.iter())) + .flatten() + .map(TryInto::try_into) + .filter_map(|transactions| transactions.ok()) + .collect(); + } + self.transactions.iter() + } + + /// Internal method to build the cache of actions on demand + fn actions_from_streamer_message(&self) -> Vec { + self.streamer_message() + .shards + .iter() + .flat_map(|shard| shard.receipt_execution_outcomes.iter()) + .filter_map(|receipt_execution_outcome| { + actions::Action::try_vec_from_receipt_view(&receipt_execution_outcome.receipt).ok() + }) + .flatten() + .collect() + } + + /// Returns an iterator of the [Actions](crate::actions::Action) executed in the [Block] + pub fn actions(&mut self) -> impl Iterator { + if self.actions.is_empty() { + self.build_actions_cache(); + } + self.actions.iter() + } + + /// Returns an iterator of the [Events](crate::events::Event) emitted in the [Block] + pub fn events(&mut self) -> impl Iterator { + if self.events.is_empty() { + self.build_events_hashmap(); + } + self.events.values().flatten() + } + + /// Returns an iterator of the [StateChanges](crate::state_changes::StateChange) happened in the [Block] + pub fn state_changes(&mut self) -> impl Iterator { + if self.state_changes.is_empty() { + self.state_changes = self + .streamer_message + .shards + .iter() + .flat_map(|shard| shard.state_changes.iter()) + .map(Into::into) + .collect(); + } + self.state_changes.iter() + } + + /// Helper to get all the [Actions](crate::actions::Action) by the single [Receipt](crate::receipts::Receipt) + /// + /// **Heads up!** This methods searches for the actions in the current [Block] only. + pub fn actions_by_receipt_id<'a>( + &'a mut self, + receipt_id: &'a super::ReceiptId, + ) -> impl Iterator + 'a { + self.actions() + .filter(move |action| &action.receipt_id() == receipt_id) + } + + /// Helper to get all the [Events](crate::events::Event) emitted by the specific [Receipt](crate::receipts::Receipt) + pub fn events_by_receipt_id(&mut self, receipt_id: &super::ReceiptId) -> Vec { + if self.events.is_empty() { + self.build_events_hashmap(); + } + if let Some(events) = self.events.get(receipt_id) { + events.to_vec() + } else { + vec![] + } + } + + /// Helper to get all the [Events](crate::events::Event) emitted by the specific contract ([AccountId](crate::near_indexer_primitives::types::AccountId)) + pub fn events_by_contract_id<'a>( + &'a mut self, + account_id: &'a crate::near_indexer_primitives::types::AccountId, + ) -> impl Iterator + 'a { + self.events() + .filter(move |event| event.is_emitted_by_contract(&account_id.clone())) + } + + /// Helper to get a specific [Receipt](crate::receipts::Receipt) by the [ReceiptId](crate::types::ReceiptId) + pub fn receipt_by_id(&mut self, receipt_id: &super::ReceiptId) -> Option<&receipts::Receipt> { + self.receipts() + .find(|receipt| &receipt.receipt_id() == receipt_id) + } +} + +impl Block { + // Internal method to build the cache of actions on demand + fn build_actions_cache(&mut self) { + self.actions = self.actions_from_streamer_message().to_vec(); + } + + // Internal method to build the cache of events on demand + fn build_events_hashmap(&mut self) { + self.events = self + .receipts() + .map(|receipt| (receipt.receipt_id(), receipt.events())) + .collect(); + } +} + +impl From for Block { + fn from(streamer_message: StreamerMessage) -> Self { + Self { + streamer_message, + executed_receipts: vec![], + postponed_receipts: vec![], + transactions: vec![], + actions: vec![], + events: HashMap::new(), + state_changes: vec![], + } + } +} + +/// Replacement for [`BlockHeaderView`](near_primitives::views::BlockHeaderView) from `near-primitives`. Shrank and simplified. +/// We were trying to leave only the fields indexer developers might be interested in. +/// +/// Friendly reminder, the original [`BlockHeaderView`](near_primitives::views::BlockHeaderView) is still accessible +/// via [`.streamer_message()`](Block::streamer_message()) method. +#[derive(Debug, Clone)] +pub struct BlockHeader { + height: u64, + hash: CryptoHash, + prev_hash: CryptoHash, + author: AccountId, + timestamp_nanosec: u64, + epoch_id: CryptoHash, + next_epoch_id: CryptoHash, + gas_price: u128, + total_supply: u128, + latest_protocol_version: u32, + random_value: CryptoHash, + chunks_included: u64, + validator_proposals: Vec, +} + +impl BlockHeader { + /// The height of the [Block] + pub fn height(&self) -> u64 { + self.height + } + + /// The hash of the [Block] + pub fn hash(&self) -> CryptoHash { + self.hash + } + + /// The hash of the previous [Block] + pub fn prev_hash(&self) -> CryptoHash { + self.prev_hash + } + + /// The [AccountId](crate::near_indexer_primitives::types::AccountId) of the author of the [Block] + pub fn author(&self) -> AccountId { + self.author.clone() + } + + /// The timestamp of the [Block] in nanoseconds + pub fn timestamp_nanosec(&self) -> u64 { + self.timestamp_nanosec + } + + /// The [CryptoHash] of the epoch the [Block] belongs to + pub fn epoch_id(&self) -> CryptoHash { + self.epoch_id + } + + /// The [CryptoHash] of the next epoch + pub fn next_epoch_id(&self) -> CryptoHash { + self.next_epoch_id + } + + /// The gas price of the [Block] + pub fn gas_price(&self) -> u128 { + self.gas_price + } + + /// The total supply of the [Block] + pub fn total_supply(&self) -> u128 { + self.total_supply + } + + /// The latest protocol version of the [Block] + pub fn latest_protocol_version(&self) -> u32 { + self.latest_protocol_version + } + + /// The random value of the [Block] + pub fn random_value(&self) -> CryptoHash { + self.random_value + } + + /// The number of chunks included in the [Block] + pub fn chunks_included(&self) -> u64 { + self.chunks_included + } + + /// The validator proposals of the [Block] + /// + /// **Heads up!** This methods returns types defined in the `near-primitives` crate as is. + /// It is a subject of change in the future (once we define the corresponding Lake Primitives types) + pub fn validator_proposals(&self) -> Vec { + self.validator_proposals.clone() + } +} + +impl From<&StreamerMessage> for BlockHeader { + fn from(streamer_message: &StreamerMessage) -> Self { + Self { + height: streamer_message.block.header.height, + hash: streamer_message.block.header.hash, + prev_hash: streamer_message.block.header.prev_hash, + author: streamer_message.block.author.clone(), + timestamp_nanosec: streamer_message.block.header.timestamp_nanosec, + epoch_id: streamer_message.block.header.epoch_id, + next_epoch_id: streamer_message.block.header.next_epoch_id, + gas_price: streamer_message.block.header.gas_price, + total_supply: streamer_message.block.header.total_supply, + latest_protocol_version: streamer_message.block.header.latest_protocol_version, + random_value: streamer_message.block.header.random_value, + chunks_included: streamer_message.block.header.chunks_included, + validator_proposals: streamer_message.block.header.validator_proposals.clone(), + } + } +} diff --git a/lake-primitives/src/types/delegate_actions.rs b/lake-primitives/src/types/delegate_actions.rs new file mode 100644 index 0000000..5375303 --- /dev/null +++ b/lake-primitives/src/types/delegate_actions.rs @@ -0,0 +1,271 @@ +use near_crypto::PublicKey; +use near_indexer_primitives::{ + types::{AccountId, Balance, Gas}, + views::{self, AccessKeyView}, +}; + +/// Similarly to the [Action](super::actions::Action) enum, this enum represents the different types of actions that can be +/// delegated to a contract. +/// +/// `DelegateAction` enum has a corresponding `Action` variant for every possible `Action` except the `DelegateAction` itself. +/// Thus forbidding the nesting of `DelegateActions` and making the `Action` enum exhaustive. +/// Another difference is that `DelegateAction` itself and it's variants do not hold metadata and don't implement `ActionMetaDataExt`. +#[derive(Debug, Clone)] +pub enum DelegateAction { + DelegateCreateAccount(DelegateCreateAccount), + DelegateDeployContract(DelegateDeployContract), + DelegateFunctionCall(DelegateFunctionCall), + DelegateTransfer(DelegateTransfer), + DelegateStake(DelegateStake), + DelegateAddKey(DelegateAddKey), + DelegateDeleteKey(DelegateDeleteKey), + DelegateDeleteAccount(DelegateDeleteAccount), +} + +impl DelegateAction { + /// Attempts to return the [DelegateFunctionCall](struct@DelegateFunctionCall) struct if the variant is [DelegateAction::DelegateFunctionCall]. Otherwise returns `None`. + pub fn as_delegate_function_call(&self) -> Option<&DelegateFunctionCall> { + match self { + DelegateAction::DelegateFunctionCall(action) => Some(action), + _ => None, + } + } + + /// Attempts to return the [DelegateCreateAccount] struct if the variant is [DelegateAction::DelegateCreateAccount]. Otherwise returns `None`. + pub fn as_delegate_create_account(&self) -> Option<&DelegateCreateAccount> { + match self { + DelegateAction::DelegateCreateAccount(action) => Some(action), + _ => None, + } + } + + /// Attempts to return the [DelegateDeployContract] struct if the variant is [DelegateAction::DelegateDeployContract]. Otherwise returns `None`. + pub fn as_delegate_deploy_contract(&self) -> Option<&DelegateDeployContract> { + match self { + DelegateAction::DelegateDeployContract(action) => Some(action), + _ => None, + } + } + + /// Attempts to return the [DelegateTransfer] struct if the variant is [DelegateAction::DelegateTransfer]. Otherwise returns `None`. + pub fn as_delegate_transfer(&self) -> Option<&DelegateTransfer> { + match self { + DelegateAction::DelegateTransfer(action) => Some(action), + _ => None, + } + } + + /// Attempts to return the [DelegateStake] struct if the variant is [DelegateAction::DelegateStake]. Otherwise returns `None`. + pub fn as_delegate_stake(&self) -> Option<&DelegateStake> { + match self { + DelegateAction::DelegateStake(action) => Some(action), + _ => None, + } + } + + /// Attempts to return the [DelegateAddKey] struct if the variant is [DelegateAction::DelegateAddKey]. Otherwise returns `None`. + pub fn as_delegate_add_key(&self) -> Option<&DelegateAddKey> { + match self { + DelegateAction::DelegateAddKey(action) => Some(action), + _ => None, + } + } + + /// Attempts to return the [DelegateDeleteKey] struct if the variant is [DelegateAction::DelegateDeleteKey]. Otherwise returns `None`. + pub fn as_delegate_delete_key(&self) -> Option<&DelegateDeleteKey> { + match self { + DelegateAction::DelegateDeleteKey(action) => Some(action), + _ => None, + } + } + + /// Attempts to return the [DelegateDeleteAccount] struct if the variant is [DelegateAction::DelegateDeleteAccount]. Otherwise returns `None`. + pub fn as_delegate_delete_account(&self) -> Option<&DelegateDeleteAccount> { + match self { + DelegateAction::DelegateDeleteAccount(action) => Some(action), + _ => None, + } + } +} + +/// Similarly to [CreateAccount](super::actions::CreateAccount), this struct represents the `CreateAccount` action that is delegated. +#[derive(Debug, Clone)] +pub struct DelegateCreateAccount; + +/// Similarly to [DeployContract](super::actions::DeployContract), this struct represents the `DeployContract` action that is delegated. +#[derive(Debug, Clone)] +pub struct DelegateDeployContract { + pub(crate) code: Vec, +} + +impl DelegateDeployContract { + /// Returns the bytes of the contract code that is being deployed. + pub fn code(&self) -> &[u8] { + &self.code + } +} + +/// Similarly to [FunctionCall](super::actions::FunctionCall), this struct represents the `FunctionCall` action that is delegated. +#[derive(Debug, Clone)] +pub struct DelegateFunctionCall { + pub(crate) method_name: String, + pub(crate) args: Vec, + pub(crate) gas: Gas, + pub(crate) deposit: Balance, +} + +impl DelegateFunctionCall { + /// Returns the name of the method that is being called. + pub fn method_name(&self) -> &str { + &self.method_name + } + + /// Returns the bytes of the arguments that are being passed to the method. + pub fn args(&self) -> &[u8] { + &self.args + } + + /// Returns the amount of gas that is being used for the method call. + pub fn gas(&self) -> Gas { + self.gas + } + + /// Returns the amount of tokens that are being deposited to the contract. + pub fn deposit(&self) -> Balance { + self.deposit + } +} + +/// Similarly to [Transfer](super::actions::Transfer), this struct represents the `Transfer` action that is delegated. +#[derive(Debug, Clone)] +pub struct DelegateTransfer { + pub(crate) deposit: Balance, +} + +impl DelegateTransfer { + /// Returns the amount of tokens that are being transferred. + pub fn deposit(&self) -> Balance { + self.deposit + } +} + +/// Similarly to [Stake](super::actions::Stake), this struct represents the `Stake` action that is delegated. +#[derive(Debug, Clone)] +pub struct DelegateStake { + pub(crate) stake: Balance, + pub(crate) public_key: PublicKey, +} + +impl DelegateStake { + /// Returns the amount of tokens that are being staked. + pub fn stake(&self) -> Balance { + self.stake + } + + /// Returns the public key of the staking pool. + pub fn public_key(&self) -> &PublicKey { + &self.public_key + } +} + +/// Similarly to [AddKey](super::actions::AddKey), this struct represents the `AddKey` action that is delegated. +#[derive(Debug, Clone)] +pub struct DelegateAddKey { + pub(crate) public_key: PublicKey, + pub(crate) access_key: AccessKeyView, +} + +impl DelegateAddKey { + /// Returns the public key that is being added. + pub fn public_key(&self) -> &PublicKey { + &self.public_key + } + + /// Returns the access key that is being added. + pub fn access_key(&self) -> &AccessKeyView { + &self.access_key + } +} + +/// Similarly to [DeleteKey](super::actions::DeleteKey), this struct represents the `DeleteKey` action that is delegated. +#[derive(Debug, Clone)] +pub struct DelegateDeleteKey { + pub(crate) public_key: PublicKey, +} + +impl DelegateDeleteKey { + /// Returns the public key that is being deleted. + pub fn public_key(&self) -> &PublicKey { + &self.public_key + } +} + +/// Similarly to [DeleteAccount](super::actions::DeleteAccount), this struct represents the `DeleteAccount` action that is delegated. +#[derive(Debug, Clone)] +pub struct DelegateDeleteAccount { + pub(crate) beneficiary_id: AccountId, +} + +impl DelegateDeleteAccount { + /// Returns the account ID of the beneficiary. + pub fn beneficiary_id(&self) -> &AccountId { + &self.beneficiary_id + } +} + +impl DelegateAction { + // Tries to convert a `near_primitives::delegate_action::DelegateAction` into a [Vec]. + pub fn try_from_delegate_action( + delegate_action: &near_primitives::delegate_action::DelegateAction, + ) -> Result, &'static str> { + let mut actions = Vec::with_capacity(delegate_action.actions.len()); + + for nearcore_action in delegate_action.clone().actions { + let action = match views::ActionView::from( + >::into(nearcore_action), + ) { + views::ActionView::CreateAccount => { + Self::DelegateCreateAccount(DelegateCreateAccount) + } + views::ActionView::DeployContract { code } => { + Self::DelegateDeployContract(DelegateDeployContract { code }) + } + views::ActionView::FunctionCall { + method_name, + args, + gas, + deposit, + } => Self::DelegateFunctionCall(DelegateFunctionCall { + method_name, + args: args.into(), + gas, + deposit, + }), + views::ActionView::Transfer { deposit } => { + Self::DelegateTransfer(DelegateTransfer { deposit }) + } + views::ActionView::Stake { stake, public_key } => { + Self::DelegateStake(DelegateStake { stake, public_key }) + } + views::ActionView::AddKey { + public_key, + access_key, + } => Self::DelegateAddKey(DelegateAddKey { + public_key, + access_key, + }), + views::ActionView::DeleteKey { public_key } => { + Self::DelegateDeleteKey(DelegateDeleteKey { public_key }) + } + views::ActionView::DeleteAccount { beneficiary_id } => { + Self::DelegateDeleteAccount(DelegateDeleteAccount { beneficiary_id }) + } + _ => return Err("Cannot delegate DelegateAction"), + }; + actions.push(action); + } + Ok(actions) + } +} diff --git a/lake-primitives/src/types/events.rs b/lake-primitives/src/types/events.rs new file mode 100644 index 0000000..59de0e3 --- /dev/null +++ b/lake-primitives/src/types/events.rs @@ -0,0 +1,102 @@ +use crate::AccountId; + +use super::receipts::Receipt; + +/// Hight-level representation of the Event according to the [Events Format](https://nomicon.io/Standards/EventsFormat.html). +/// In addition to the event this structure holds the data about the related [Receipt]: `receipt_id`, `receiver_id` and `predecessor_id`. All these fields are accessible via the corresponding getters. +#[derive(Clone, Debug)] +pub struct Event { + pub(crate) related_receipt_id: crate::CryptoHash, + pub(crate) receiver_id: AccountId, + pub(crate) predecessor_id: AccountId, + pub(crate) raw_event: RawEvent, +} + +impl Event { + /// Returns the `event` value from the [RawEvent]. + pub fn event(&self) -> &str { + &self.raw_event.event + } + + /// Returns the `standard` value from the [RawEvent]. + pub fn standard(&self) -> &str { + &self.raw_event.standard + } + + /// Returns the `version` value from the [RawEvent]. + pub fn version(&self) -> &str { + &self.raw_event.version + } + + /// Returns the `data` value from the [RawEvent] if present, otherwise returns `None`. + pub fn data(&self) -> Option<&serde_json::Value> { + self.raw_event.data.as_ref() + } + + /// Returns the [CryptoHash](crate::CryptoHash) id of the related [Receipt]. + /// + /// **Please note** that events are emitted through the `ExecutionOutcome` logs. In turn, the `ExecutionOutcome` + /// is a result of the execution of the [Receipt]. + pub fn related_receipt_id(&self) -> crate::CryptoHash { + self.related_receipt_id + } + + /// Returns the [AccountId] of the receiver of the related [Receipt]. + pub fn related_receipt_receiver_id(&self) -> &AccountId { + &self.receiver_id + } + + /// Returns the [AccountId] of the predecessor of the related [Receipt]. + pub fn related_receipt_predecessor_id(&self) -> &AccountId { + &self.predecessor_id + } + + /// Returns true if the event is produced by the given contract id. + pub fn is_emitted_by_contract(&self, contract_account_id: &AccountId) -> bool { + &self.receiver_id == contract_account_id + } +} + +/// This structure is an honest representation of the Events Format standard described here +/// +#[derive(Clone, Debug, serde::Deserialize)] +pub struct RawEvent { + pub event: String, + pub standard: String, + pub version: String, + pub data: Option, +} + +impl RawEvent { + /// Parses the log message (originated from `ExecutionOutcome` but not limited) and returns the RawEvent. + pub fn from_log(log: &str) -> anyhow::Result { + let prefix = "EVENT_JSON:"; + if !log.starts_with(prefix) { + anyhow::bail!("log message doesn't start from required prefix"); + } + + Ok(serde_json::from_str::<'_, Self>( + log[prefix.len()..].trim(), + )?) + } +} + +pub trait EventsTrait { + fn events(&self) -> Vec; +} + +impl EventsTrait for Receipt { + /// Reads the logs from the [Receipt] and extracts all the [Events](Event) from it into a Vec. + fn events(&self) -> Vec { + self.logs() + .iter() + .filter_map(|log| RawEvent::from_log(log).ok()) + .map(|raw_event| Event { + related_receipt_id: self.receipt_id(), + receiver_id: self.receiver_id(), + predecessor_id: self.predecessor_id(), + raw_event, + }) + .collect() + } +} diff --git a/lake-primitives/src/types/impl_actions.rs b/lake-primitives/src/types/impl_actions.rs new file mode 100644 index 0000000..c6cfb98 --- /dev/null +++ b/lake-primitives/src/types/impl_actions.rs @@ -0,0 +1,200 @@ +use near_indexer_primitives::{views, IndexerTransactionWithOutcome}; + +use crate::actions::{Action, ActionMetadata, DelegateAction}; + +impl Action { + // Tries to convert a [&ReceiptView](views::ReceiptView) into a vector of [Action]. + pub fn try_vec_from_receipt_view( + receipt_view: &views::ReceiptView, + ) -> Result, &'static str> { + if let views::ReceiptEnumView::Action { + actions, + signer_id, + signer_public_key, + .. + } = &receipt_view.receipt + { + let metadata = ActionMetadata { + receipt_id: receipt_view.receipt_id, + predecessor_id: receipt_view.predecessor_id.clone(), + receiver_id: receipt_view.receiver_id.clone(), + signer_id: signer_id.clone(), + signer_public_key: signer_public_key.clone(), + }; + + let mut result = Vec::with_capacity(actions.len()); + + for action in actions { + let action_kind = match action { + views::ActionView::CreateAccount => { + Self::CreateAccount(crate::actions::CreateAccount { + metadata: metadata.clone(), + }) + } + views::ActionView::DeployContract { code } => { + Self::DeployContract(crate::actions::DeployContract { + metadata: metadata.clone(), + code: code.clone(), + }) + } + views::ActionView::FunctionCall { + method_name, + args, + gas, + deposit, + } => Self::FunctionCall(crate::actions::FunctionCall { + metadata: metadata.clone(), + method_name: method_name.clone(), + args: args.clone().into(), + gas: *gas, + deposit: *deposit, + }), + views::ActionView::Transfer { deposit } => { + Self::Transfer(crate::actions::Transfer { + metadata: metadata.clone(), + deposit: *deposit, + }) + } + views::ActionView::Stake { stake, public_key } => { + Self::Stake(crate::actions::Stake { + metadata: metadata.clone(), + stake: *stake, + public_key: public_key.clone(), + }) + } + views::ActionView::AddKey { + public_key, + access_key, + } => Self::AddKey(crate::actions::AddKey { + metadata: metadata.clone(), + public_key: public_key.clone(), + access_key: access_key.clone(), + }), + views::ActionView::DeleteKey { public_key } => { + Self::DeleteKey(crate::actions::DeleteKey { + metadata: metadata.clone(), + public_key: public_key.clone(), + }) + } + views::ActionView::DeleteAccount { beneficiary_id } => { + Self::DeleteAccount(crate::actions::DeleteAccount { + metadata: metadata.clone(), + beneficiary_id: beneficiary_id.clone(), + }) + } + views::ActionView::Delegate { + delegate_action, + signature, + } => { + let delegate_actions = + DelegateAction::try_from_delegate_action(delegate_action)?; + + Self::Delegate(crate::actions::Delegate { + metadata: metadata.clone(), + delegate_action: delegate_actions, + signature: signature.clone(), + }) + } + }; + result.push(action_kind); + } + Ok(result) + } else { + Err("Only `ReceiptEnumView::Action` can be converted into Vec") + } + } + + // Tries to convert a [IndexerTransactionWithOutcome] to a [Vec] + pub fn try_vec_from_transaction_outcome( + transaction_with_outcome: &IndexerTransactionWithOutcome, + ) -> Result, &'static str> { + let metadata = ActionMetadata { + receipt_id: *transaction_with_outcome + .outcome + .execution_outcome + .outcome + .receipt_ids + .get(0) + .ok_or("Transaction conversion ReceiptId is missing")?, + predecessor_id: transaction_with_outcome.transaction.signer_id.clone(), + receiver_id: transaction_with_outcome.transaction.receiver_id.clone(), + signer_id: transaction_with_outcome.transaction.signer_id.clone(), + signer_public_key: transaction_with_outcome.transaction.public_key.clone(), + }; + + let mut actions: Vec = vec![]; + + for nearcore_action in &transaction_with_outcome.transaction.actions { + let action = match nearcore_action { + views::ActionView::CreateAccount => { + Self::CreateAccount(crate::actions::CreateAccount { + metadata: metadata.clone(), + }) + } + views::ActionView::DeployContract { code } => { + Self::DeployContract(crate::actions::DeployContract { + metadata: metadata.clone(), + code: code.to_vec(), + }) + } + views::ActionView::FunctionCall { + method_name, + args, + gas, + deposit, + } => Self::FunctionCall(crate::actions::FunctionCall { + metadata: metadata.clone(), + method_name: method_name.to_string(), + args: args.to_vec(), + gas: *gas, + deposit: *deposit, + }), + views::ActionView::Transfer { deposit } => { + Self::Transfer(crate::actions::Transfer { + metadata: metadata.clone(), + deposit: *deposit, + }) + } + views::ActionView::Stake { stake, public_key } => { + Self::Stake(crate::actions::Stake { + metadata: metadata.clone(), + stake: *stake, + public_key: public_key.clone(), + }) + } + views::ActionView::AddKey { + public_key, + access_key, + } => Self::AddKey(crate::actions::AddKey { + metadata: metadata.clone(), + public_key: public_key.clone(), + access_key: access_key.clone(), + }), + views::ActionView::DeleteKey { public_key } => { + Self::DeleteKey(crate::actions::DeleteKey { + metadata: metadata.clone(), + public_key: public_key.clone(), + }) + } + views::ActionView::DeleteAccount { beneficiary_id } => { + Self::DeleteAccount(crate::actions::DeleteAccount { + metadata: metadata.clone(), + beneficiary_id: beneficiary_id.clone(), + }) + } + views::ActionView::Delegate { + delegate_action, + signature, + } => Self::Delegate(crate::actions::Delegate { + metadata: metadata.clone(), + delegate_action: DelegateAction::try_from_delegate_action(delegate_action)?, + signature: signature.clone(), + }), + }; + + actions.push(action); + } + + Ok(actions) + } +} diff --git a/lake-primitives/src/types/mod.rs b/lake-primitives/src/types/mod.rs new file mode 100644 index 0000000..2b3f554 --- /dev/null +++ b/lake-primitives/src/types/mod.rs @@ -0,0 +1,12 @@ +pub mod actions; +pub mod block; +pub mod delegate_actions; +pub mod events; +mod impl_actions; +pub mod receipts; +pub mod state_changes; +pub mod transactions; + +/// Since both [transactions::Transaction] hash and [receipts::Receipt] id are the [crate::CryptoHash] type, +/// we use this type alias to make the code more readable. +pub type ReceiptId = near_indexer_primitives::CryptoHash; diff --git a/lake-primitives/src/types/receipts.rs b/lake-primitives/src/types/receipts.rs new file mode 100644 index 0000000..7250a2a --- /dev/null +++ b/lake-primitives/src/types/receipts.rs @@ -0,0 +1,158 @@ +use crate::near_indexer_primitives::{ + types::AccountId, views, CryptoHash, IndexerExecutionOutcomeWithReceipt, +}; + +/// Simplified representation of the `Receipt`. +/// +/// This is a simplification from the [near_primitives::views::ReceiptView] and [near_primitives::views::ReceiptEnumView] into a more flat structure. +/// The [ReceiptKind] is used to distinguish between the different types of receipts: Action and Data. +/// +/// #### Important notes on the Receipt +/// +/// The original low-level Receipt is represented by the enum that differentiates between the Action and Data receipts. In turn this enum is a field +/// `receipt` in the parent `ReceiptView` struct. +/// Parent structure has a set of fields that are common for both Action and Data receipts. +/// During the simplification we have put the common fields into the [Receipt] struct itself and extracted the `actions` from Action Receipt into a separate struct. +/// Since the high-level NEAR Lake Framework update we encourage developers to create more actions-and-events oriented indexers instead. +#[derive(Debug, Clone)] +pub struct Receipt { + receipt_kind: ReceiptKind, + receipt_id: CryptoHash, + receiver_id: AccountId, + predecessor_id: AccountId, + status: ExecutionStatus, + execution_outcome_id: Option, + logs: Vec, +} + +impl Receipt { + /// Returns the [ReceiptKind](ReceiptKind) of the receipt. + /// + /// This is a simplification from the [near_primitives::views::ReceiptEnumView::Action] into a more flat structure + /// that has a type. + pub fn receipt_kind(&self) -> ReceiptKind { + self.receipt_kind.clone() + } + + /// Returns the [CryptoHash] id of the receipt. + pub fn receipt_id(&self) -> CryptoHash { + self.receipt_id + } + + /// Returns the [AccountId] of the receiver of the receipt. + pub fn receiver_id(&self) -> AccountId { + self.receiver_id.clone() + } + + /// Returns the [AccountId] of the predecessor of the receipt. + pub fn predecessor_id(&self) -> AccountId { + self.predecessor_id.clone() + } + + /// Returns the [ExecutionStatus] of the corresponding ExecutionOutcome. + /// + /// Note that the status will be `Postponed` for the receipts that are included in the block but not executed yet. + pub fn status(&self) -> ExecutionStatus { + self.status.clone() + } + + /// Returns the [CryptoHash] id of the corresponding ExecutionOutcome if it exists. + /// + /// Note that this is an optional field because the ExecutionOutcome might not be available + /// if the [Receipt] is "postponed" (included in the block but not executed yet) + pub fn execution_outcome_id(&self) -> Option { + self.execution_outcome_id + } + + /// Returns the logs of the corresponding ExecutionOutcome. + /// Might be an empty Vec if the ExecutionOutcome is not available. + pub fn logs(&self) -> Vec { + self.logs.clone() + } +} + +impl From<&IndexerExecutionOutcomeWithReceipt> for Receipt { + fn from(outcome_with_receipt: &IndexerExecutionOutcomeWithReceipt) -> Self { + Self { + receipt_kind: (&outcome_with_receipt.receipt.receipt).into(), + receipt_id: outcome_with_receipt.receipt.receipt_id, + receiver_id: outcome_with_receipt.receipt.receiver_id.clone(), + predecessor_id: outcome_with_receipt.receipt.predecessor_id.clone(), + execution_outcome_id: Some(outcome_with_receipt.execution_outcome.id), + logs: outcome_with_receipt + .execution_outcome + .outcome + .logs + .iter() + .map(Clone::clone) + .collect(), + status: (&outcome_with_receipt.execution_outcome.outcome.status).into(), + } + } +} + +impl From<&views::ReceiptView> for Receipt { + fn from(receipt: &views::ReceiptView) -> Self { + Self { + receipt_kind: (&receipt.receipt).into(), + receipt_id: receipt.receipt_id, + receiver_id: receipt.receiver_id.clone(), + predecessor_id: receipt.predecessor_id.clone(), + status: ExecutionStatus::Postponed, + execution_outcome_id: None, + logs: vec![], + } + } +} + +/// Represents the Receipt kind: Action or Data. +#[derive(Debug, Clone)] +pub enum ReceiptKind { + /// For the Action Receipt + Action, + /// For the Data Receipt + Data, +} + +impl From<&views::ReceiptEnumView> for ReceiptKind { + fn from(receipt_enum: &views::ReceiptEnumView) -> Self { + match receipt_enum { + views::ReceiptEnumView::Action { .. } => Self::Action, + views::ReceiptEnumView::Data { .. } => Self::Data, + } + } +} + +/// Representation of the execution status for the [Receipt]. +#[derive(Debug, Clone)] +pub enum ExecutionStatus { + /// Execution succeeded with a value, value is represented by [`Vec`] and literally can be anything. + SuccessValue(Vec), + /// Execution succeeded and a result of the execution is a new [Receipt] with the id represented by [CryptoHash] + SuccessReceiptId(CryptoHash), + // TODO: handle the Failure and all the nested errors it has + /// Execution failed with an error represented by a [String] + /// **WARNINNG!** Here must be our representation of the `TxExecutionError from `near-primitives` instead of the [String]. + /// It requires some additional work on our version of the error, meanwhile we’ve left the [String] here, **this is subject to change + /// in the nearest updates**. + Failure(String), + /// Execution hasn’t started yet, it is postponed (delayed) and will be later. + /// The Receipt with such status is considered as postponed too (included, yet not executed) + Postponed, +} + +impl From<&views::ExecutionStatusView> for ExecutionStatus { + fn from(execution_status_view: &views::ExecutionStatusView) -> Self { + match execution_status_view { + views::ExecutionStatusView::Unknown => Self::Postponed, + views::ExecutionStatusView::SuccessValue(value) => Self::SuccessValue(value.clone()), + views::ExecutionStatusView::SuccessReceiptId(receipt_id) => { + Self::SuccessReceiptId(*receipt_id) + } + views::ExecutionStatusView::Failure(tx_execution_error) => { + // TODO: handle the Failure and all the nested errors it has instead of stringifying + Self::Failure(tx_execution_error.to_string()) + } + } + } +} diff --git a/lake-primitives/src/types/state_changes.rs b/lake-primitives/src/types/state_changes.rs new file mode 100644 index 0000000..6e17231 --- /dev/null +++ b/lake-primitives/src/types/state_changes.rs @@ -0,0 +1,209 @@ +use near_crypto::PublicKey; + +use crate::near_indexer_primitives::{ + types::AccountId, + views::{ + AccessKeyView, AccountView, StateChangeCauseView, StateChangeValueView, + StateChangeWithCauseView, + }, + CryptoHash, +}; + +/// Represents the changes to the state of the account. +#[derive(Debug, Clone)] +pub struct StateChange { + affected_account_id: AccountId, + cause: StateChangeCause, + value: StateChangeValue, +} + +impl StateChange { + /// Returns the [AccountId] of the account that was affected by the state change. + pub fn affected_account_id(&self) -> AccountId { + self.affected_account_id.clone() + } + + /// Returns the [StateChangeCause] of the state change. + pub fn cause(&self) -> StateChangeCause { + self.cause.clone() + } + + /// Returns the [StateChangeValue] of the state change. + pub fn value(&self) -> StateChangeValue { + self.value.clone() + } +} + +impl From<&StateChangeWithCauseView> for StateChange { + fn from(state_change_with_cause_view: &StateChangeWithCauseView) -> Self { + let cause: StateChangeCause = (&state_change_with_cause_view.cause).into(); + let value: StateChangeValue = (&state_change_with_cause_view.value).into(); + Self { + affected_account_id: value.affected_account_id(), + cause, + value, + } + } +} + +#[derive(Debug, Clone)] +pub enum StateChangeCause { + NotWritableToDisk, + InitialState, + TransactionProcessing { tx_hash: CryptoHash }, + ActionReceiptProcessingStarted { receipt_hash: CryptoHash }, + ActionReceiptGasReward { receipt_hash: CryptoHash }, + ReceiptProcessing { receipt_hash: CryptoHash }, + PostponedReceipt { receipt_hash: CryptoHash }, + UpdatedDelayedReceipts, + ValidatorAccountsUpdate, + Migration, + Resharding, +} + +impl From<&StateChangeCauseView> for StateChangeCause { + fn from(state_change_cause: &StateChangeCauseView) -> Self { + match state_change_cause { + StateChangeCauseView::NotWritableToDisk => Self::NotWritableToDisk, + StateChangeCauseView::InitialState => Self::InitialState, + StateChangeCauseView::TransactionProcessing { tx_hash } => { + Self::TransactionProcessing { tx_hash: *tx_hash } + } + StateChangeCauseView::ActionReceiptProcessingStarted { receipt_hash } => { + Self::ActionReceiptProcessingStarted { + receipt_hash: *receipt_hash, + } + } + StateChangeCauseView::ActionReceiptGasReward { receipt_hash } => { + Self::ActionReceiptGasReward { + receipt_hash: *receipt_hash, + } + } + StateChangeCauseView::ReceiptProcessing { receipt_hash } => Self::ReceiptProcessing { + receipt_hash: *receipt_hash, + }, + StateChangeCauseView::PostponedReceipt { receipt_hash } => Self::PostponedReceipt { + receipt_hash: *receipt_hash, + }, + StateChangeCauseView::UpdatedDelayedReceipts => Self::UpdatedDelayedReceipts, + StateChangeCauseView::ValidatorAccountsUpdate => Self::ValidatorAccountsUpdate, + StateChangeCauseView::Migration => Self::Migration, + StateChangeCauseView::Resharding => Self::Resharding, + } + } +} + +#[derive(Debug, Clone)] +pub enum StateChangeValue { + AccountUpdate { + account_id: AccountId, + account: AccountView, + }, + AccountDeletion { + account_id: AccountId, + }, + AccessKeyUpdate { + account_id: AccountId, + public_key: PublicKey, + access_key: AccessKeyView, + }, + AccessKeyDeletion { + account_id: AccountId, + public_key: PublicKey, + }, + DataUpdate { + account_id: AccountId, + key: Vec, + value: Vec, + }, + DataDeletion { + account_id: AccountId, + key: Vec, + }, + ContractCodeUpdate { + account_id: AccountId, + code: Vec, + }, + ContractCodeDeletion { + account_id: AccountId, + }, +} + +impl StateChangeValue { + pub fn affected_account_id(&self) -> AccountId { + match self { + Self::AccountUpdate { account_id, .. } => account_id.clone(), + Self::AccountDeletion { account_id } => account_id.clone(), + Self::AccessKeyUpdate { account_id, .. } => account_id.clone(), + Self::AccessKeyDeletion { account_id, .. } => account_id.clone(), + Self::DataUpdate { account_id, .. } => account_id.clone(), + Self::DataDeletion { account_id, .. } => account_id.clone(), + Self::ContractCodeUpdate { account_id, .. } => account_id.clone(), + Self::ContractCodeDeletion { account_id } => account_id.clone(), + } + } +} + +impl From<&StateChangeValueView> for StateChangeValue { + fn from(state_change_value: &StateChangeValueView) -> Self { + match state_change_value { + StateChangeValueView::AccountUpdate { + account_id, + account, + } => Self::AccountUpdate { + account_id: account_id.clone(), + account: account.clone(), + }, + StateChangeValueView::AccountDeletion { account_id } => Self::AccountDeletion { + account_id: account_id.clone(), + }, + StateChangeValueView::AccessKeyUpdate { + account_id, + public_key, + access_key, + } => Self::AccessKeyUpdate { + account_id: account_id.clone(), + public_key: public_key.clone(), + access_key: access_key.clone(), + }, + StateChangeValueView::AccessKeyDeletion { + account_id, + public_key, + } => Self::AccessKeyDeletion { + account_id: account_id.clone(), + public_key: public_key.clone(), + }, + StateChangeValueView::DataUpdate { + account_id, + key, + value, + } => { + let key: &[u8] = key.as_ref(); + let value: &[u8] = value.as_ref(); + Self::DataUpdate { + account_id: account_id.clone(), + key: key.to_vec(), + value: value.to_vec(), + } + } + StateChangeValueView::DataDeletion { account_id, key } => { + let key: &[u8] = key.as_ref(); + Self::DataDeletion { + account_id: account_id.clone(), + key: key.to_vec(), + } + } + StateChangeValueView::ContractCodeUpdate { account_id, code } => { + Self::ContractCodeUpdate { + account_id: account_id.clone(), + code: code.clone(), + } + } + StateChangeValueView::ContractCodeDeletion { account_id } => { + Self::ContractCodeDeletion { + account_id: account_id.clone(), + } + } + } + } +} diff --git a/lake-primitives/src/types/transactions.rs b/lake-primitives/src/types/transactions.rs new file mode 100644 index 0000000..c453da1 --- /dev/null +++ b/lake-primitives/src/types/transactions.rs @@ -0,0 +1,86 @@ +use near_crypto::{PublicKey, Signature}; + +use super::receipts::ExecutionStatus; +use crate::near_indexer_primitives::{types::AccountId, CryptoHash, IndexerTransactionWithOutcome}; + +/// High-level representation of the `Transaction`. +/// +/// The structure basically combines the `Transaction` itself and the corresponding `ExecutionOutcome`. +/// **Reminder**: the result of the transaction execution is always a [Receipt](super::receipts::Receipt) +/// that looks pretty much like the `Transaction` itself. +/// +/// #### Important notes on the Transaction +/// +/// Transaction's `actions` are represented by the [Action](super::actions::Action) enum. Actions are +/// included for the informational purpose to help developers to know what exactly should happen after the +/// `Transaction` is executed. +#[derive(Debug, Clone)] +pub struct Transaction { + transaction_hash: CryptoHash, + signer_id: AccountId, + signer_public_key: PublicKey, + signature: Signature, + receiver_id: AccountId, + status: ExecutionStatus, + execution_outcome_id: CryptoHash, + actions: Vec, +} + +impl Transaction { + /// Returns the [CryptoHash] hash of the transaction. + pub fn transaction_hash(&self) -> CryptoHash { + self.transaction_hash + } + + /// Returns the [AccountId] of the signer of the transaction. + pub fn signer_id(&self) -> &AccountId { + &self.signer_id + } + + /// Returns the [PublicKey] of the signer of the transaction. + pub fn signer_public_key(&self) -> &PublicKey { + &self.signer_public_key + } + + /// Returns the [Signature] of the transaction. + pub fn signature(&self) -> &Signature { + &self.signature + } + + /// Returns the [AccountId] of the receiver of the transaction. + pub fn receiver_id(&self) -> &AccountId { + &self.receiver_id + } + + /// Returns the [ExecutionStatus] of the corresponding ExecutionOutcome. + pub fn status(&self) -> &ExecutionStatus { + &self.status + } + + /// Returns the [CryptoHash] id of the corresponding ExecutionOutcome. + pub fn execution_outcome_id(&self) -> CryptoHash { + self.execution_outcome_id + } + + /// Returns the [Action](super::actions::Action) of the transaction. + pub fn actions_included(&self) -> impl Iterator { + self.actions.iter() + } +} + +impl TryFrom<&IndexerTransactionWithOutcome> for Transaction { + type Error = &'static str; + + fn try_from(tx_with_outcome: &IndexerTransactionWithOutcome) -> Result { + Ok(Self { + transaction_hash: tx_with_outcome.transaction.hash, + signer_id: tx_with_outcome.transaction.signer_id.clone(), + signer_public_key: tx_with_outcome.transaction.public_key.clone(), + signature: tx_with_outcome.transaction.signature.clone(), + receiver_id: tx_with_outcome.transaction.receiver_id.clone(), + execution_outcome_id: tx_with_outcome.outcome.execution_outcome.id, + status: (&tx_with_outcome.outcome.execution_outcome.outcome.status).into(), + actions: super::actions::Action::try_vec_from_transaction_outcome(tx_with_outcome)?, + }) + } +} diff --git a/near-lake-framework/src/s3_fetchers.rs b/near-lake-framework/src/s3_fetchers.rs new file mode 100644 index 0000000..6bbac1e --- /dev/null +++ b/near-lake-framework/src/s3_fetchers.rs @@ -0,0 +1,357 @@ +use async_trait::async_trait; +use std::str::FromStr; + +use aws_sdk_s3::output::{GetObjectOutput, ListObjectsV2Output}; + +#[async_trait] +pub trait S3Client { + async fn get_object( + &self, + bucket: &str, + prefix: &str, + ) -> Result>; + + async fn list_objects( + &self, + bucket: &str, + start_after: &str, + ) -> Result< + ListObjectsV2Output, + aws_sdk_s3::types::SdkError, + >; +} + +#[derive(Clone, Debug)] +pub struct LakeS3Client { + s3: aws_sdk_s3::Client, +} + +impl LakeS3Client { + pub fn new(s3: aws_sdk_s3::Client) -> Self { + Self { s3 } + } +} + +#[async_trait] +impl S3Client for LakeS3Client { + async fn get_object( + &self, + bucket: &str, + prefix: &str, + ) -> Result> + { + Ok(self + .s3 + .get_object() + .bucket(bucket) + .key(prefix) + .request_payer(aws_sdk_s3::model::RequestPayer::Requester) + .send() + .await?) + } + + async fn list_objects( + &self, + bucket: &str, + start_after: &str, + ) -> Result< + ListObjectsV2Output, + aws_sdk_s3::types::SdkError, + > { + Ok(self + .s3 + .list_objects_v2() + .max_keys(1000) // 1000 is the default and max value for this parameter + .delimiter("/".to_string()) + .start_after(start_after) + .request_payer(aws_sdk_s3::model::RequestPayer::Requester) + .bucket(bucket) + .send() + .await?) + } +} + +/// Queries the list of the objects in the bucket, grouped by "/" delimiter. +/// Returns the list of block heights that can be fetched +pub(crate) async fn list_block_heights( + lake_s3_client: &impl S3Client, + s3_bucket_name: &str, + start_from_block_height: crate::types::BlockHeight, +) -> Result< + Vec, + crate::types::LakeError, +> { + tracing::debug!( + target: crate::LAKE_FRAMEWORK, + "Fetching block heights from S3, after #{}...", + start_from_block_height + ); + let response = lake_s3_client + .list_objects(s3_bucket_name, &format!("{:0>12}", start_from_block_height)) + .await?; + + Ok(match response.common_prefixes { + None => vec![], + Some(common_prefixes) => common_prefixes + .into_iter() + .filter_map(|common_prefix| common_prefix.prefix) + .collect::>() + .into_iter() + .filter_map(|prefix_string| { + prefix_string + .split('/') + .next() + .map(u64::from_str) + .and_then(|num| num.ok()) + }) + .collect(), + }) +} + +/// By the given block height gets the objects: +/// - block.json +/// - shard_N.json +/// Reads the content of the objects and parses as a JSON. +/// Returns the result in `near_indexer_primitives::StreamerMessage` +pub(crate) async fn fetch_streamer_message( + lake_s3_client: &impl S3Client, + s3_bucket_name: &str, + block_height: crate::types::BlockHeight, +<<<<<<< HEAD:src/s3_fetchers.rs +) -> Result< + near_indexer_primitives::StreamerMessage, + crate::types::LakeError, +> { +======= +) -> anyhow::Result { +>>>>>>> 8bcd5c5 (feat: NEAR Lake Helper (high-level Lake Framework) (#51)):near-lake-framework/src/s3_fetchers.rs + let block_view = { + let body_bytes = loop { + match lake_s3_client + .get_object(s3_bucket_name, &format!("{:0>12}/block.json", block_height)) + .await + { + Ok(response) => { + match response.body.collect().await { + Ok(bytes_stream) => break bytes_stream.into_bytes(), + Err(err) => { + tracing::debug!( + target: crate::LAKE_FRAMEWORK, + "Failed to read bytes from the block #{:0>12} response. Retrying immediately.\n{:#?}", + block_height, + err, + ); + } + }; + } + Err(err) => { + tracing::debug!( + target: crate::LAKE_FRAMEWORK, + "Failed to get {:0>12}/block.json. Retrying immediately\n{:#?}", + block_height, + err + ); + } + }; + }; + +<<<<<<< HEAD:src/s3_fetchers.rs + serde_json::from_slice::(body_bytes.as_ref())? + }; + + let fetch_shards_futures = (0..block_view.chunks.len() as u64) +======= + let body_bytes = response.body.collect().await?.into_bytes(); + + serde_json::from_slice::( + body_bytes.as_ref(), + )? + }; + + let shards: Vec = (0..block_view.chunks.len() + as u64) +>>>>>>> 8bcd5c5 (feat: NEAR Lake Helper (high-level Lake Framework) (#51)):near-lake-framework/src/s3_fetchers.rs + .collect::>() + .into_iter() + .map(|shard_id| { + fetch_shard_or_retry(lake_s3_client, s3_bucket_name, block_height, shard_id) + }); + + let shards = futures::future::try_join_all(fetch_shards_futures).await?; + + Ok(crate::near_indexer_primitives::StreamerMessage { + block: block_view, + shards, + }) +} + +/// Fetches the shard data JSON from AWS S3 and returns the `IndexerShard` +async fn fetch_shard_or_retry( + lake_s3_client: &impl S3Client, + s3_bucket_name: &str, + block_height: crate::types::BlockHeight, + shard_id: u64, +<<<<<<< HEAD:src/s3_fetchers.rs +) -> Result< + near_indexer_primitives::IndexerShard, + crate::types::LakeError, +> { + let body_bytes = loop { + match lake_s3_client + .get_object( + s3_bucket_name, + &format!("{:0>12}/shard_{}.json", block_height, shard_id), + ) +======= +) -> crate::near_indexer_primitives::IndexerShard { + loop { + match s3_client + .get_object() + .bucket(s3_bucket_name) + .key(format!("{:0>12}/shard_{}.json", block_height, shard_id)) + .request_payer(aws_sdk_s3::model::RequestPayer::Requester) + .send() +>>>>>>> 8bcd5c5 (feat: NEAR Lake Helper (high-level Lake Framework) (#51)):near-lake-framework/src/s3_fetchers.rs + .await + { + Ok(response) => { + let body_bytes = match response.body.collect().await { + Ok(body) => body.into_bytes(), + Err(err) => { + tracing::debug!( + target: crate::LAKE_FRAMEWORK, + "Failed to read the {:0>12}/shard_{}.json. Retrying in 1s...\n {:#?}", + block_height, + shard_id, + err, + ); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + continue; + } + }; + +<<<<<<< HEAD:src/s3_fetchers.rs + break body_bytes; +======= + let indexer_shard = match serde_json::from_slice::< + crate::near_indexer_primitives::IndexerShard, + >(body_bytes.as_ref()) + { + Ok(indexer_shard) => indexer_shard, + Err(err) => { + tracing::debug!( + target: crate::LAKE_FRAMEWORK, + "Failed to parse the {:0>12}/shard_{}.json. Retrying in 1s...\n {:#?}", + block_height, + shard_id, + err, + ); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + continue; + } + }; + + break indexer_shard; +>>>>>>> 8bcd5c5 (feat: NEAR Lake Helper (high-level Lake Framework) (#51)):near-lake-framework/src/s3_fetchers.rs + } + Err(err) => { + tracing::debug!( + target: crate::LAKE_FRAMEWORK, + "Failed to fetch shard #{}, retrying immediately\n{:#?}", + shard_id, + err + ); + } + } + }; + + Ok(serde_json::from_slice::< + near_indexer_primitives::IndexerShard, + >(body_bytes.as_ref())?) +} + +#[cfg(test)] +mod test { + use super::*; + + use async_trait::async_trait; + + use aws_sdk_s3::output::{get_object_output, list_objects_v2_output}; + use aws_sdk_s3::types::ByteStream; + + use aws_smithy_http::body::SdkBody; + + #[derive(Clone, Debug)] + pub struct LakeS3Client {} + + #[async_trait] + impl S3Client for LakeS3Client { + async fn get_object( + &self, + _bucket: &str, + prefix: &str, + ) -> Result> + { + let path = format!("{}/blocks/{}", env!("CARGO_MANIFEST_DIR"), prefix); + let file_bytes = tokio::fs::read(path).await.unwrap(); + let stream = ByteStream::new(SdkBody::from(file_bytes)); + Ok(get_object_output::Builder::default().body(stream).build()) + } + + async fn list_objects( + &self, + _bucket: &str, + _start_after: &str, + ) -> Result< + ListObjectsV2Output, + aws_sdk_s3::types::SdkError, + > { + Ok(list_objects_v2_output::Builder::default().build()) + } + } + + #[tokio::test] + async fn deserializes_meta_transactions() { + let lake_client = LakeS3Client {}; + + let streamer_message = + fetch_streamer_message(&lake_client, "near-lake-data-mainnet", 879765) + .await + .unwrap(); + + let delegate_action = &streamer_message.shards[0] + .chunk + .as_ref() + .unwrap() + .transactions[0] + .transaction + .actions[0]; + + assert_eq!( + serde_json::to_value(delegate_action).unwrap(), + serde_json::json!({ + "Delegate": { + "delegate_action": { + "sender_id": "test.near", + "receiver_id": "test.near", + "actions": [ + { + "AddKey": { + "public_key": "ed25519:CnQMksXTTtn81WdDujsEMQgKUMkFvDJaAjDeDLTxVrsg", + "access_key": { + "nonce": 0, + "permission": "FullAccess" + } + } + } + ], + "nonce": 879546, + "max_block_height": 100, + "public_key": "ed25519:8Rn4FJeeRYcrLbcrAQNFVgvbZ2FCEQjgydbXwqBwF1ib" + }, + "signature": "ed25519:25uGrsJNU3fVgUpPad3rGJRy2XQum8gJxLRjKFCbd7gymXwUxQ9r3tuyBCD6To7SX5oSJ2ScJZejwqK1ju8WdZfS" + } + }) + ); + } +} diff --git a/release-plz.toml b/release-plz.toml new file mode 100644 index 0000000..dba96f1 --- /dev/null +++ b/release-plz.toml @@ -0,0 +1,8 @@ +[workspace] +# Use `near-lake-framework-rs` crate CHANGELOG as top-level one +changelog_update = false + +[[package]] +name = "lake-framework" +changelog_update = true +changelog_path = "./CHANGELOG.md" diff --git a/rust-toolchain b/rust-toolchain new file mode 100644 index 0000000..4934985 --- /dev/null +++ b/rust-toolchain @@ -0,0 +1 @@ +1.69.0 diff --git a/src/types.rs b/src/types.rs deleted file mode 100644 index 6d64e00..0000000 --- a/src/types.rs +++ /dev/null @@ -1,133 +0,0 @@ -/// Type alias represents the block height -pub type BlockHeight = u64; - -/// Configuration struct for NEAR Lake Framework -/// NB! Consider using [`LakeConfigBuilder`] -/// Building the `LakeConfig` example: -/// ``` -/// use near_lake_framework::LakeConfigBuilder; -/// -/// # async fn main() { -/// let config = LakeConfigBuilder::default() -/// .testnet() -/// .start_block_height(82422587) -/// .build() -/// .expect("Failed to build LakeConfig"); -/// # } -/// ``` -#[derive(Default, Builder, Debug)] -#[builder(pattern = "owned")] -pub struct LakeConfig { - /// AWS S3 Bucket name - #[builder(setter(into))] - pub(crate) s3_bucket_name: String, - /// AWS S3 Region name - #[builder(setter(into))] - pub(crate) s3_region_name: String, - /// Defines the block height to start indexing from - pub(crate) start_block_height: u64, - /// Custom aws_sdk_s3::config::Config - /// ## Use-case: custom endpoint - /// You might want to stream data from the custom S3-compatible source () . In order to do that you'd need to pass `aws_sdk_s3::config::Config` configured - /// ``` - /// use aws_sdk_s3::Endpoint; - /// use http::Uri; - /// use near_lake_framework::LakeConfigBuilder; - /// - /// # async fn main() { - /// let aws_config = aws_config::from_env().load().await; - /// let mut s3_conf = aws_sdk_s3::config::Builder::from(&aws_config); - /// s3_conf = s3_conf - /// .endpoint_resolver( - /// Endpoint::immutable("http://0.0.0.0:9000".parse::().unwrap())) - /// .build(); - /// - /// let config = LakeConfigBuilder::default() - /// .s3_config(s3_conf) - /// .s3_bucket_name("near-lake-data-custom") - /// .start_block_height(1) - /// .build() - /// .expect("Failed to build LakeConfig"); - /// # } - /// ``` - #[builder(setter(strip_option), default)] - pub(crate) s3_config: Option, - #[builder(default = "100")] - pub(crate) blocks_preload_pool_size: usize, -} - -impl LakeConfigBuilder { - /// Shortcut to set up [LakeConfigBuilder::s3_bucket_name] for mainnet - /// ``` - /// use near_lake_framework::LakeConfigBuilder; - /// - /// # async fn main() { - /// let config = LakeConfigBuilder::default() - /// .mainnet() - /// .start_block_height(65231161) - /// .build() - /// .expect("Failed to build LakeConfig"); - /// # } - /// ``` - pub fn mainnet(mut self) -> Self { - self.s3_bucket_name = Some("near-lake-data-mainnet".to_string()); - self.s3_region_name = Some("eu-central-1".to_string()); - self - } - - /// Shortcut to set up [LakeConfigBuilder::s3_bucket_name] for testnet - /// ``` - /// use near_lake_framework::LakeConfigBuilder; - /// - /// # async fn main() { - /// let config = LakeConfigBuilder::default() - /// .testnet() - /// .start_block_height(82422587) - /// .build() - /// .expect("Failed to build LakeConfig"); - /// # } - /// ``` - pub fn testnet(mut self) -> Self { - self.s3_bucket_name = Some("near-lake-data-testnet".to_string()); - self.s3_region_name = Some("eu-central-1".to_string()); - self - } - - /// Shortcut to set up [LakeConfigBuilder::s3_bucket_name] for betanet - /// ``` - /// use near_lake_framework::LakeConfigBuilder; - /// - /// # async fn main() { - /// let config = LakeConfigBuilder::default() - /// .betanet() - /// .start_block_height(82422587) - /// .build() - /// .expect("Failed to build LakeConfig"); - /// # } - /// ``` - pub fn betanet(mut self) -> Self { - self.s3_bucket_name = Some("near-lake-data-betanet".to_string()); - self.s3_region_name = Some("us-east-1".to_string()); - self - } -} - -#[allow(clippy::enum_variant_names)] -#[derive(thiserror::Error, Debug)] -pub enum LakeError { - #[error("Failed to parse structure from JSON: {error_message}")] - ParseError { - #[from] - error_message: serde_json::Error, - }, - #[error("AWS S3 error")] - AwsError { - #[from] - error: aws_sdk_s3::types::SdkError, - }, - #[error("Failed to convert integer")] - IntConversionError { - #[from] - error: std::num::TryFromIntError, - }, -}