feat: phantom (#64)

* docs: instructions for how to work locally with CLI

* commit example repo

* feat: support for phantom events

* run in prod

* fix: last path fix to inherit from dev computer

* docs: update readme for phantom

* docs: phantom docs

* docs: reference etherscan API keys

* fix: create API key to post overlay

* fix: resolve to the correct out after compile

* docs: update phantom in places missed

* feat: WIP shadow

* Write serialization logic from Forge output JSON to ethers_solc::CompilerOutput

* Ensure that source files are properly parsed

* serde::rename DeployShadowResponse

* fix: resolve issue with no inputs in events syntax error for postgres

* feat: phantom last touches

* docs: update changelog.mdx

* fix: better error message when etherscan is not supported for network

* refactor: use --contract-name and --network in the cli for phantom

* docs: fix contract mapping without string

* fix: some regressions

* fix: some regressions

* fix: abi mapping

* fix: inject shadow API key into jsonrpc provider

* fix: do not inject http headers into createClient

* fix: oops swap

* fix: provider expose RetryClientError

* fix: remove odd syncing log

---------

Co-authored-by: Alexander <alex@deekerno.com>

.gitignore

@@ -21,6 +21,7 @@ Cargo.lock
 cli/rindexer.yaml
 cli/.env
 examples/*
+!examples/rindexer_demo_cli
 documentation/node_modules
 documentation/dist
 documentation/docs/dist

README.md

@@ -1,6 +1,6 @@
 # 🦀 rindexer 🦀
 
-Note rindexer is brand new and only in beta and actively under development, things will change and bugs will exist - if you find any bugs or have any
+Note rindexer is brand new and actively under development, things will change and bugs will exist - if you find any bugs or have any
 feature requests please open an issue on [github](https://github.com/joshstevens19/rindexer/issues).
 
 rindexer is an opensource powerful, high-speed indexing toolset developed in Rust, designed for compatibility with any EVM chain.
@@ -39,6 +39,7 @@ Commands:
   add           Add elements such as contracts to the rindexer.yaml file
   codegen       Generates rust code based on rindexer.yaml or graphql queries
   delete        Delete data from the postgres database or csv files
+  phantom       Use phantom events to add your own events to contracts
   help          Print this message or the help of the given subcommand(s)
 
 Options:
@@ -111,6 +112,13 @@ you can run `cargo fmt` to format the code, rules have been mapped in the `rustf
 Anyone is welcome to contribute to rindexer, feel free to look over the issues or open a new one if you have
 any new ideas or bugs you have found.
 
+### Playing around with the CLI locally
+
+You can use the `make` commands to run the CLI commands locally, this is useful for testing and developing.
+These are located in the `cli` folder > `Makefile`. It uses `CURDIR` to resolve the paths for you, so they should work
+out of the box. The examples repo has a `rindexer_demo_cli` folder which you can modify (please do not commit any changes though) 
+or spin up a new no-code project using the make commands.
+
 ## Release
 
 To release a new rindexer you have to do a few things:

cli/Cargo.toml

@@ -15,6 +15,8 @@ clap = { version = "4.4.11", features = ["derive"] }
 regex = "1.5.4"
 colored = "2.0"
 tokio = "1.35.1"
+serde = { version = "1.0.204", features = ["derive"] }
+serde_json = "1.0.120"
 
 # build
 jemallocator = { version = "0.5.0", optional = true }

cli/Makefile

@@ -1,36 +1,43 @@
+prod_build:
+	RUSTFLAGS='-C target-cpu=native' cargo build --release --features jemalloc
 new_no_code:
-	cargo run -- new --path /Users/joshstevens/code/rindexer/examples no-code
+	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- new --path $(CURDIR)/../examples no-code
 new_rust:
-	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- new --path /Users/joshstevens/code rust
-start:
-	cargo run -- start --path /Users/joshstevens/code/rindexer/examples/rindexer_demo_cli all
-start_prod:
-	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- start all
+	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- new --path $(CURDIR)/../../ rust
 start_indexer:
-	cargo run -- start --path /Users/joshstevens/code/rindexer/examples/rindexer_demo_cli indexer
-start_indexer_base_paint:
-	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- start --path /Users/joshstevens/code/rindexer/examples/base_paint indexer
-start_graphql_base_paint:
-	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- start --path /Users/joshstevens/code/rindexer/examples/base_paint graphql
-start_indexer_prod:
-	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- start --path /Users/joshstevens/code/rindexer/examples/rindexer_demo_cli all
-start_indexer_lens_mirrors:
-	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- start --path /Users/joshstevens/code/rindexer/examples/lens_mirrors all
+	RUSTFLAGS='-C target-cpu=native' RUST_BACKTRACE='full' cargo run --release --features jemalloc -- start --path $(CURDIR)/../examples/rindexer_demo_cli indexer
+start_all:
+	RUSTFLAGS='-C target-cpu=native' RUST_BACKTRACE='full' cargo run --release --features jemalloc -- start --path $(CURDIR)/../examples/rindexer_demo_cli all
 start_graphql:
-	cargo run -- start --path /Users/joshstevens/code/rindexer/examples/rindexer_demo_cli graphql
-start_indexer_uniswap_v3_factory:
-	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- start --path /Users/joshstevens/code/rindexer/examples/uniswap_v3_factory all
-codegen:
-	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- codegen --path /Users/joshstevens/code/kami typings
+	RUSTFLAGS='-C target-cpu=native' RUST_BACKTRACE='full' cargo run --release --features jemalloc -- start --path $(CURDIR)/../examples/rindexer_demo_cli graphql
 codegen_typings:
-	cargo run -- codegen typings
+	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- codegen --path $(CURDIR)/../rindexer_rust_playground typings
 codegen_indexer:
-	cargo run -- codegen indexer
+	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- codegen --path $(CURDIR)/../rindexer_rust_playground indexer
 codegen_graphql:
-	cargo run -- codegen --path /Users/joshstevens/code/rindexer/examples/base_paint graphql --endpoint  http://0.0.0.0:5005/graphql
+	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- codegen --path $(CURDIR)/../examples/rindexer_demo_cli graphql --endpoint  http://0.0.0.0:5005/graphql
 add_contract:
-	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- add --path /Users/joshstevens/code/rindexer/examples/rindexer_demo_cli contract
+	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- add --path $(CURDIR)/../examples/rindexer_demo_cli contract
 delete:
-	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- delete --path /Users/joshstevens/code/rindexer/examples/rindexer_demo_cli
-prod_build:
-	RUSTFLAGS='-C target-cpu=native' cargo build --release --features jemalloc
+	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- delete --path $(CURDIR)/../examples/rindexer_demo_cli
+phantom_init:
+	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- phantom --path $(CURDIR)/../examples/rindexer_demo_cli init
+phantom_clone:
+	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- phantom --path $(CURDIR)/../examples/rindexer_demo_cli clone --contract-name RocketPoolETH --network ethereum
+phantom_compile:
+	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- phantom --path $(CURDIR)/../examples/rindexer_demo_cli compile --contract-name RocketPoolETH --network ethereum
+phantom_deploy:
+	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- phantom --path $(CURDIR)/../examples/rindexer_demo_cli deploy --contract-name RocketPoolETH --network ethereum
+help:
+	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- phantom --help
+
+
+################################################################################
+# LOCAL NONE CHECKED IN PROJECT COMMANDS
+################################################################################
+start_indexer_base_paint:
+	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- start --path $(CURDIR)/../examples/base_paint indexer
+start_graphql_base_paint:
+	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- start --path $(CURDIR)/../examples/base_paint graphql
+start_indexer_lens_mirrors:
+	RUSTFLAGS='-C target-cpu=native' cargo run --release --features jemalloc -- start --path $(CURDIR)/../examples/lens_mirrors all

cli/README.md

@@ -32,6 +32,7 @@ Commands:
   add           Add elements such as contracts to the rindexer.yaml file
   codegen       Generates rust code based on rindexer.yaml or graphql queries
   delete        Delete data from the postgres database or csv files
+  phantom       Use phantom events to add your own events to contracts
   help          Print this message or the help of the given subcommand(s)
 
 Options:
@@ -41,19 +42,10 @@ Options:
 
 ## Working with CLI locally
 
-The best way to work with the CLI is to use the `cargo run` command with args after it inside the CLI project, 
-for example if I wanted to create a new project I would run:
+The best way to work with the CLI is to use the `Makefile` predefined commands.
 
-```bash
-cargo run -- new --path PATH_TO_CREATE_PROJECT no-code
-```
-
-This would create a new no-code project in the path you specified.
-
-If you wanted to look at the help you can run:
+You can also run your own commands using cargo run, example below would create a new no-code project in the path you specified.
 
 ```bash
-cargo run -- help
-```
-
-This will show you all the commands available to you.
+cargo run -- new --path PATH_TO_CREATE_PROJECT no-code
+```

cli/src/cli_interface.rs

@@ -1,4 +1,4 @@
-use clap::{Parser, Subcommand};
+use clap::{Args, Parser, Subcommand};
 
 #[allow(clippy::upper_case_acronyms)]
 #[derive(Parser, Debug)]
@@ -99,6 +99,24 @@ pub enum Commands {
         #[clap(long, short)]
         path: Option<String>,
     },
+    /// Use phantom events to add your own events to contracts
+    ///
+    /// This command helps you use phantom events within rindexer.
+    ///
+    /// Example:
+    /// `rindexer phantom init` or
+    /// `rindexer phantom clone --contract-name <CONTRACT_NAME> --network <NETWORK>` or
+    /// `rindexer phantom compile --contract-name <CONTRACT_NAME> --network <NETWORK>` or
+    /// `rindexer phantom deploy --contract-name <CONTRACT_NAME> --network <NETWORK>`
+    #[clap(name = "phantom")]
+    Phantom {
+        #[clap(subcommand)]
+        subcommand: PhantomSubcommands,
+
+        /// optional - The path to create the project in, default will be where the command is run.
+        #[clap(long, short)]
+        path: Option<String>,
+    },
 }
 
 #[derive(Subcommand, Debug)]
@@ -197,3 +215,77 @@ pub enum CodegenSubcommands {
         endpoint: Option<String>,
     },
 }
+
+#[derive(Args, Debug)]
+pub struct PhantomBaseArgs {
+    /// The name of the contract
+    #[clap(value_parser)]
+    pub contract_name: String,
+
+    /// The network the contract is on
+    #[clap(value_parser)]
+    pub network: String,
+}
+
+#[derive(Subcommand, Debug)]
+pub enum PhantomSubcommands {
+    /// Sets up phantom events on rindexer
+    ///
+    /// Want to add your own custom events to contracts? This command will help you do that.
+    ///
+    /// Example:
+    /// `rindexer phantom init`
+    #[clap(name = "init")]
+    Init,
+
+    /// Clone the contract with the network you wish to add phantom events to.
+    ///
+    /// Note contract name and network are your values in your rindexer.yaml file.
+    ///
+    /// Example:
+    /// `rindexer phantom clone --contract-name <CONTRACT_NAME> --network <NETWORK>`
+    #[clap(name = "clone")]
+    Clone {
+        /// The name of the contract to clone
+        #[arg(long)]
+        contract_name: String,
+
+        /// The network
+        #[arg(long)]
+        network: String,
+    },
+
+    /// Compiles the phantom contract
+    ///
+    /// Note contract name and network are your values in your rindexer.yaml file.
+    ///
+    /// Example:
+    /// `rindexer phantom compile --contract-name <CONTRACT_NAME> --network <NETWORK>`
+    #[clap(name = "compile")]
+    Compile {
+        /// The name of the contract to clone
+        #[arg(long)]
+        contract_name: String,
+
+        /// The network
+        #[arg(long)]
+        network: String,
+    },
+
+    /// Deploy the modified phantom contract
+    ///
+    /// This will compile and update your rindexer project with the phantom events.
+    ///
+    /// Example:
+    /// `rindexer phantom deploy --contract-name <CONTRACT_NAME> --network <NETWORK>`
+    #[clap(name = "deploy")]
+    Deploy {
+        /// The name of the contract to clone
+        #[arg(long)]
+        contract_name: String,
+
+        /// The network
+        #[arg(long)]
+        network: String,
+    },
+}

cli/src/commands/add.rs

@@ -8,9 +8,9 @@ use ethers_etherscan::Client;
 use rindexer::{
     manifest::{
         contract::{Contract, ContractDetails},
-        yaml::{read_manifest, write_manifest, YAML_CONFIG_NAME},
+        yaml::{read_manifest_raw, write_manifest, YAML_CONFIG_NAME},
     },
-    write_file,
+    public_read_env_value, write_file,
 };
 
 use crate::{
@@ -28,7 +28,7 @@ pub async fn handle_add_contract_command(
 
     let rindexer_yaml_path = project_path.join(YAML_CONFIG_NAME);
 
-    let mut manifest = read_manifest(&rindexer_yaml_path).inspect_err(|e| {
+    let mut manifest = read_manifest_raw(&rindexer_yaml_path).inspect_err(|e| {
         print_error_message(&format!("Could not read the rindexer.yaml file: {}", e))
     })?;
 
@@ -62,15 +62,15 @@ pub async fn handle_add_contract_command(
         .1;
 
     let chain_network = Chain::try_from(chain_id)
-        .inspect_err(|_| print_error_message("Network is not supported by etherscan API"))?;
+        .inspect_err(|_| print_error_message("Network is not supported by etherscan API, please add the contract manually in the rindexer.yaml file"))?;
     let contract_address =
         prompt_for_input(&format!("Enter {} Contract Address", network), None, None, None);
 
-    let etherscan_api_key = manifest
-        .global
-        .as_ref()
-        .and_then(|global| global.etherscan_api_key.as_ref())
-        .map_or(BACKUP_ETHERSCAN_API_KEY, String::as_str);
+    let etherscan_api_key =
+        manifest.global.as_ref().and_then(|global| global.etherscan_api_key.as_ref()).map_or_else(
+            || BACKUP_ETHERSCAN_API_KEY.to_string(),
+            |key| public_read_env_value(key).unwrap_or_else(|_| key.to_string()),
+        );
 
     let client = Client::builder()
         .with_api_key(etherscan_api_key)

cli/src/commands/mod.rs

@@ -2,6 +2,7 @@ pub mod add;
 pub mod codegen;
 pub mod delete;
 pub mod new;
+pub mod phantom;
 pub mod start;
 
 const BACKUP_ETHERSCAN_API_KEY: &str = "DHBPB1EJ84JMSWP7C86387NK7IIRRQJVV1";

cli/src/commands/new.rs

@@ -152,6 +152,7 @@ pub fn handle_new_command(
             reorg_safe_distance: None,
             generate_csv: None,
         }],
+        phantom: None,
         global: None,
         storage: Storage {
             postgres: if postgres_enabled {