Merge branch 'main' into transformers/add-null-ignoring-agg-functions

Makefile

@@ -18,6 +18,7 @@ build:
 	docker compose build transformer
 
 extract:
+	docker compose run extractors python main.py configs/eth_mainnet.yaml
 	docker compose run extractors python main.py configs/base_mainnet.yaml
 	docker compose run extractors python main.py configs/base_sepolia.yaml
 	docker compose run extractors python main.py configs/arbitrum_mainnet.yaml

dashboard/About.py

@@ -62,6 +62,7 @@
 ]
 mainnet_pages = [
     Page(f"{PAGE_PREFIX}pages/All_Chains.py", "All Chains"),
+    Page(f"{PAGE_PREFIX}pages/Ethereum_Mainnet.py", "Ethereum"),
     Page(f"{PAGE_PREFIX}pages/Base_Mainnet.py", "Base"),
     Page(f"{PAGE_PREFIX}pages/Arbitrum_Mainnet.py", "Arbitrum"),
     Page(f"{PAGE_PREFIX}pages/Optimism_Mainnet.py", "Optimism"),

dashboard/modules/eth_mainnet/core_stats.py

@@ -0,0 +1,217 @@
+import streamlit as st
+import pandas as pd
+from datetime import datetime, timedelta
+from utils import get_connection
+from utils import chart_bars, chart_lines, export_data
+
+## set default filters
+filters = {
+    "start_date": datetime.today().date() - timedelta(days=14),
+    "end_date": datetime.today().date() + timedelta(days=1),
+    "resolution": "24h",
+}
+
+
+## data
+@st.cache_data(ttl=600)
+def fetch_data(filters):
+    # get filters
+    start_date = filters["start_date"]
+    end_date = filters["end_date"]
+    resolution = filters["resolution"]
+
+    # initialize connection
+    db = get_connection()
+
+    # get account data
+    df_account_delegation = pd.read_sql_query(
+        f"""
+        SELECT * FROM prod_eth_mainnet.fct_core_account_delegation_eth_mainnet
+        WHERE ts >= '{start_date}' and ts <= '{end_date}'
+    """,
+        db,
+    )
+
+    df_apr = pd.read_sql_query(
+        f"""
+        SELECT 
+            ts,
+            coalesce(tk.token_symbol, collateral_type) as collateral_type,
+            collateral_value,
+            debt,
+            hourly_pnl,
+            rewards_usd,
+            hourly_issuance,
+            cumulative_issuance,
+            cumulative_pnl,
+            apr_{resolution} as apr,
+            apr_{resolution}_pnl as apr_pnl,
+            apr_{resolution}_rewards as apr_rewards
+        FROM prod_eth_mainnet.fct_core_apr_eth_mainnet apr
+        LEFT JOIN prod_seeds.eth_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
+        WHERE ts >= '{start_date}' and ts <= '{end_date}'
+        and pool_id = 1
+        ORDER BY ts
+    """,
+        db,
+    )
+
+    df_apr_token = pd.read_sql_query(
+        f"""
+        SELECT 
+            ts,
+            coalesce(tk.token_symbol, collateral_type) as collateral_type,
+            apr.reward_token,
+            concat(coalesce(tk.token_symbol, collateral_type), ' : ', apr.reward_token) as token_pair,
+            collateral_value,
+            rewards_usd,
+            apr_{resolution}_rewards as apr_rewards
+        FROM prod_eth_mainnet.fct_core_apr_rewards_eth_mainnet apr
+        LEFT JOIN prod_seeds.eth_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
+        WHERE ts >= '{start_date}' and ts <= '{end_date}'
+        and pool_id = 1
+        and apr.reward_token is not null
+        ORDER BY ts
+    """,
+        db,
+    )
+
+    db.close()
+
+    return {
+        "account_delegation": df_account_delegation,
+        "apr": df_apr,
+        "apr_token": df_apr_token,
+    }
+
+
+def make_charts(data, filters):
+    resolution = filters["resolution"]
+    return {
+        "tvl": chart_lines(
+            data["apr"],
+            "ts",
+            ["collateral_value"],
+            "TVL",
+            "collateral_type",
+        ),
+        "debt": chart_lines(
+            data["apr"],
+            "ts",
+            ["debt"],
+            "Debt",
+            "collateral_type",
+        ),
+        "hourly_issuance": chart_bars(
+            data["apr"],
+            "ts",
+            ["hourly_issuance"],
+            "Hourly Issuance",
+            "collateral_type",
+        ),
+        "issuance": chart_lines(
+            data["apr"],
+            "ts",
+            ["cumulative_issuance"],
+            "Issuance",
+            "collateral_type",
+        ),
+        "pnl": chart_lines(
+            data["apr"],
+            "ts",
+            ["cumulative_pnl"],
+            "Pnl",
+            "collateral_type",
+        ),
+        "hourly_pnl": chart_bars(
+            data["apr"],
+            "ts",
+            ["hourly_pnl"],
+            "Hourly Pnl",
+            "collateral_type",
+        ),
+        "hourly_rewards": chart_bars(
+            data["apr"],
+            "ts",
+            ["rewards_usd"],
+            "Hourly Rewards",
+            "collateral_type",
+        ),
+        "hourly_rewards_token": chart_bars(
+            data["apr_token"],
+            "ts",
+            ["rewards_usd"],
+            "Hourly Rewards (Collateral : Reward)",
+            "token_pair",
+        ),
+        "apr": chart_lines(
+            data["apr"],
+            "ts",
+            "apr",
+            f"APR - {resolution} average",
+            y_format="%",
+            color="collateral_type",
+        ),
+        "apr_token": chart_lines(
+            data["apr_token"],
+            "ts",
+            "apr_rewards",
+            f"Reward APR by Token - {resolution} average",
+            y_format="%",
+            color="token_pair",
+        ),
+    }
+
+
+def main():
+    ## title
+    st.markdown("## V3 Core")
+
+    ## inputs
+    with st.expander("Filters"):
+        filters["resolution"] = st.radio(
+            "Resolution",
+            ["28d", "7d", "24h"],
+        )
+
+        filt_col1, filt_col2 = st.columns(2)
+        with filt_col1:
+            filters["start_date"] = st.date_input("Start", filters["start_date"])
+
+        with filt_col2:
+            filters["end_date"] = st.date_input("End", filters["end_date"])
+
+    data = fetch_data(filters)
+
+    ## make the charts
+    charts = make_charts(data, filters)
+
+    ## display
+    st.plotly_chart(charts["apr"], use_container_width=True)
+
+    col1, col2 = st.columns(2)
+    with col1:
+        st.plotly_chart(charts["tvl"], use_container_width=True)
+        st.plotly_chart(charts["hourly_pnl"], use_container_width=True)
+        st.plotly_chart(charts["hourly_issuance"], use_container_width=True)
+        st.plotly_chart(charts["hourly_rewards"], use_container_width=True)
+        st.plotly_chart(charts["apr_token"], use_container_width=True)
+
+    with col2:
+        st.plotly_chart(charts["debt"], use_container_width=True)
+        st.plotly_chart(charts["pnl"], use_container_width=True)
+        st.plotly_chart(charts["issuance"], use_container_width=True)
+        st.plotly_chart(charts["hourly_rewards_token"], use_container_width=True)
+
+    st.markdown("## Top Delegators")
+    st.dataframe(
+        data["account_delegation"]
+        .sort_values("amount_delegated", ascending=False)
+        .head(25)
+    )
+
+    ## export
+    exports = [{"title": export, "df": data[export]} for export in data.keys()]
+    with st.expander("Exports"):
+        for export in exports:
+            export_data(export["title"], export["df"])

dashboard/pages/Ethereum_Mainnet.py

@@ -0,0 +1,18 @@
+import streamlit as st
+from modules.eth_mainnet import (
+    core_stats,
+)
+
+st.set_page_config(page_title="Ethereum Mainnet", layout="wide")
+
+
+pages = {
+    "Core Stats": core_stats.main,
+}
+state_page = None
+state_page = st.sidebar.radio(
+    "Ethereum Mainnet",
+    tuple(pages.keys()),
+    index=tuple(pages.keys()).index(state_page) if state_page else 0,
+)
+pages[state_page]()

docker-compose.yml

@@ -27,6 +27,22 @@ services:
           cpus: "4.0"
           memory: 8192M
 
+  eth-mainnet-processor:
+    build:
+      context: ./indexers/eth-mainnet
+    networks:
+      - data
+    depends_on:
+      - db
+    restart: always
+    environment:
+      DB_HOST: db
+      DB_PORT: 5432
+      DB_NAME: eth_mainnet
+      DB_PASS: $PG_PASSWORD
+      GQL_PORT: 4350
+      RPC_ENDPOINT: https://ethereum-rpc.publicnode.com
+
   base-mainnet-processor:
     build:
       context: ./indexers/base-mainnet

extractors/configs/eth_mainnet.yaml

@@ -0,0 +1,25 @@
+network_id: 1
+
+blocks:
+  min_block: "20000000"
+  requests_per_second: 25
+  block_increment: 150
+
+eth_calls:
+  - contract_name: "CoreProxy"
+    package_name: "system"
+    function_name: "getVaultCollateral"
+    inputs:
+      - [1, "0xC011a73ee8576Fb46F5E1c5751cA3B9Fe0af2a6F"]
+    min_block: "20000000"
+    requests_per_second: 25
+    block_increment: 150
+
+  - contract_name: "CoreProxy"
+    package_name: "system"
+    function_name: "getVaultDebt"
+    inputs:
+      - [1, "0xC011a73ee8576Fb46F5E1c5751cA3B9Fe0af2a6F"]
+    min_block: "20000000"
+    requests_per_second: 25
+    block_increment: 150

extractors/src/constants.py

@@ -5,6 +5,16 @@
 
 
 CHAIN_CONFIGS = {
+    1: {
+        "name": "eth_mainnet",
+        "rpc": os.getenv("NETWORK_1_RPC"),
+        "network_id": 1,
+        "cannon_config": {
+            "package": "synthetix-omnibus",
+            "version": "latest",
+            "preset": "main"
+        }
+    },
     10: {
         "name": "optimism_mainnet",
         "rpc": os.getenv("NETWORK_10_RPC"),

indexers/eth-mainnet/.dockerignore

@@ -0,0 +1,5 @@
+/.git
+/node_modules
+/lib
+/*Versions.json
+.env

indexers/eth-mainnet/Dockerfile

@@ -0,0 +1,14 @@
+FROM node:16
+
+WORKDIR /app
+
+COPY package*.json ./
+
+RUN npm ci
+
+COPY . .
+
+RUN npm run generate:processor
+RUN npm run build
+
+CMD npm run generate:migration ; npm run start