Merge pull request #101 from Synthetixio/finish-l1

Complete L1 Additions

.streamlit/secrets_example.toml

@@ -4,6 +4,7 @@ DB_USER = 'analytics'
 DB_PASS = 'analytics'
 DB_HOST = 'db'
 DB_PORT = '5432'
+DB_ENV = 'prod'
 
 [settings]
 SHOW_TESTNETS = 'false'

extractors/requirements.txt

@@ -5,4 +5,5 @@ duckdb==0.10.2
 polars-lts-cpu==1.1.0
 pandas
 numpy
-synthetix==0.1.13
+synthetix==0.1.13
+web3==6.20.2

scheduler/dags/v3_etl.py

@@ -8,6 +8,7 @@
 # environment variables
 WORKING_DIR = os.getenv("WORKING_DIR")
 NETWORK_RPCS = {
+    "eth_mainnet": "NETWORK_1_RPC",
     "base_mainnet": "NETWORK_8453_RPC",
     "base_sepolia": "NETWORK_84532_RPC",
     "arbitrum_mainnet": "NETWORK_42161_RPC",
@@ -49,7 +50,7 @@ def create_docker_operator(dag, task_id, config_file, image, command, network_en
     )
 
 
-def create_dag(network, rpc_var, target='dev'):
+def create_dag(network, rpc_var, target="dev"):
     version = f"{network}_{target}"
 
     dag = DAG(
@@ -78,10 +79,10 @@ def create_dag(network, rpc_var, target='dev'):
         config_file=None,
         image="data-transformer",
         command=f"dbt test --target {target if network != 'optimism_mainnet' else target + '-op'} --select tag:{network} --profiles-dir profiles --profile synthetix",
-        network_env_var=rpc_var
+        network_env_var=rpc_var,
     )
 
-    if target == 'prod':
+    if target == "prod":
         extract_task_id = f"extract_{version}"
         config_file = f"configs/{network}.yaml"
         extract_task = create_docker_operator(
@@ -101,5 +102,5 @@ def create_dag(network, rpc_var, target='dev'):
 
 
 for network, rpc_var in NETWORK_RPCS.items():
-    for target in ['dev', 'prod']:
+    for target in ["dev", "prod"]:
         globals()[f"v3_etl_{network}_{target}"] = create_dag(network, rpc_var, target)

transformers/synthetix/models/marts/eth/mainnet/core/fct_pool_pnl_hourly_eth_mainnet.sql

@@ -4,6 +4,7 @@
 ) }}
 
 WITH dim AS (
+
     SELECT
         generate_series(DATE_TRUNC('hour', MIN(t.ts)), DATE_TRUNC('hour', MAX(t.ts)), '1 hour' :: INTERVAL) AS ts,
         p.pool_id,
@@ -22,9 +23,9 @@ WITH dim AS (
             FROM
                 {{ ref('fct_pool_debt_eth_mainnet') }}
         ) AS p
-GROUP BY
-    p.pool_id,
-    p.collateral_type
+    GROUP BY
+        p.pool_id,
+        p.collateral_type
 ),
 issuance AS (
     SELECT
@@ -72,22 +73,14 @@ ffill AS (
         dim.ts,
         dim.pool_id,
         dim.collateral_type,
-        coalesce(
-            last(debt) over (
-                partition by dim.collateral_type, dim.pool_id 
-                order by dim.ts
-                rows between unbounded preceding and current row
-            ),
-            0
-        ) as debt,
-        coalesce(
-            last(collateral_value) over (
-                partition by dim.collateral_type, dim.pool_id 
-                order by dim.ts
-                rows between unbounded preceding and current row 
-            ),
-            0
-        ) as collateral_value
+        COALESCE(last(debt) over (PARTITION BY dim.collateral_type, dim.pool_id
+    ORDER BY
+        dim.ts rows BETWEEN unbounded preceding
+        AND CURRENT ROW), 0) AS debt,
+        COALESCE(last(collateral_value) over (PARTITION BY dim.collateral_type, dim.pool_id
+    ORDER BY
+        dim.ts rows BETWEEN unbounded preceding
+        AND CURRENT ROW), 0) AS collateral_value
     FROM
         dim
         LEFT JOIN debt
@@ -121,6 +114,15 @@ hourly_rewards AS (
     FROM
         {{ ref('fct_pool_rewards_hourly_eth_mainnet') }}
 ),
+hourly_migration AS (
+    SELECT
+        ts,
+        pool_id,
+        collateral_type,
+        hourly_debt_migrated
+    FROM
+        {{ ref('fct_core_migration_hourly_eth_mainnet') }}
+),
 hourly_returns AS (
     SELECT
         pnl.ts,
@@ -132,9 +134,16 @@ hourly_returns AS (
             iss.hourly_issuance,
             0
         ) hourly_issuance,
+        COALESCE(
+            migration.hourly_debt_migrated,
+            0
+        ) AS hourly_debt_migrated,
         pnl.hourly_pnl + COALESCE(
             iss.hourly_issuance,
             0
+        ) + COALESCE(
+            migration.hourly_debt_migrated,
+            0
         ) AS hourly_pnl,
         COALESCE(
             rewards.rewards_usd,
@@ -149,11 +158,11 @@ hourly_returns AS (
         END AS hourly_rewards_pct,
         CASE
             WHEN pnl.collateral_value = 0 THEN 0
-            ELSE (COALESCE(iss.hourly_issuance, 0) + pnl.hourly_pnl) / pnl.collateral_value
+            ELSE (COALESCE(iss.hourly_issuance, 0) + pnl.hourly_pnl + COALESCE(migration.hourly_debt_migrated, 0)) / pnl.collateral_value
         END AS hourly_pnl_pct,
         CASE
             WHEN pnl.collateral_value = 0 THEN 0
-            ELSE (COALESCE(rewards.rewards_usd, 0) + pnl.hourly_pnl + COALESCE(iss.hourly_issuance, 0)) / pnl.collateral_value
+            ELSE (COALESCE(rewards.rewards_usd, 0) + pnl.hourly_pnl + COALESCE(iss.hourly_issuance, 0) + COALESCE(migration.hourly_debt_migrated, 0)) / pnl.collateral_value
         END AS hourly_total_pct
     FROM
         hourly_pnl pnl
@@ -169,8 +178,15 @@ hourly_returns AS (
         ) = LOWER(
             iss.collateral_type
         )
+        LEFT JOIN hourly_migration migration
+        ON pnl.ts = migration.ts
+        AND pnl.pool_id = migration.pool_id
+        AND LOWER(
+            pnl.collateral_type
+        ) = LOWER(
+            migration.collateral_type
+        )
 )
-
 SELECT
     ts,
     pool_id,
@@ -179,6 +195,7 @@ SELECT
     debt,
     hourly_issuance,
     hourly_pnl,
+    hourly_debt_migrated,
     rewards_usd,
     hourly_pnl_pct,
     hourly_rewards_pct,

transformers/synthetix/models/raw/base/mainnet/core/core_vault_debt_base_mainnet.sql

@@ -1,36 +1,40 @@
-with base as (
-    select
+WITH base AS (
+    SELECT
         block_number,
         contract_address,
         chain_id,
         pool_id,
         collateral_type,
-        cast(
-            value_1 as numeric
-        ) as debt
-    from
+        CAST(
+            value_1 AS numeric
+        ) AS debt
+    FROM
         {{ source(
             'raw_base_mainnet',
             "core_get_vault_debt"
         ) }}
-    where
-        value_1 is not null
+    WHERE
+        value_1 IS NOT NULL
 )
-
-select
-    to_timestamp(blocks.timestamp) as ts,
-    cast(
-        blocks.block_number as integer
-    ) as block_number,
+SELECT
+    TO_TIMESTAMP(
+        blocks.timestamp
+    ) AS ts,
+    CAST(
+        blocks.block_number AS INTEGER
+    ) AS block_number,
     base.contract_address,
-    cast(
-        base.pool_id as integer
-    ) as pool_id,
-    cast(
-        base.collateral_type as varchar
-    ) as collateral_type,
-    {{ convert_wei('base.debt') }} as debt
-from
+    CAST(
+        base.pool_id AS INTEGER
+    ) AS pool_id,
+    CAST(
+        base.collateral_type AS VARCHAR
+    ) AS collateral_type,
+    {{ convert_wei('base.debt') }} AS debt
+FROM
     base
-inner join {{ source('raw_base_mainnet', 'blocks_parquet') }} as blocks
-    on base.block_number = blocks.block_number
+    INNER JOIN {{ source(
+        'raw_base_mainnet',
+        'blocks_parquet'
+    ) }} AS blocks
+    ON base.block_number = blocks.block_number

transformers/synthetix/models/raw/eth/mainnet/core/core_vault_collateral_eth_mainnet.sql

@@ -20,7 +20,9 @@ WITH base AS (
         amount IS NOT NULL
 )
 SELECT
-    blocks.ts,
+    TO_TIMESTAMP(
+        blocks.timestamp
+    ) AS ts,
     base.block_number,
     base.contract_address,
     CAST(
@@ -33,5 +35,8 @@ SELECT
     {{ convert_wei('base.collateral_value') }} AS collateral_value
 FROM
     base
-    JOIN {{ source('raw_eth_mainnet', 'blocks_parquet') }} AS blocks
+    JOIN {{ source(
+        'raw_eth_mainnet',
+        'blocks_parquet'
+    ) }} AS blocks
     ON base.block_number = blocks.block_number

transformers/synthetix/models/raw/eth/mainnet/core/core_vault_debt_eth_mainnet.sql

@@ -17,7 +17,9 @@ WITH base AS (
         value_1 IS NOT NULL
 )
 SELECT
-    blocks.ts,
+    TO_TIMESTAMP(
+        blocks.timestamp
+    ) AS ts,
     base.block_number,
     base.contract_address,
     CAST(
@@ -29,5 +31,8 @@ SELECT
     {{ convert_wei('base.debt') }} AS debt
 FROM
     base
-    JOIN {{ source('raw_eth_mainnet', 'blocks_parquet') }} AS blocks
+    JOIN {{ source(
+        'raw_eth_mainnet',
+        'blocks_parquet'
+    ) }} AS blocks
     ON base.block_number = blocks.block_number