Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add Development Environment #100

Merged
merged 5 commits into from
Aug 22, 2024
Merged

Add Development Environment #100

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e697fd2
Add dev environment to transformers and schedulers
marcus-snx Aug 20, 2024
2241406
Add dev environment to dashboard
marcus-snx Aug 20, 2024
bbd4866
Rename scheduler version
marcus-snx Aug 21, 2024
fb25b42
Exclude extractor task from dev run
marcus-snx Aug 22, 2024
a9ea6da
add seed functions
Tburm Aug 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,5 +33,11 @@ import:
dbt: build
docker compose run transformer dbt run --target prod --profiles-dir profiles --profile synthetix

seed-prod: build
docker compose run transformer dbt seed --target prod --profiles-dir profiles --profile synthetix

seed-dev: build
docker compose run transformer dbt seed --target dev --profiles-dir profiles --profile synthetix

dbt-op: build
docker compose run transformer dbt run --target prod-op --profiles-dir profiles --profile synthetix
16 changes: 8 additions & 8 deletions dashboard/modules/all_chains/core_stats.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ def fetch_data(filters):
apr_{resolution} as apr,
apr_{resolution}_pnl as apr_pnl,
apr_{resolution}_rewards as apr_rewards
FROM prod_arbitrum_mainnet.fct_core_apr_arbitrum_mainnet apr
LEFT JOIN prod_seeds.arbitrum_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
FROM {st.secrets.database.DB_ENV}_arbitrum_mainnet.fct_core_apr_arbitrum_mainnet apr
LEFT JOIN {st.secrets.database.DB_ENV}_seeds.arbitrum_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
WHERE ts >= '{start_date}' and ts <= '{end_date}'

UNION ALL
Expand All @@ -57,8 +57,8 @@ def fetch_data(filters):
apr_{resolution} as apr,
apr_{resolution}_pnl as apr_pnl,
apr_{resolution}_rewards as apr_rewards
FROM prod_base_mainnet.fct_core_apr_base_mainnet apr
LEFT JOIN prod_seeds.base_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_core_apr_base_mainnet apr
LEFT JOIN {st.secrets.database.DB_ENV}_seeds.base_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
WHERE ts >= '{start_date}' and ts <= '{end_date}'

ORDER BY ts
Expand All @@ -80,8 +80,8 @@ def fetch_data(filters):
'Arbitrum' as label,
collateral_value,
cumulative_pnl
FROM prod_arbitrum_mainnet.fct_core_apr_arbitrum_mainnet apr
LEFT JOIN prod_seeds.arbitrum_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
FROM {st.secrets.database.DB_ENV}_arbitrum_mainnet.fct_core_apr_arbitrum_mainnet apr
LEFT JOIN {st.secrets.database.DB_ENV}_seeds.arbitrum_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
WHERE ts >= '{start_date}' and ts <= '{end_date}'
) as a
group by ts, label
Expand All @@ -98,8 +98,8 @@ def fetch_data(filters):
'Base' as label,
collateral_value,
cumulative_pnl
FROM prod_base_mainnet.fct_core_apr_base_mainnet apr
LEFT JOIN prod_seeds.base_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_core_apr_base_mainnet apr
LEFT JOIN {st.secrets.database.DB_ENV}_seeds.base_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
WHERE ts >= '{start_date}' and ts <= '{end_date}'
) as b
group by ts, label
Expand Down
4 changes: 2 additions & 2 deletions dashboard/modules/all_chains/perp_stats.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def fetch_data(filters):
trades,
exchange_fees as fees,
liquidated_accounts as liquidations
FROM prod_base_mainnet.fct_perp_stats_{resolution}_base_mainnet
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_stats_{resolution}_base_mainnet
WHERE ts >= '{start_date}' and ts <= '{end_date}'
),
optimism as (
Expand All @@ -47,7 +47,7 @@ def fetch_data(filters):
trades,
exchange_fees + liquidation_fees as fees,
liquidations
FROM prod_optimism_mainnet.fct_v2_stats_{resolution}_optimism_mainnet
FROM {st.secrets.database.DB_ENV}_optimism_mainnet.fct_v2_stats_{resolution}_optimism_mainnet
where ts >= '{filters["start_date"]}'
and ts <= '{filters["end_date"]}'
)
Expand Down
10 changes: 5 additions & 5 deletions dashboard/modules/arbitrum_mainnet/core_stats.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ def fetch_data(filters):
# get account data
df_account_delegation = pd.read_sql_query(
f"""
SELECT * FROM prod_arbitrum_mainnet.fct_core_account_delegation_arbitrum_mainnet
SELECT * FROM {st.secrets.database.DB_ENV}_arbitrum_mainnet.fct_core_account_delegation_arbitrum_mainnet
WHERE ts >= '{start_date}' and ts <= '{end_date}'
""",
db,
Expand All @@ -47,8 +47,8 @@ def fetch_data(filters):
apr_{resolution} as apr,
apr_{resolution}_pnl as apr_pnl,
apr_{resolution}_rewards as apr_rewards
FROM prod_arbitrum_mainnet.fct_core_apr_arbitrum_mainnet apr
LEFT JOIN prod_seeds.arbitrum_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
FROM {st.secrets.database.DB_ENV}_arbitrum_mainnet.fct_core_apr_arbitrum_mainnet apr
LEFT JOIN {st.secrets.database.DB_ENV}_seeds.arbitrum_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
WHERE ts >= '{start_date}' and ts <= '{end_date}'
and pool_id = 1
ORDER BY ts
Expand All @@ -66,8 +66,8 @@ def fetch_data(filters):
collateral_value,
rewards_usd,
apr_{resolution}_rewards as apr_rewards
FROM prod_arbitrum_mainnet.fct_core_apr_rewards_arbitrum_mainnet apr
LEFT JOIN prod_seeds.arbitrum_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
FROM {st.secrets.database.DB_ENV}_arbitrum_mainnet.fct_core_apr_rewards_arbitrum_mainnet apr
LEFT JOIN {st.secrets.database.DB_ENV}_seeds.arbitrum_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
WHERE ts >= '{start_date}' and ts <= '{end_date}'
and pool_id = 1
and apr.reward_token is not null
Expand Down
6 changes: 3 additions & 3 deletions dashboard/modules/arbitrum_mainnet/spot_markets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def fetch_data(settings):
tx_hash,
synth_market_id,
amount_wrapped
FROM prod_arbitrum_mainnet.fct_spot_wrapper_arbitrum_mainnet
FROM {st.secrets.database.DB_ENV}_arbitrum_mainnet.fct_spot_wrapper_arbitrum_mainnet
WHERE ts >= '{start_date}' and ts <= '{end_date}'
""",
db,
Expand All @@ -47,7 +47,7 @@ def fetch_data(settings):
synth_market_id,
amount,
price
FROM prod_arbitrum_mainnet.fct_spot_atomics_arbitrum_mainnet
FROM {st.secrets.database.DB_ENV}_arbitrum_mainnet.fct_spot_atomics_arbitrum_mainnet
WHERE ts >= '{start_date}' and ts <= '{end_date}'
""",
db,
Expand All @@ -59,7 +59,7 @@ def fetch_data(settings):
ts,
synth_market_id,
supply
FROM prod_arbitrum_mainnet.fct_synth_supply_arbitrum_mainnet
FROM {st.secrets.database.DB_ENV}_arbitrum_mainnet.fct_synth_supply_arbitrum_mainnet
WHERE ts >= '{start_date}' and ts <= '{end_date}'
""",
db,
Expand Down
6 changes: 3 additions & 3 deletions dashboard/modules/arbitrum_sepolia/core_stats.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ def fetch_data(filters):
# get account data
df_account_delegation = pd.read_sql_query(
f"""
SELECT * FROM prod_arbitrum_sepolia.fct_core_account_delegation_arbitrum_sepolia
SELECT * FROM {st.secrets.database.DB_ENV}_arbitrum_sepolia.fct_core_account_delegation_arbitrum_sepolia
WHERE ts >= '{start_date}' and ts <= '{end_date}'
""",
db,
Expand All @@ -47,8 +47,8 @@ def fetch_data(filters):
apr_{resolution} as apr,
apr_{resolution}_pnl as apr_pnl,
apr_{resolution}_rewards as apr_rewards
FROM prod_arbitrum_sepolia.fct_core_apr_arbitrum_sepolia apr
LEFT JOIN prod_seeds.arbitrum_sepolia_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
FROM {st.secrets.database.DB_ENV}_arbitrum_sepolia.fct_core_apr_arbitrum_sepolia apr
LEFT JOIN {st.secrets.database.DB_ENV}_seeds.arbitrum_sepolia_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
WHERE ts >= '{start_date}' and ts <= '{end_date}'
and pool_id = 1
ORDER BY ts
Expand Down
6 changes: 3 additions & 3 deletions dashboard/modules/arbitrum_sepolia/spot_markets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def fetch_data(settings):
tx_hash,
synth_market_id,
amount_wrapped
FROM prod_arbitrum_sepolia.fct_spot_wrapper_arbitrum_sepolia
FROM {st.secrets.database.DB_ENV}_arbitrum_sepolia.fct_spot_wrapper_arbitrum_sepolia
WHERE ts >= '{start_date}' and ts <= '{end_date}'
""",
db,
Expand All @@ -47,7 +47,7 @@ def fetch_data(settings):
synth_market_id,
amount,
price
FROM prod_arbitrum_sepolia.fct_spot_atomics_arbitrum_sepolia
FROM {st.secrets.database.DB_ENV}_arbitrum_sepolia.fct_spot_atomics_arbitrum_sepolia
WHERE ts >= '{start_date}' and ts <= '{end_date}'
""",
db,
Expand All @@ -59,7 +59,7 @@ def fetch_data(settings):
ts,
synth_market_id,
supply
FROM prod_arbitrum_sepolia.fct_synth_supply_arbitrum_sepolia
FROM {st.secrets.database.DB_ENV}_arbitrum_sepolia.fct_synth_supply_arbitrum_sepolia
WHERE ts >= '{start_date}' and ts <= '{end_date}'
""",
db,
Expand Down
10 changes: 5 additions & 5 deletions dashboard/modules/base_mainnet/core_stats.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ def fetch_data(filters):
# get account data
df_account_delegation = pd.read_sql_query(
f"""
SELECT * FROM prod_base_mainnet.fct_core_account_delegation_base_mainnet
SELECT * FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_core_account_delegation_base_mainnet
WHERE ts >= '{start_date}' and ts <= '{end_date}'
""",
db,
Expand All @@ -47,8 +47,8 @@ def fetch_data(filters):
apr_{resolution} as apr,
apr_{resolution}_pnl as apr_pnl,
apr_{resolution}_rewards as apr_rewards
FROM prod_base_mainnet.fct_core_apr_base_mainnet apr
LEFT JOIN prod_seeds.base_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_core_apr_base_mainnet apr
LEFT JOIN {st.secrets.database.DB_ENV}_seeds.base_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
WHERE ts >= '{start_date}' and ts <= '{end_date}'
and pool_id = 1
ORDER BY ts
Expand All @@ -66,8 +66,8 @@ def fetch_data(filters):
collateral_value,
rewards_usd,
apr_{resolution}_rewards as apr_rewards
FROM prod_base_mainnet.fct_core_apr_rewards_base_mainnet apr
LEFT JOIN prod_seeds.base_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_core_apr_rewards_base_mainnet apr
LEFT JOIN {st.secrets.database.DB_ENV}_seeds.base_mainnet_tokens tk on lower(apr.collateral_type) = lower(tk.token_address)
WHERE ts >= '{start_date}' and ts <= '{end_date}'
and pool_id = 1
and apr.reward_token is not null
Expand Down
14 changes: 7 additions & 7 deletions dashboard/modules/base_mainnet/perp_account.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ def fetch_data(filters):
# read data
df_order = pd.read_sql_query(
f"""
SELECT distinct account_id, sender FROM prod_base_mainnet.fct_perp_orders_base_mainnet
SELECT distinct account_id, sender FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_orders_base_mainnet
""",
db,
)
Expand All @@ -40,7 +40,7 @@ def fetch_data(filters):
market_id,
acceptable_price,
commitment_time
FROM prod_base_mainnet.fct_perp_previous_order_expired_base_mainnet
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_previous_order_expired_base_mainnet
WHERE account_id = {account_id if account_id else 'NULL'}
and date(block_timestamp) >= '{start_date}' and date(block_timestamp) <= '{end_date}'
""",
Expand All @@ -61,7 +61,7 @@ def fetch_data(filters):
total_fees,
accrued_funding,
tracking_code
FROM prod_base_mainnet.fct_perp_trades_base_mainnet
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_trades_base_mainnet
WHERE account_id = '{account_id}'
and ts >= '{start_date}' and ts <= '{end_date}'
""",
Expand All @@ -74,7 +74,7 @@ def fetch_data(filters):
cast(account_id as text) as account_id,
synth_market_id,
amount_delta
FROM prod_base_mainnet.fct_perp_collateral_modified_base_mainnet
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_collateral_modified_base_mainnet
WHERE account_id = {account_id if account_id else 'NULL'}
and date(block_timestamp) >= '{start_date}' and date(block_timestamp) <= '{end_date}'
""",
Expand All @@ -87,7 +87,7 @@ def fetch_data(filters):
transaction_hash,
cast(account_id as text) as account_id,
interest
FROM prod_base_mainnet.fct_perp_interest_charged_base_mainnet
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_interest_charged_base_mainnet
WHERE account_id = {account_id if account_id else 'NULL'}
and date(block_timestamp) >= '{start_date}' and date(block_timestamp) <= '{end_date}'
""",
Expand All @@ -100,7 +100,7 @@ def fetch_data(filters):
ts,
account_id,
total_reward
FROM prod_base_mainnet.fct_perp_liq_account_base_mainnet
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_liq_account_base_mainnet
WHERE account_id = '{account_id}'
and ts >= '{start_date}' and ts <= '{end_date}'
""",
Expand All @@ -112,7 +112,7 @@ def fetch_data(filters):
ts,
cumulative_volume,
cumulative_fees
FROM prod_base_mainnet.fct_perp_account_stats_hourly_base_mainnet
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_account_stats_hourly_base_mainnet
WHERE account_id = '{account_id}'
and ts >= '{start_date}' and ts <= '{end_date}'
order by ts
Expand Down
2 changes: 1 addition & 1 deletion dashboard/modules/base_mainnet/perp_integrators.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ def fetch_data(filters):
exchange_fees_share,
referral_fees,
referral_fees_share
FROM prod_base_mainnet.fct_perp_tracking_stats_{resolution}_base_mainnet
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_tracking_stats_{resolution}_base_mainnet
WHERE ts >= '{start_date}' and ts <= '{end_date}'
""",
db,
Expand Down
2 changes: 1 addition & 1 deletion dashboard/modules/base_mainnet/perp_keepers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def fetch_data(filters):
amount_settled_pct,
settlement_rewards,
settlement_rewards_pct
FROM prod_base_mainnet.fct_perp_keeper_stats_{resolution}_base_mainnet
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_keeper_stats_{resolution}_base_mainnet
WHERE ts >= '{start_date}' and ts <= '{end_date}'
ORDER BY ts
""",
Expand Down
4 changes: 2 additions & 2 deletions dashboard/modules/base_mainnet/perp_markets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ def fetch_data(filters):
size_usd,
short_oi_pct,
long_oi_pct
FROM prod_base_mainnet.fct_perp_market_history_base_mainnet
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_market_history_base_mainnet
WHERE ts >= '{start_date}' and ts <= '{end_date}'
ORDER BY ts
""",
Expand All @@ -54,7 +54,7 @@ def fetch_data(filters):
trades,
exchange_fees,
liquidations
FROM prod_base_mainnet.fct_perp_market_stats_daily_base_mainnet
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_market_stats_daily_base_mainnet
WHERE ts >= '{start_date}' and ts <= '{end_date}'
""",
db,
Expand Down
12 changes: 6 additions & 6 deletions dashboard/modules/base_mainnet/perp_monitor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def fetch_data(filters):
acceptable_price,
commitment_time,
tracking_code
FROM prod_base_mainnet.fct_perp_previous_order_expired_base_mainnet
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_previous_order_expired_base_mainnet
WHERE date(block_timestamp) >= '{start_date}' and date(block_timestamp) <= '{end_date}'
ORDER BY block_timestamp
""",
Expand All @@ -56,7 +56,7 @@ def fetch_data(filters):
accrued_funding,
tracking_code,
transaction_hash
FROM prod_base_mainnet.fct_perp_trades_base_mainnet
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_trades_base_mainnet
WHERE ts >= '{start_date}' and ts <= '{end_date}'
ORDER BY ts
""",
Expand All @@ -69,7 +69,7 @@ def fetch_data(filters):
ts,
account_id,
total_reward
FROM prod_base_mainnet.fct_perp_liq_account_base_mainnet
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_liq_account_base_mainnet
WHERE ts >= '{start_date}' and ts <= '{end_date}'
ORDER BY ts
""",
Expand All @@ -83,7 +83,7 @@ def fetch_data(filters):
market_symbol,
skew,
skew * price as skew_usd
FROM prod_base_mainnet.fct_perp_market_history_base_mainnet
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_market_history_base_mainnet
WHERE ts >= '{start_date}' and ts <= '{end_date}'
ORDER BY ts
""",
Expand All @@ -100,7 +100,7 @@ def fetch_data(filters):
trades,
exchange_fees,
liquidations
FROM prod_base_mainnet.fct_perp_market_stats_{resolution}_base_mainnet
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_market_stats_{resolution}_base_mainnet
WHERE ts >= '{start_date}' and ts <= '{end_date}'
""",
db,
Expand All @@ -111,7 +111,7 @@ def fetch_data(filters):
ts,
liquidated_accounts,
liquidation_rewards
FROM prod_base_mainnet.fct_perp_stats_{resolution}_base_mainnet
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_stats_{resolution}_base_mainnet
WHERE ts >= '{start_date}' and ts <= '{end_date}'
""",
db,
Expand Down
4 changes: 2 additions & 2 deletions dashboard/modules/base_mainnet/perp_stats.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ def fetch_data(filters):
liquidation_rewards,
cumulative_exchange_fees,
cumulative_volume
FROM prod_base_mainnet.fct_perp_stats_{resolution}_base_mainnet
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_perp_stats_{resolution}_base_mainnet
WHERE ts >= '{start_date}' and ts <= '{end_date}'
""",
db,
Expand All @@ -51,7 +51,7 @@ def fetch_data(filters):
usd_amount,
cumulative_snx_amount,
cumulative_usd_amount
FROM prod_base_mainnet.fct_buyback_{resolution}_base_mainnet
FROM {st.secrets.database.DB_ENV}_base_mainnet.fct_buyback_{resolution}_base_mainnet
WHERE ts >= '{start_date}' and ts <= '{end_date}'
""",
db,
Expand Down
Loading
Loading