Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tigris fix plus incremental update - ready for review #5748

Merged
merged 1 commit into from
Apr 5, 2024
Merged

Tigris fix plus incremental update - ready for review #5748

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion models/tigris/arbitrum/events/tigris_arbitrum_events_add_margin.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
materialized = 'incremental',
file_format = 'delta',
incremental_strategy = 'merge',
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')],
unique_key = ['evt_block_time', 'evt_tx_hash', 'position_id', 'trader', 'margin', 'protocol_version']
)
}}
Expand Down Expand Up @@ -119,7 +120,7 @@ add_margin_v2 AS (
contract_address as project_contract_address
FROM {{ source('tigristrade_v2_arbitrum', add_margin_trading_evt) }}
{% if is_incremental() %}
WHERE evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('evt_block_time') }}
{% endif %}
{% if not loop.last %}
UNION ALL
Expand Down
3 changes: 2 additions & 1 deletion models/tigris/arbitrum/events/tigris_arbitrum_events_close_position.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
materialized = 'incremental',
file_format = 'delta',
incremental_strategy = 'merge',
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')],
unique_key = ['evt_block_time', 'evt_tx_hash', 'position_id', 'evt_index', 'protocol_version']
)
}}
Expand Down Expand Up @@ -72,7 +73,7 @@ close_position_v2 AS (
contract_address as project_contract_address
FROM {{ source('tigristrade_v2_arbitrum', close_position_trading_evt) }}
{% if is_incremental() %}
WHERE evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('evt_block_time') }}
{% endif %}
{% if not loop.last %}
UNION ALL
Expand Down
3 changes: 2 additions & 1 deletion models/tigris/arbitrum/events/tigris_arbitrum_events_fees_distributed.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
materialized = 'incremental',
file_format = 'delta',
incremental_strategy = 'merge',
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')],
unique_key = ['evt_block_time', 'evt_tx_hash', 'version', 'protocol_version']
)
}}
Expand Down Expand Up @@ -63,7 +64,7 @@ fees_v2 AS (
contract_address as project_contract_address
FROM {{ source('tigristrade_v2_arbitrum', fees_evt) }}
{% if is_incremental() %}
WHERE evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('evt_block_time') }}
{% endif %}
{% if not loop.last %}
UNION ALL
Expand Down
3 changes: 2 additions & 1 deletion models/tigris/arbitrum/events/tigris_arbitrum_events_limit_cancel.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
materialized = 'incremental',
file_format = 'delta',
incremental_strategy = 'merge',
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')],
unique_key = ['evt_block_time', 'evt_tx_hash', 'position_id', 'protocol_version']
)
}}
Expand Down Expand Up @@ -66,7 +67,7 @@ limit_orders_v2 AS (
contract_address as project_contract_address
FROM {{ source('tigristrade_v2_arbitrum', limit_cancel_trading_evt) }} t
{% if is_incremental() %}
WHERE t.evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('t.evt_block_time') }}
{% endif %}
{% if not loop.last %}
UNION ALL
Expand Down
3 changes: 2 additions & 1 deletion models/tigris/arbitrum/events/tigris_arbitrum_events_limit_order.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
materialized = 'incremental',
file_format = 'delta',
incremental_strategy = 'merge',
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')],
unique_key = ['evt_block_time', 'evt_tx_hash', 'position_id', 'protocol_version']
)
}}
Expand Down Expand Up @@ -95,7 +96,7 @@ limit_orders_v2 AS (
ON t.asset = ta.asset_id
AND ta.protocol_version = '2'
{% if is_incremental() %}
WHERE t.evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('t.evt_block_time') }}
{% endif %}
{% if not loop.last %}
UNION ALL
Expand Down
3 changes: 2 additions & 1 deletion models/tigris/arbitrum/events/tigris_arbitrum_events_liquidate_position.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
materialized = 'incremental',
file_format = 'delta',
incremental_strategy = 'merge',
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')],
unique_key = ['evt_block_time', 'evt_tx_hash', 'position_id', 'trader', 'protocol_version']
)
}}
Expand Down Expand Up @@ -68,7 +69,7 @@ liquidate_position_v2 AS (
contract_address as project_contract_address
FROM {{ source('tigristrade_v2_arbitrum', liquidate_position_trading_evt) }}
{% if is_incremental() %}
WHERE evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('evt_block_time') }}
{% endif %}
{% if not loop.last %}
UNION ALL
Expand Down
7 changes: 4 additions & 3 deletions models/tigris/arbitrum/events/tigris_arbitrum_events_modify_margin.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
materialized = 'incremental',
file_format = 'delta',
incremental_strategy = 'merge',
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')],
unique_key = ['evt_block_time', 'evt_tx_hash', 'position_id', 'trader', 'margin', 'leverage', 'protocol_version']
)
}}
Expand Down Expand Up @@ -160,7 +161,7 @@ modify_margin_events_v2 AS (
mm.contract_address as project_contract_address
FROM {{ source('tigristrade_v2_arbitrum', modify_margin_trading_evt) }} mm
{% if is_incremental() %}
WHERE mm.evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('mm.evt_block_time') }}
{% endif %}
{% if not loop.last %}
UNION ALL
Expand All @@ -178,7 +179,7 @@ add_margin_calls_v2 AS (
ap._addMargin/1e18 as margin_change
FROM {{ source('tigristrade_v2_arbitrum', add_margin_trading_call) }} ap
{% if is_incremental() %}
WHERE ap.call_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('ap.call_block_time') }}
{% endif %}
{% if not loop.last %}
UNION ALL
Expand All @@ -196,7 +197,7 @@ remove_margin_calls_v2 AS (
ap._removeMargin/1e18 as margin_change
FROM {{ source('tigristrade_v2_arbitrum', remove_margin_trading_call) }} ap
{% if is_incremental() %}
WHERE ap.call_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('ap.call_block_time') }}
{% endif %}
{% if not loop.last %}
UNION ALL
Expand Down
3 changes: 2 additions & 1 deletion models/tigris/arbitrum/events/tigris_arbitrum_events_open_position.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
materialized = 'incremental',
file_format = 'delta',
incremental_strategy = 'merge',
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')],
unique_key = ['evt_block_time', 'evt_tx_hash', 'position_id', 'protocol_version']
)
}}
Expand Down Expand Up @@ -109,7 +110,7 @@ open_position_v2 AS (
ON CAST(json_extract_scalar(tradeInfo, '$.asset') as double) = CAST(ta.asset_id as double)
AND ta.protocol_version = '2'
{% if is_incremental() %}
WHERE t.evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('t.evt_block_time') }}
{% endif %}
{% if not loop.last %}
UNION ALL
Expand Down
3 changes: 2 additions & 1 deletion models/tigris/arbitrum/events/tigris_arbitrum_events_options_close_position.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
materialized = 'incremental',
file_format = 'delta',
incremental_strategy = 'merge',
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')],
unique_key = ['evt_block_time', 'evt_tx_hash', 'position_id', 'positions_contract']
)
}}
Expand Down Expand Up @@ -34,7 +35,7 @@ close_position_v2 AS (
contract_address as project_contract_address
FROM {{ source('tigristrade_v2_arbitrum', close_position) }} t
{% if is_incremental() %}
WHERE t.evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('t.evt_block_time') }}
{% endif %}
{% if not loop.last %}
UNION ALL
Expand Down
3 changes: 2 additions & 1 deletion models/tigris/arbitrum/events/tigris_arbitrum_events_options_fees_distributed.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
materialized = 'incremental',
file_format = 'delta',
incremental_strategy = 'merge',
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')],
unique_key = ['evt_block_time', 'evt_tx_hash', 'evt_index']
)
}}
Expand All @@ -32,7 +33,7 @@ fees_v2 AS (
contract_address as project_contract_address
FROM {{ source('tigristrade_v2_arbitrum', fees_evt) }}
{% if is_incremental() %}
WHERE evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('evt_block_time') }}
{% endif %}
{% if not loop.last %}
UNION ALL
Expand Down
3 changes: 2 additions & 1 deletion models/tigris/arbitrum/events/tigris_arbitrum_events_options_limit_cancel.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
materialized = 'incremental',
file_format = 'delta',
incremental_strategy = 'merge',
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')],
unique_key = ['evt_block_time', 'evt_tx_hash', 'position_id', 'positions_contract']
)
}}
Expand All @@ -32,7 +33,7 @@ limit_cancel_v2 AS (
contract_address as project_contract_address
FROM {{ source('tigristrade_v2_arbitrum', limit_cancel) }} t
{% if is_incremental() %}
WHERE t.evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('t.evt_block_time') }}
{% endif %}
{% if not loop.last %}
UNION ALL
Expand Down
3 changes: 2 additions & 1 deletion models/tigris/arbitrum/events/tigris_arbitrum_events_options_limit_order.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
materialized = 'incremental',
file_format = 'delta',
incremental_strategy = 'merge',
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')],
unique_key = ['evt_block_time', 'evt_tx_hash', 'position_id', 'positions_contract']
)
}}
Expand Down Expand Up @@ -34,7 +35,7 @@ limit_order_v2 AS (
contract_address as project_contract_address
FROM {{ source('tigristrade_v2_arbitrum', limit_order) }} t
{% if is_incremental() %}
WHERE t.evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('t.evt_block_time') }}
{% endif %}
{% if not loop.last %}
UNION ALL
Expand Down
3 changes: 2 additions & 1 deletion models/tigris/arbitrum/events/tigris_arbitrum_events_options_open_position.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
materialized = 'incremental',
file_format = 'delta',
incremental_strategy = 'merge',
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')],
unique_key = ['evt_block_time', 'evt_tx_hash', 'position_id', 'positions_contract']
)
}}
Expand Down Expand Up @@ -53,7 +54,7 @@ open_position_v2 AS (
ON CAST(json_extract_scalar(tradeInfo, '$.asset') as double) = CAST(ta.asset_id as double)
AND ta.protocol_version = '2'
{% if is_incremental() %}
WHERE t.evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('t.evt_block_time') }}
{% endif %}
{% if not loop.last %}
UNION ALL
Expand Down
11 changes: 6 additions & 5 deletions models/tigris/arbitrum/tigris_arbitrum_options_trades.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
materialized = 'incremental',
file_format = 'delta',
incremental_strategy = 'merge',
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')],
unique_key = ['evt_block_time', 'evt_tx_hash', 'position_id', 'trade_type', 'positions_contract', 'protocol_version']
)
}}
Expand Down Expand Up @@ -45,7 +46,7 @@ open_position as (
{{ ref('tigris_arbitrum_events_options_open_position') }} op
WHERE order_type = '0'
{% if is_incremental() %}
AND op.evt_block_time >= date_trunc('day', now() - interval '7' day)
AND {{ incremental_predicate('op.evt_block_time') }}
{% endif %}
),

Expand Down Expand Up @@ -85,7 +86,7 @@ limit_order as (
FROM
{{ ref('tigris_arbitrum_events_options_limit_order') }} op
{% if is_incremental() %}
WHERE op.evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('op.evt_block_time') }}
{% endif %}
),

Expand Down Expand Up @@ -131,10 +132,10 @@ close_position as (
AND op.evt_join_index = f.evt_join_index
-- can't join on position id as position id isn't present in fees event so using this workaround
{% if is_incremental() %}
AND f.evt_block_time >= date_trunc('day', now() - interval '7' day)
AND {{ incremental_predicate('f.evt_block_time') }}
{% endif %}
{% if is_incremental() %}
WHERE op.evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('op.evt_block_time') }}
{% endif %}
),

Expand Down Expand Up @@ -174,7 +175,7 @@ limit_cancel as (
FROM
{{ ref('tigris_arbitrum_events_options_limit_cancel') }} op
{% if is_incremental() %}
WHERE op.evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('op.evt_block_time') }}
{% endif %}
)

Expand Down
5 changes: 3 additions & 2 deletions models/tigris/arbitrum/tigris_arbitrum_perpetual_trades.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
materialized = 'incremental',
file_format = 'delta',
incremental_strategy = 'merge',
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')],
unique_key = ['block_date', 'blockchain', 'project', 'version', 'tx_hash', 'evt_index'],
post_hook='{{ expose_spells(\'["arbitrum"]\',
"project",
Expand Down Expand Up @@ -140,7 +141,7 @@ INNER JOIN
AND tx.block_time >= TIMESTAMP '{{project_start_date}}'
{% endif %}
{% if is_incremental() %}
AND tx.block_time >= date_trunc('day', now() - interval '7' Day)
AND {{ incremental_predicate('tx.block_time') }}
{% endif %}
LEFT JOIN
{{ source('tokens_arbitrum', 'erc20') }} er
Expand All @@ -150,7 +151,7 @@ LEFT JOIN {{ source('prices', 'usd') }} pe
AND pe.blockchain = 'arbitrum'
AND pe.symbol = 'WETH'
{% if is_incremental() %}
AND pe.minute >= date_trunc('day', now() - interval '7' day)
AND {{ incremental_predicate('pe.minute') }}
{% endif %}


Expand Down
17 changes: 9 additions & 8 deletions models/tigris/arbitrum/tigris_arbitrum_trades.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
materialized = 'incremental',
file_format = 'delta',
incremental_strategy = 'merge',
incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.evt_block_time')],
unique_key = ['evt_block_time', 'evt_tx_hash', 'position_id', 'trade_type', 'positions_contract', 'protocol_version']
)
}}
Expand Down Expand Up @@ -38,7 +39,7 @@ open_position as (
FROM {{ ref('tigris_arbitrum_events_open_position') }}
WHERE open_type = 'open_position'
{% if is_incremental() %}
AND evt_block_time >= date_trunc('day', now() - interval '7' day)
AND {{ incremental_predicate('evt_block_time') }}
{% endif %}
),

Expand Down Expand Up @@ -122,7 +123,7 @@ limit_order as (
protocol_version
FROM {{ ref('tigris_arbitrum_events_limit_order') }}
{% if is_incremental() %}
WHERE evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('evt_block_time') }}
{% endif %}
),

Expand Down Expand Up @@ -188,7 +189,7 @@ close_position as (
AND c.positions_contract = lo.positions_contract
AND c.protocol_version = lo.protocol_version
{% if is_incremental() %}
WHERE c.evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('c.evt_block_time') }}
{% endif %}

),
Expand Down Expand Up @@ -229,7 +230,7 @@ liquidate_position as (
AND lp.positions_contract = lo.positions_contract
AND lp.protocol_version = lo.protocol_version
{% if is_incremental() %}
WHERE lp.evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('lp.evt_block_time') }}
{% endif %}
),

Expand Down Expand Up @@ -288,7 +289,7 @@ add_margin as (
AND am.protocol_version = l.protocol_version
AND am.evt_block_time > l.evt_block_time
{% if is_incremental() %}
WHERE am.evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('am.evt_block_time') }}
{% endif %}
GROUP BY 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
) tmp
Expand Down Expand Up @@ -346,7 +347,7 @@ modify_margin as (
AND mm.positions_contract = lo.positions_contract
AND mm.protocol_version = lo.protocol_version
{% if is_incremental() %}
WHERE mm.evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('mm.evt_block_time') }}
{% endif %}
),

Expand Down Expand Up @@ -381,7 +382,7 @@ limit_cancel as (
AND lc.positions_contract = op.positions_contract
AND lc.protocol_version = op.protocol_version
{% if is_incremental() %}
WHERE lc.evt_block_time >= date_trunc('day', now() - interval '7' day)
WHERE {{ incremental_predicate('lc.evt_block_time') }}
{% endif %}
),

Expand Down Expand Up @@ -457,5 +458,5 @@ LEFT JOIN {{ source('prices', 'usd') }} pe
AND pe.blockchain = 'arbitrum'
AND pe.symbol = 'WETH'
{% if is_incremental() %}
AND pe.minute >= date_trunc('day', now() - interval '7' day)
AND {{ incremental_predicate('pe.minute') }}
{% endif %}
Loading
Loading