Revert "Revert transfers merges from today (duneanalytics#6545)" (dun…

…eanalytics#6551) This reverts commit 0f1b482.
bancorprotocol · Aug 13, 2024 · f3cfd20 · f3cfd20
1 parent 0f1b482
commit f3cfd20
Show file tree

Hide file tree

Showing 43 changed files with 732 additions and 161 deletions.
diff --git a/dbt_subprojects/tokens/macros/transfers/transfers_base.sql b/dbt_subprojects/tokens/macros/transfers/transfers_base.sql
@@ -1,7 +1,8 @@
-{% macro transfers_base(blockchain, traces, transactions, erc20_transfers, native_contract_address = null) %}
+{% macro transfers_base(blockchain, traces, transactions, erc20_transfers, native_contract_address = null, include_traces = true) %}
 {%- set token_standard_20 = 'bep20' if blockchain == 'bnb' else 'erc20' -%}
 
 WITH transfers AS (
+    {% if include_traces %}
     SELECT
         block_date
         , block_time
@@ -16,7 +17,7 @@ WITH transfers AS (
         {% endif %}
         , 'native' AS token_standard
         , "from"
-        , to
+        , COALESCE(to,address) AS to -- Contract Creation has NULL "to" address, but transaction might have value that goes to contract created
         , value AS amount_raw
     FROM {{ traces }}
     WHERE success
@@ -27,8 +28,9 @@ WITH transfers AS (
         {% endif %}
 
     UNION ALL
+    {% endif %}
 
-    SELECT 
+    SELECT
         cast(date_trunc('day', t.evt_block_time) as date) AS block_date
         , t.evt_block_time AS block_time
         , t.evt_block_number AS block_number
@@ -59,6 +61,7 @@ SELECT
     -- We have to create this unique key because evt_index and trace_address can be null
     {{dbt_utils.generate_surrogate_key(['t.block_number', 'tx.index', 't.evt_index', "array_join(t.trace_address, ',')"])}} as unique_key
     , '{{blockchain}}' as blockchain
+    , cast(date_trunc('month', t.block_date) as date) AS block_month
     , t.block_date
     , t.block_time
     , t.block_number

diff --git a/dbt_subprojects/tokens/macros/transfers/transfers_base_wrapped_token.sql b/dbt_subprojects/tokens/macros/transfers/transfers_base_wrapped_token.sql
@@ -42,6 +42,7 @@ SELECT
     -- We have to create this unique key because evt_index and trace_address can be null
     {{dbt_utils.generate_surrogate_key(['t.block_number', 'tx.index', 't.evt_index', "array_join(trace_address, ',')"])}} as unique_key
     , '{{blockchain}}' as blockchain
+    , cast(date_trunc('month', t.block_time) as date) AS block_month
     , cast(date_trunc('day', t.block_time) as date) as block_date
     , t.block_time
     , t.block_number

diff --git a/dbt_subprojects/tokens/macros/transfers/transfers_enrich.sql b/dbt_subprojects/tokens/macros/transfers/transfers_enrich.sql
@@ -14,6 +14,7 @@ END
     , evms_info_model = null
     , transfers_start_date = '2000-01-01'
     , blockchain = null
+    , usd_amount_threshold = 25000000000
     )
 %}
 
@@ -45,49 +46,80 @@ WITH base_transfers as (
         minute >= TIMESTAMP '{{ transfers_start_date }}'
     {% endif %}
 )
+, transfers as (
+    SELECT
+        t.unique_key
+        , t.blockchain
+        , t.block_month
+        , t.block_date
+        , t.block_time
+        , t.block_number
+        , t.tx_hash
+        , t.evt_index
+        , t.trace_address
+        , t.token_standard
+        , t.tx_from
+        , t.tx_to
+        , t.tx_index
+        , t."from"
+        , t.to
+        , t.contract_address
+        , {{case_when_token_standard('evms_info.native_token_symbol', 'tokens_erc20.symbol', 'NULL')}} AS symbol
+        , t.amount_raw
+        , {{case_when_token_standard('t.amount_raw / power(10, 18)', 't.amount_raw / power(10, tokens_erc20.decimals)', 'cast(t.amount_raw as double)')}} AS amount
+        , prices.price AS price_usd
+        , {{case_when_token_standard('(t.amount_raw / power(10, 18)) * prices.price',
+            '(t.amount_raw / power(10, tokens_erc20.decimals)) * prices.price',
+            'NULL')}} AS amount_usd
+    FROM
+        base_transfers as t
+    INNER JOIN
+        {{ evms_info_model }} as evms_info
+        ON evms_info.blockchain = t.blockchain
+    LEFT JOIN
+        {{ tokens_erc20_model }} as tokens_erc20
+        ON tokens_erc20.blockchain = t.blockchain
+        AND tokens_erc20.contract_address = t.contract_address
+    LEFT JOIN
+        prices
+        ON date_trunc('minute', t.block_time) = prices.minute
+        AND CASE
+            WHEN t.token_standard = 'native'
+                THEN
+                prices.blockchain IS NULL
+                AND prices.contract_address IS NULL
+                AND evms_info.native_token_symbol = prices.symbol
+            ELSE
+                prices.blockchain = '{{ blockchain }}'
+                AND t.contract_address = prices.contract_address
+            END
+)
 SELECT
-    t.unique_key
-    , t.blockchain
-    , t.block_date
-    , t.block_time
-    , t.block_number
-    , t.tx_hash
-    , t.evt_index
-    , t.trace_address
-    , t.token_standard
-    , t.tx_from
-    , t.tx_to
-    , t.tx_index
-    , t."from"
-    , t.to
-    , t.contract_address
-    , {{case_when_token_standard('evms_info.native_token_symbol', 'tokens_erc20.symbol', 'NULL')}} AS symbol
-    , t.amount_raw
-    , {{case_when_token_standard('t.amount_raw / power(10, 18)', 't.amount_raw / power(10, tokens_erc20.decimals)', 'cast(t.amount_raw as double)')}} AS amount
-    , prices.price AS price_usd
-    , {{case_when_token_standard('(t.amount_raw / power(10, 18)) * prices.price',
-        '(t.amount_raw / power(10, tokens_erc20.decimals)) * prices.price',
-        'NULL')}} AS amount_usd
+    unique_key
+    , blockchain
+    , block_month
+    , block_date
+    , block_time
+    , block_number
+    , tx_hash
+    , evt_index
+    , trace_address
+    , token_standard
+    , tx_from
+    , tx_to
+    , tx_index
+    , "from"
+    , to
+    , contract_address
+    , symbol
+    , amount_raw
+    , amount
+    , price_usd
+    , CASE
+        WHEN amount_usd >= {{ usd_amount_threshold }}
+            THEN CAST(NULL as double)
+            ELSE amount_usd -- Select only transfers where USD amount is less than the threshold
+        END AS amount_usd
 FROM
-    base_transfers as t
-INNER JOIN
-    {{ evms_info_model }} as evms_info
-    ON evms_info.blockchain = t.blockchain
-LEFT JOIN
-    {{ tokens_erc20_model }} as tokens_erc20
-    ON tokens_erc20.blockchain = t.blockchain
-    AND tokens_erc20.contract_address = t.contract_address
-LEFT JOIN
-    prices
-    ON date_trunc('minute', t.block_time) = prices.minute
-    AND CASE
-        WHEN t.token_standard = 'native'
-            THEN
-            prices.blockchain IS NULL
-            AND prices.contract_address IS NULL
-            AND evms_info.native_token_symbol = prices.symbol
-        ELSE
-            prices.blockchain = '{{ blockchain }}'
-            AND t.contract_address = prices.contract_address
-        END
+    transfers
 {%- endmacro %}
diff --git a/...projects/tokens/models/transfers_and_balances/arbitrum/tokens_arbitrum_base_transfers.sql b/...projects/tokens/models/transfers_and_balances/arbitrum/tokens_arbitrum_base_transfers.sql
@@ -1,7 +1,7 @@
 {{config(
     schema = 'tokens_arbitrum',
     alias = 'base_transfers',
-    partition_by = ['block_date'],
+    partition_by = ['block_month'],
     materialized = 'incremental',
     file_format = 'delta',
     incremental_strategy = 'merge',
@@ -15,6 +15,6 @@
     traces = source('arbitrum','traces'),
     transactions = source('arbitrum','transactions'),
     erc20_transfers = source('erc20_arbitrum','evt_transfer'),
-    native_contract_address = null,
+    native_contract_address = null
 )
 }}
diff --git a/dbt_subprojects/tokens/models/transfers_and_balances/arbitrum/tokens_arbitrum_transfers.sql b/dbt_subprojects/tokens/models/transfers_and_balances/arbitrum/tokens_arbitrum_transfers.sql
@@ -1,7 +1,7 @@
 {{config(
     schema = 'tokens_arbitrum',
     alias = 'transfers',
-    partition_by = ['block_date'],
+    partition_by = ['block_month'],
     materialized = 'incremental',
     file_format = 'delta',
     incremental_strategy = 'merge',

diff --git a/...ts/tokens/models/transfers_and_balances/avalanche_c/tokens_avalanche_c_base_transfers.sql b/...ts/tokens/models/transfers_and_balances/avalanche_c/tokens_avalanche_c_base_transfers.sql
@@ -1,7 +1,7 @@
 {{config(
     schema = 'tokens_avalanche_c',
     alias = 'base_transfers',
-    partition_by = ['block_date'],
+    partition_by = ['block_month'],
     materialized = 'incremental',
     file_format = 'delta',
     incremental_strategy = 'merge',

diff --git a/...rojects/tokens/models/transfers_and_balances/avalanche_c/tokens_avalanche_c_transfers.sql b/...rojects/tokens/models/transfers_and_balances/avalanche_c/tokens_avalanche_c_transfers.sql
@@ -1,7 +1,7 @@
 {{config(
     schema = 'tokens_avalanche_c',
     alias = 'transfers',
-    partition_by = ['block_date'],
+    partition_by = ['block_month'],
     materialized = 'incremental',
     file_format = 'delta',
     incremental_strategy = 'merge',

diff --git a/dbt_subprojects/tokens/models/transfers_and_balances/base/tokens_base_base_transfers.sql b/dbt_subprojects/tokens/models/transfers_and_balances/base/tokens_base_base_transfers.sql
@@ -1,7 +1,7 @@
 {{config(
     schema = 'tokens_base',
     alias = 'base_transfers',
-    partition_by = ['block_date'],
+    partition_by = ['block_month'],
     materialized = 'incremental',
     file_format = 'delta',
     incremental_strategy = 'merge',

diff --git a/dbt_subprojects/tokens/models/transfers_and_balances/base/tokens_base_transfers.sql b/dbt_subprojects/tokens/models/transfers_and_balances/base/tokens_base_transfers.sql
@@ -1,7 +1,7 @@
 {{config(
     schema = 'tokens_base',
     alias = 'transfers',
-    partition_by = ['block_date'],
+    partition_by = ['block_month'],
     materialized = 'incremental',
     file_format = 'delta',
     incremental_strategy = 'merge',

diff --git a/dbt_subprojects/tokens/models/transfers_and_balances/blast/tokens_blast_base_transfers.sql b/dbt_subprojects/tokens/models/transfers_and_balances/blast/tokens_blast_base_transfers.sql
@@ -1,7 +1,7 @@
 {{config(
     schema = 'tokens_blast',
     alias = 'base_transfers',
-    partition_by = ['block_date'],
+    partition_by = ['block_month'],
     materialized = 'incremental',
     file_format = 'delta',
     incremental_strategy = 'merge',

diff --git a/dbt_subprojects/tokens/models/transfers_and_balances/blast/tokens_blast_transfers.sql b/dbt_subprojects/tokens/models/transfers_and_balances/blast/tokens_blast_transfers.sql
@@ -1,7 +1,7 @@
 {{config(
     schema = 'tokens_blast',
     alias = 'transfers',
-    partition_by = ['block_date'],
+    partition_by = ['block_month'],
     materialized = 'incremental',
     file_format = 'delta',
     incremental_strategy = 'merge',

diff --git a/dbt_subprojects/tokens/models/transfers_and_balances/bnb/tokens_bnb_base_transfers.sql b/dbt_subprojects/tokens/models/transfers_and_balances/bnb/tokens_bnb_base_transfers.sql
@@ -1,7 +1,7 @@
 {{config(
     schema = 'tokens_bnb',
     alias = 'base_transfers',
-    partition_by = ['block_date'],
+    partition_by = ['block_month'],
     materialized = 'incremental',
     file_format = 'delta',
     incremental_strategy = 'merge',

diff --git a/dbt_subprojects/tokens/models/transfers_and_balances/bnb/tokens_bnb_transfers.sql b/dbt_subprojects/tokens/models/transfers_and_balances/bnb/tokens_bnb_transfers.sql
@@ -1,7 +1,7 @@
 {{config(
     schema = 'tokens_bnb',
     alias = 'transfers',
-    partition_by = ['block_date'],
+    partition_by = ['block_month'],
     materialized = 'incremental',
     file_format = 'delta',
     incremental_strategy = 'merge',

diff --git a/dbt_subprojects/tokens/models/transfers_and_balances/celo/tokens_celo_base_transfers.sql b/dbt_subprojects/tokens/models/transfers_and_balances/celo/tokens_celo_base_transfers.sql
@@ -1,7 +1,7 @@
 {{config(
     schema = 'tokens_celo',
     alias = 'base_transfers',
-    partition_by = ['block_date'],
+    partition_by = ['block_month'],
     materialized = 'incremental',
     file_format = 'delta',
     incremental_strategy = 'merge',

diff --git a/dbt_subprojects/tokens/models/transfers_and_balances/celo/tokens_celo_transfers.sql b/dbt_subprojects/tokens/models/transfers_and_balances/celo/tokens_celo_transfers.sql
@@ -1,7 +1,7 @@
 {{config(
     schema = 'tokens_celo',
     alias = 'transfers',
-    partition_by = ['block_date'],
+    partition_by = ['block_month'],
     materialized = 'incremental',
     file_format = 'delta',
     incremental_strategy = 'merge',

diff --git a/...projects/tokens/models/transfers_and_balances/ethereum/tokens_ethereum_base_transfers.sql b/...projects/tokens/models/transfers_and_balances/ethereum/tokens_ethereum_base_transfers.sql
@@ -1,7 +1,7 @@
 {{ config(
     schema = 'tokens_ethereum',
     alias = 'base_transfers',
-    partition_by = ['block_date'],
+    partition_by = ['block_month'],
     materialized = 'incremental',
     file_format = 'delta',
     incremental_strategy = 'merge',
@@ -15,7 +15,7 @@
     traces = source('ethereum','traces'),
     transactions = source('ethereum','transactions'),
     erc20_transfers = source('erc20_ethereum','evt_Transfer'),
-    native_contract_address = '0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
+    native_contract_address = '0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee'
 )}}
 
 UNION ALL

diff --git a/dbt_subprojects/tokens/models/transfers_and_balances/ethereum/tokens_ethereum_transfers.sql b/dbt_subprojects/tokens/models/transfers_and_balances/ethereum/tokens_ethereum_transfers.sql
@@ -1,7 +1,7 @@
 {{config(
     schema = 'tokens_ethereum',
     alias = 'transfers',
-    partition_by = ['block_date'],
+    partition_by = ['block_month'],
     materialized = 'incremental',
     file_format = 'delta',
     incremental_strategy = 'merge',

diff --git a/dbt_subprojects/tokens/models/transfers_and_balances/fantom/tokens_fantom_base_transfers.sql b/dbt_subprojects/tokens/models/transfers_and_balances/fantom/tokens_fantom_base_transfers.sql
@@ -1,7 +1,7 @@
 {{config(
     schema = 'tokens_fantom',
     alias = 'base_transfers',
-    partition_by = ['block_date'],
+    partition_by = ['block_month'],
     materialized = 'incremental',
     file_format = 'delta',
     incremental_strategy = 'merge',

diff --git a/dbt_subprojects/tokens/models/transfers_and_balances/fantom/tokens_fantom_transfers.sql b/dbt_subprojects/tokens/models/transfers_and_balances/fantom/tokens_fantom_transfers.sql
@@ -1,7 +1,7 @@
 {{config(
     schema = 'tokens_fantom',
     alias = 'transfers',
-    partition_by = ['block_date'],
+    partition_by = ['block_month'],
     materialized = 'incremental',
     file_format = 'delta',
     incremental_strategy = 'merge',