From a8e9093e95d6707751364dd36a9fd0d70f66e56a Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Tue, 5 Dec 2023 23:45:39 -0500 Subject: [PATCH 01/23] it's time --- .../orca_whirlpool_trades_sources.yml | 9 -- models/solana_utils/solana_utils_schema.yml | 34 +++++- .../tokens/solana/tokens_solana.sources.yml | 12 ++ models/tokens/solana/tokens_solana_schema.yml | 25 +++- .../tokens/solana/tokens_solana_transfers.sql | 114 ++++++++++++++++++ 5 files changed, 183 insertions(+), 11 deletions(-) create mode 100644 models/tokens/solana/tokens_solana_transfers.sql diff --git a/models/orca_whirlpool/orca_whirlpool_trades_sources.yml b/models/orca_whirlpool/orca_whirlpool_trades_sources.yml index b0d22a5560e..1f571724131 100644 --- a/models/orca_whirlpool/orca_whirlpool_trades_sources.yml +++ b/models/orca_whirlpool/orca_whirlpool_trades_sources.yml @@ -18,13 +18,4 @@ sources: - name: whirlpool_call_initializeFeeTier loaded_at_field: call_block_time - name: whirlpool_call_setDefaultFeeRate - loaded_at_field: call_block_time - - - name: spl_token_solana - description: "spl_token decoded tables" - freshness: # default freshness - warn_after: { count: 12, period: hour } - error_after: { count: 24, period: hour } - tables: - - name: spl_token_call_transfer loaded_at_field: call_block_time \ No newline at end of file diff --git a/models/solana_utils/solana_utils_schema.yml b/models/solana_utils/solana_utils_schema.yml index 60613de369a..531a1881e9a 100644 --- a/models/solana_utils/solana_utils_schema.yml +++ b/models/solana_utils/solana_utils_schema.yml @@ -86,4 +86,36 @@ models: - &owned_domains name: owned_domains - + - name: solana_utils_transfers + meta: + blockchain: solana + contributors: [ilemi] + config: + tags: ['solana','transfers','erc20','nft','spl'] + description: > + get all spl and system token transfers + columns: + - &block_time + name: block_time + - &action + name: action + - &token_mint_address + name: token_mint_address + - &amount + name: amount + - &from_owner + name: from_owner + - &to_owner + name: to_owner + - &from_token_account + name: from_token_account + - &to_token_account + name: to_token_account + - &tx_signer + name: tx_signer + - &tx_id + name: tx_id + - &outer_instruction_index + name: outer_instruction_index + - &inner_instruction_index + name: inner_instruction_index \ No newline at end of file diff --git a/models/tokens/solana/tokens_solana.sources.yml b/models/tokens/solana/tokens_solana.sources.yml index c420ec03bf5..000e978b31a 100644 --- a/models/tokens/solana/tokens_solana.sources.yml +++ b/models/tokens/solana/tokens_solana.sources.yml @@ -11,6 +11,18 @@ sources: loaded_at_field: call_block_time - name: spl_token_call_initializeMint2 loaded_at_field: call_block_time + - name: spl_token_call_transfer + loaded_at_field: call_block_time + - name: spl_token_call_transferChecked + loaded_at_field: call_block_time + - name: spl_token_call_mintTo + loaded_at_field: call_block_time + - name: spl_token_call_mintToChecked + loaded_at_field: call_block_time + - name: spl_token_call_burn + loaded_at_field: call_block_time + - name: spl_token_call_burnChecked + loaded_at_field: call_block_time - name: mpl_token_metadata_solana description: "mpl metadata decoded tables" freshness: # default freshness diff --git a/models/tokens/solana/tokens_solana_schema.yml b/models/tokens/solana/tokens_solana_schema.yml index 847f228916f..2f8ff24668c 100644 --- a/models/tokens/solana/tokens_solana_schema.yml +++ b/models/tokens/solana/tokens_solana_schema.yml @@ -62,4 +62,27 @@ models: - name: call_tx_id - name: call_block_time - name: call_block_slot - - name: call_tx_signer \ No newline at end of file + - name: call_tx_signer + + + - name: tokens_solana_transfers + meta: + blockchain: solana + contributors: [ilemi] + config: + tags: ['solana','transfers','erc20','nft','spl'] + description: > + get all spl and system token transfers + columns: + - name: block_time + - name: action + - name: token_mint_address + - name: amount + - name: from_owner + - name: to_owner + - name: from_token_account + - name: to_token_account + - name: tx_signer + - name: tx_id + - name: outer_instruction_index + - name: inner_instruction_index \ No newline at end of file diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql new file mode 100644 index 00000000000..88abfe6716e --- /dev/null +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -0,0 +1,114 @@ + {{ + config( + schema = 'tokens', + alias = 'transfers', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['tx_id','outer_instruction_index','inner_instruction_index'], + post_hook='{{ expose_spells(\'["solana"]\', + "sector", + "tokens", + \'["ilemi"]\') }}') +}} + +SELECT + tr.call_block_time as block_time + , action + , amount + , COALESCE(tk_s.token_mint_address, tk_d.token_mint_address) as token_mint_address + , tk_s.token_balance_owner as from_owner + , tk_d.token_balance_owner as to_owner + , account_source as from_token_account + , account_destination as to_token_account + , tr.call_tx_signer as tx_signer + , tr.call_tx_id as tx_id + , tr.call_outer_instruction_index as outer_instruction_index + , tr.call_inner_instruction_index as inner_instruction_index +FROM ( + SELECT account_source, account_destination, amount, call_tx_id, call_block_time, call_outer_executing_account, call_tx_signer, 'transfer' as action, call_outer_instruction_index, call_inner_instruction_index + FROM {{ source('spl_token_solana','spl_token_call_transfer') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT account_source, account_destination, amount, call_tx_id, call_block_time, call_outer_executing_account, call_tx_signer, 'transfer' as action, call_outer_instruction_index, call_inner_instruction_index + FROM {{ source('spl_token_solana','spl_token_call_transferChecked') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT null, account_account as account_destination, amount, call_tx_id, call_block_time, call_outer_executing_account, call_tx_signer, 'mint' as action, call_outer_instruction_index, call_inner_instruction_index + FROM {{ source('spl_token_solana','spl_token_call_mintTo') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT null, account_account as account_destination, amount, call_tx_id, call_block_time, call_outer_executing_account, call_tx_signer, 'mint' as action, call_outer_instruction_index, call_inner_instruction_index + FROM {{ source('spl_token_solana','spl_token_call_mintToChecked') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT account_account as account_source, null, amount, call_tx_id, call_block_time, call_outer_executing_account, call_tx_signer, 'burn' as action, call_outer_instruction_index, call_inner_instruction_index + FROM {{ source('spl_token_solana','spl_token_call_burn') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT account_account as account_source, null, amount, call_tx_id, call_block_time, call_outer_executing_account, call_tx_signer, 'burn' as action, call_outer_instruction_index, call_inner_instruction_index + FROM {{ source('spl_token_solana','spl_token_call_burnChecked') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} +) tr +--get token and accounts +LEFT JOIN {{ ref('solana_utils_token_accounts') }} tk_s ON tk_s.address = tr.account_source +LEFT JOIN {{ ref('solana_utils_token_accounts') }} tk_d ON tk_d.address = tr.account_destination + +UNION ALL + +SELECT + call_block_time as block_time + , 'transfer' as action + , lamports as amount --1e9 + , 'native' as token_mint_address + , account_from as from_owner + , account_to as to_owner + , account_from as from_token_account + , account_to as to_token_account + , call_tx_signer as tx_signer + , call_tx_id as tx_id + , call_outer_instruction_index as outer_instruction_index + , call_inner_instruction_index as inner_instruction_index +FROM ( + SELECT account_from, account_to, lamports, call_tx_signer, call_block_time, call_tx_id, call_outer_executing_account, call_inner_instruction_index FROM system_program_solana.system_program_call_Transfer + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT account_funding_account, account_recipient_account, lamports, call_tx_signer, call_block_time, call_tx_id, call_outer_executing_account, call_inner_instruction_index FROM system_program_solana.system_program_call_TransferWithSeed + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} +) \ No newline at end of file From 2e26fd83f88c50fb0491c7173832adc73cde4311 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Tue, 5 Dec 2023 23:54:23 -0500 Subject: [PATCH 02/23] add reader note --- models/tokens/solana/tokens_solana_transfers.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index 88abfe6716e..a42eeae09b0 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -84,6 +84,7 @@ LEFT JOIN {{ ref('solana_utils_token_accounts') }} tk_d ON tk_d.address = tr.acc UNION ALL +--for the reader, note that SOL is special and can be transferred without calling the transfer instruction. It is also minted and burned without instructions. So to get balances, use daily_balances or account_activity instead of transfers. SELECT call_block_time as block_time , 'transfer' as action From 845ea0b43f08d895152084c926a787483beb8783 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Tue, 5 Dec 2023 23:55:27 -0500 Subject: [PATCH 03/23] add 30 day filter first --- models/tokens/solana/tokens_solana_transfers.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index a42eeae09b0..29b6591b885 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -81,6 +81,7 @@ FROM ( --get token and accounts LEFT JOIN {{ ref('solana_utils_token_accounts') }} tk_s ON tk_s.address = tr.account_source LEFT JOIN {{ ref('solana_utils_token_accounts') }} tk_d ON tk_d.address = tr.account_destination +WHERE call_block_time > now() - interval '30' day UNION ALL @@ -112,4 +113,5 @@ FROM ( {% if is_incremental() %} AND {{incremental_predicate('call_block_time')}} {% endif %} -) \ No newline at end of file +) +WHERE call_block_time > now() - interval '30' day From 2c9ccd9334f7012fdd3753446f7cf44a6ce45ced Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Tue, 5 Dec 2023 23:57:21 -0500 Subject: [PATCH 04/23] remove changes --- models/solana_utils/solana_utils_schema.yml | 36 +-------------------- 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/models/solana_utils/solana_utils_schema.yml b/models/solana_utils/solana_utils_schema.yml index 531a1881e9a..0e2f083effa 100644 --- a/models/solana_utils/solana_utils_schema.yml +++ b/models/solana_utils/solana_utils_schema.yml @@ -84,38 +84,4 @@ models: - &favorite_domain name: favorite_domain - &owned_domains - name: owned_domains - - - name: solana_utils_transfers - meta: - blockchain: solana - contributors: [ilemi] - config: - tags: ['solana','transfers','erc20','nft','spl'] - description: > - get all spl and system token transfers - columns: - - &block_time - name: block_time - - &action - name: action - - &token_mint_address - name: token_mint_address - - &amount - name: amount - - &from_owner - name: from_owner - - &to_owner - name: to_owner - - &from_token_account - name: from_token_account - - &to_token_account - name: to_token_account - - &tx_signer - name: tx_signer - - &tx_id - name: tx_id - - &outer_instruction_index - name: outer_instruction_index - - &inner_instruction_index - name: inner_instruction_index \ No newline at end of file + name: owned_domains \ No newline at end of file From 7eeeccc5ee56c6a63214952bbd283e4d6923a369 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Wed, 6 Dec 2023 00:08:26 -0500 Subject: [PATCH 05/23] fixes --- models/tokens/solana/tokens_solana.sources.yml | 10 ++++++++++ models/tokens/solana/tokens_solana_schema.yml | 3 ++- .../tokens/solana/tokens_solana_transfers.sql | 18 +++++++++++------- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/models/tokens/solana/tokens_solana.sources.yml b/models/tokens/solana/tokens_solana.sources.yml index 000e978b31a..aeacb67070a 100644 --- a/models/tokens/solana/tokens_solana.sources.yml +++ b/models/tokens/solana/tokens_solana.sources.yml @@ -23,6 +23,16 @@ sources: loaded_at_field: call_block_time - name: spl_token_call_burnChecked loaded_at_field: call_block_time + - name: system_program_solana + description: "system program decoded tables" + freshness: # default freshness + warn_after: { count: 12, period: hour } + error_after: { count: 24, period: hour } + tables: + - name: system_program_call_Transfer + loaded_at_field: call_block_time + - name: system_program_call_TransferWithSeed + loaded_at_field: call_block_time - name: mpl_token_metadata_solana description: "mpl metadata decoded tables" freshness: # default freshness diff --git a/models/tokens/solana/tokens_solana_schema.yml b/models/tokens/solana/tokens_solana_schema.yml index 2f8ff24668c..95efde0d9c7 100644 --- a/models/tokens/solana/tokens_solana_schema.yml +++ b/models/tokens/solana/tokens_solana_schema.yml @@ -85,4 +85,5 @@ models: - name: tx_signer - name: tx_id - name: outer_instruction_index - - name: inner_instruction_index \ No newline at end of file + - name: inner_instruction_index + - name: outer_executing_account \ No newline at end of file diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index 29b6591b885..baea0831046 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -13,7 +13,7 @@ }} SELECT - tr.call_block_time as block_time + call_block_time as block_time , action , amount , COALESCE(tk_s.token_mint_address, tk_d.token_mint_address) as token_mint_address @@ -21,10 +21,11 @@ SELECT , tk_d.token_balance_owner as to_owner , account_source as from_token_account , account_destination as to_token_account - , tr.call_tx_signer as tx_signer - , tr.call_tx_id as tx_id - , tr.call_outer_instruction_index as outer_instruction_index - , tr.call_inner_instruction_index as inner_instruction_index + , call_tx_signer as tx_signer + , call_tx_id as tx_id + , call_outer_instruction_index as outer_instruction_index + , call_inner_instruction_index as inner_instruction_index + , call_outer_executing_account as outer_executing_account FROM ( SELECT account_source, account_destination, amount, call_tx_id, call_block_time, call_outer_executing_account, call_tx_signer, 'transfer' as action, call_outer_instruction_index, call_inner_instruction_index FROM {{ source('spl_token_solana','spl_token_call_transfer') }} @@ -99,8 +100,10 @@ SELECT , call_tx_id as tx_id , call_outer_instruction_index as outer_instruction_index , call_inner_instruction_index as inner_instruction_index + , call_outer_executing_account as outer_executing_account FROM ( - SELECT account_from, account_to, lamports, call_tx_signer, call_block_time, call_tx_id, call_outer_executing_account, call_inner_instruction_index FROM system_program_solana.system_program_call_Transfer + SELECT account_from, account_to, lamports, call_tx_signer, call_block_time, call_tx_id, call_outer_instruction_index, call_inner_instruction_index, call_outer_executing_account + FROM {{ source('system_program_solana','system_program_call_Transfer') }} WHERE 1=1 {% if is_incremental() %} AND {{incremental_predicate('call_block_time')}} @@ -108,7 +111,8 @@ FROM ( UNION ALL - SELECT account_funding_account, account_recipient_account, lamports, call_tx_signer, call_block_time, call_tx_id, call_outer_executing_account, call_inner_instruction_index FROM system_program_solana.system_program_call_TransferWithSeed + SELECT account_funding_account, account_recipient_account, lamports, call_tx_signer, call_block_time, call_tx_id, call_outer_instruction_index, call_inner_instruction_index, call_outer_executing_account + FROM {{ source('system_program_solana','system_program_call_TransferWithSeed') }} WHERE 1=1 {% if is_incremental() %} AND {{incremental_predicate('call_block_time')}} From 233058de478bce241c566f93257ad99082c822a9 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Wed, 6 Dec 2023 00:21:13 -0500 Subject: [PATCH 06/23] coalesce null to 0 --- models/tokens/solana/tokens_solana_transfers.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index baea0831046..f1cef95736c 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -24,7 +24,7 @@ SELECT , call_tx_signer as tx_signer , call_tx_id as tx_id , call_outer_instruction_index as outer_instruction_index - , call_inner_instruction_index as inner_instruction_index + , COALESCE(call_inner_instruction_index,0) as inner_instruction_index , call_outer_executing_account as outer_executing_account FROM ( SELECT account_source, account_destination, amount, call_tx_id, call_block_time, call_outer_executing_account, call_tx_signer, 'transfer' as action, call_outer_instruction_index, call_inner_instruction_index @@ -99,7 +99,7 @@ SELECT , call_tx_signer as tx_signer , call_tx_id as tx_id , call_outer_instruction_index as outer_instruction_index - , call_inner_instruction_index as inner_instruction_index + , COALESCE(call_inner_instruction_index,0) as inner_instruction_index , call_outer_executing_account as outer_executing_account FROM ( SELECT account_from, account_to, lamports, call_tx_signer, call_block_time, call_tx_id, call_outer_instruction_index, call_inner_instruction_index, call_outer_executing_account From 48a65d338386e37cc6788d6163d4b0ee50e97fd3 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Wed, 6 Dec 2023 00:44:25 -0500 Subject: [PATCH 07/23] just make token accounts unique, dgaf about the 214 weird ones --- models/solana_utils/solana_utils_token_accounts.sql | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/models/solana_utils/solana_utils_token_accounts.sql b/models/solana_utils/solana_utils_token_accounts.sql index a83b555089c..3cb45c9c39b 100644 --- a/models/solana_utils/solana_utils_token_accounts.sql +++ b/models/solana_utils/solana_utils_token_accounts.sql @@ -5,7 +5,7 @@ materialized = 'incremental', file_format = 'delta', incremental_strategy = 'merge', - unique_key = ['token_mint_address','address'], + unique_key = ['address'], post_hook='{{ expose_spells(\'["solana"]\', "sector", "solana_utils", @@ -15,16 +15,15 @@ WITH distinct_accounts as ( SELECT - aa.token_mint_address - , aa.address - , max_by(aa.token_balance_owner, aa.block_time) as token_balance_owner --some account created before and then again after token owner schema change, so then it created dupes. - , min(aa.block_time) as created_at + aa.address + , max_by(aa.token_balance_owner, aa.block_time) as token_balance_owner + , max_by(aa.token_mint_address, aa.block_time) as token_mint_address FROM {{ source('solana','account_activity') }} aa WHERE aa.token_mint_address is not null {% if is_incremental() %} AND {{incremental_predicate('aa.block_time')}} {% endif %} - group by 1,2 + group by 1 ) SELECT * FROM distinct_accounts \ No newline at end of file From a963c2a49e19b39b3d74b6664117e347c4b4e4ae Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Wed, 6 Dec 2023 08:31:45 -0500 Subject: [PATCH 08/23] try full history --- models/tokens/solana/tokens_solana_transfers.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index f1cef95736c..96021e9d3fe 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -82,7 +82,7 @@ FROM ( --get token and accounts LEFT JOIN {{ ref('solana_utils_token_accounts') }} tk_s ON tk_s.address = tr.account_source LEFT JOIN {{ ref('solana_utils_token_accounts') }} tk_d ON tk_d.address = tr.account_destination -WHERE call_block_time > now() - interval '30' day +-- WHERE call_block_time > now() - interval '30' day UNION ALL @@ -118,4 +118,4 @@ FROM ( AND {{incremental_predicate('call_block_time')}} {% endif %} ) -WHERE call_block_time > now() - interval '30' day +-- WHERE call_block_time > now() - interval '30' day From 85a27a5de0d794f9a7acb58c84cc7b3214d91ac8 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Wed, 6 Dec 2023 14:07:33 -0500 Subject: [PATCH 09/23] add predicate config --- models/tokens/solana/tokens_solana_transfers.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index 96021e9d3fe..8c14966f64b 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -5,6 +5,7 @@ materialized = 'incremental', file_format = 'delta', incremental_strategy = 'merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], unique_key = ['tx_id','outer_instruction_index','inner_instruction_index'], post_hook='{{ expose_spells(\'["solana"]\', "sector", From 80d38c2a3ea91607a59a70b2194d4f228223b627 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Wed, 6 Dec 2023 16:00:26 -0500 Subject: [PATCH 10/23] add block_time to partition? --- .../tokens/solana/tokens_solana_transfers.sql | 30 ++++--------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index 8c14966f64b..281d83e84c5 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -6,7 +6,7 @@ file_format = 'delta', incremental_strategy = 'merge', incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], - unique_key = ['tx_id','outer_instruction_index','inner_instruction_index'], + unique_key = ['tx_id','outer_instruction_index','inner_instruction_index','block_time'], post_hook='{{ expose_spells(\'["solana"]\', "sector", "tokens", @@ -30,59 +30,39 @@ SELECT FROM ( SELECT account_source, account_destination, amount, call_tx_id, call_block_time, call_outer_executing_account, call_tx_signer, 'transfer' as action, call_outer_instruction_index, call_inner_instruction_index FROM {{ source('spl_token_solana','spl_token_call_transfer') }} - WHERE 1=1 - {% if is_incremental() %} - AND {{incremental_predicate('call_block_time')}} - {% endif %} UNION ALL SELECT account_source, account_destination, amount, call_tx_id, call_block_time, call_outer_executing_account, call_tx_signer, 'transfer' as action, call_outer_instruction_index, call_inner_instruction_index FROM {{ source('spl_token_solana','spl_token_call_transferChecked') }} - WHERE 1=1 - {% if is_incremental() %} - AND {{incremental_predicate('call_block_time')}} - {% endif %} UNION ALL SELECT null, account_account as account_destination, amount, call_tx_id, call_block_time, call_outer_executing_account, call_tx_signer, 'mint' as action, call_outer_instruction_index, call_inner_instruction_index FROM {{ source('spl_token_solana','spl_token_call_mintTo') }} - WHERE 1=1 - {% if is_incremental() %} - AND {{incremental_predicate('call_block_time')}} - {% endif %} UNION ALL SELECT null, account_account as account_destination, amount, call_tx_id, call_block_time, call_outer_executing_account, call_tx_signer, 'mint' as action, call_outer_instruction_index, call_inner_instruction_index FROM {{ source('spl_token_solana','spl_token_call_mintToChecked') }} - WHERE 1=1 - {% if is_incremental() %} - AND {{incremental_predicate('call_block_time')}} - {% endif %} UNION ALL SELECT account_account as account_source, null, amount, call_tx_id, call_block_time, call_outer_executing_account, call_tx_signer, 'burn' as action, call_outer_instruction_index, call_inner_instruction_index FROM {{ source('spl_token_solana','spl_token_call_burn') }} - WHERE 1=1 - {% if is_incremental() %} - AND {{incremental_predicate('call_block_time')}} - {% endif %} UNION ALL SELECT account_account as account_source, null, amount, call_tx_id, call_block_time, call_outer_executing_account, call_tx_signer, 'burn' as action, call_outer_instruction_index, call_inner_instruction_index FROM {{ source('spl_token_solana','spl_token_call_burnChecked') }} - WHERE 1=1 - {% if is_incremental() %} - AND {{incremental_predicate('call_block_time')}} - {% endif %} ) tr --get token and accounts LEFT JOIN {{ ref('solana_utils_token_accounts') }} tk_s ON tk_s.address = tr.account_source LEFT JOIN {{ ref('solana_utils_token_accounts') }} tk_d ON tk_d.address = tr.account_destination +WHERE 1=1 +{% if is_incremental() %} +AND {{incremental_predicate('call_block_time')}} +{% endif %} -- WHERE call_block_time > now() - interval '30' day UNION ALL From ef23f48a19536aceb455c7fc01f732c80f2b7256 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:27:21 -0500 Subject: [PATCH 11/23] add block_slot instead --- models/tokens/solana/tokens_solana_schema.yml | 1 + .../tokens/solana/tokens_solana_transfers.sql | 20 ++++++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/models/tokens/solana/tokens_solana_schema.yml b/models/tokens/solana/tokens_solana_schema.yml index 95efde0d9c7..d9a6d3bffec 100644 --- a/models/tokens/solana/tokens_solana_schema.yml +++ b/models/tokens/solana/tokens_solana_schema.yml @@ -75,6 +75,7 @@ models: get all spl and system token transfers columns: - name: block_time + - name: block_slot - name: action - name: token_mint_address - name: amount diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index 281d83e84c5..4e5310da73d 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -6,7 +6,7 @@ file_format = 'delta', incremental_strategy = 'merge', incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], - unique_key = ['tx_id','outer_instruction_index','inner_instruction_index','block_time'], + unique_key = ['tx_id','outer_instruction_index','inner_instruction_index', 'block_slot'], post_hook='{{ expose_spells(\'["solana"]\', "sector", "tokens", @@ -15,6 +15,7 @@ SELECT call_block_time as block_time + , call_block_slot as block_slot , action , amount , COALESCE(tk_s.token_mint_address, tk_d.token_mint_address) as token_mint_address @@ -28,32 +29,32 @@ SELECT , COALESCE(call_inner_instruction_index,0) as inner_instruction_index , call_outer_executing_account as outer_executing_account FROM ( - SELECT account_source, account_destination, amount, call_tx_id, call_block_time, call_outer_executing_account, call_tx_signer, 'transfer' as action, call_outer_instruction_index, call_inner_instruction_index + SELECT account_source, account_destination, amount, call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer, 'transfer' as action, call_outer_instruction_index, call_inner_instruction_index FROM {{ source('spl_token_solana','spl_token_call_transfer') }} UNION ALL - SELECT account_source, account_destination, amount, call_tx_id, call_block_time, call_outer_executing_account, call_tx_signer, 'transfer' as action, call_outer_instruction_index, call_inner_instruction_index + SELECT account_source, account_destination, amount, call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer, 'transfer' as action, call_outer_instruction_index, call_inner_instruction_index FROM {{ source('spl_token_solana','spl_token_call_transferChecked') }} UNION ALL - SELECT null, account_account as account_destination, amount, call_tx_id, call_block_time, call_outer_executing_account, call_tx_signer, 'mint' as action, call_outer_instruction_index, call_inner_instruction_index + SELECT null, account_account as account_destination, amount, call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer, 'mint' as action, call_outer_instruction_index, call_inner_instruction_index FROM {{ source('spl_token_solana','spl_token_call_mintTo') }} UNION ALL - SELECT null, account_account as account_destination, amount, call_tx_id, call_block_time, call_outer_executing_account, call_tx_signer, 'mint' as action, call_outer_instruction_index, call_inner_instruction_index + SELECT null, account_account as account_destination, amount, call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer, 'mint' as action, call_outer_instruction_index, call_inner_instruction_index FROM {{ source('spl_token_solana','spl_token_call_mintToChecked') }} UNION ALL - SELECT account_account as account_source, null, amount, call_tx_id, call_block_time, call_outer_executing_account, call_tx_signer, 'burn' as action, call_outer_instruction_index, call_inner_instruction_index + SELECT account_account as account_source, null, amount, call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer, 'burn' as action, call_outer_instruction_index, call_inner_instruction_index FROM {{ source('spl_token_solana','spl_token_call_burn') }} UNION ALL - SELECT account_account as account_source, null, amount, call_tx_id, call_block_time, call_outer_executing_account, call_tx_signer, 'burn' as action, call_outer_instruction_index, call_inner_instruction_index + SELECT account_account as account_source, null, amount, call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer, 'burn' as action, call_outer_instruction_index, call_inner_instruction_index FROM {{ source('spl_token_solana','spl_token_call_burnChecked') }} ) tr --get token and accounts @@ -70,6 +71,7 @@ UNION ALL --for the reader, note that SOL is special and can be transferred without calling the transfer instruction. It is also minted and burned without instructions. So to get balances, use daily_balances or account_activity instead of transfers. SELECT call_block_time as block_time + , call_block_slot as block_slot , 'transfer' as action , lamports as amount --1e9 , 'native' as token_mint_address @@ -83,7 +85,7 @@ SELECT , COALESCE(call_inner_instruction_index,0) as inner_instruction_index , call_outer_executing_account as outer_executing_account FROM ( - SELECT account_from, account_to, lamports, call_tx_signer, call_block_time, call_tx_id, call_outer_instruction_index, call_inner_instruction_index, call_outer_executing_account + SELECT account_from, account_to, lamports, call_tx_signer, call_block_time, call_block_slot, call_tx_id, call_outer_instruction_index, call_inner_instruction_index, call_outer_executing_account FROM {{ source('system_program_solana','system_program_call_Transfer') }} WHERE 1=1 {% if is_incremental() %} @@ -92,7 +94,7 @@ FROM ( UNION ALL - SELECT account_funding_account, account_recipient_account, lamports, call_tx_signer, call_block_time, call_tx_id, call_outer_instruction_index, call_inner_instruction_index, call_outer_executing_account + SELECT account_funding_account, account_recipient_account, lamports, call_tx_signer, call_block_time, call_block_slot, call_tx_id, call_outer_instruction_index, call_inner_instruction_index, call_outer_executing_account FROM {{ source('system_program_solana','system_program_call_TransferWithSeed') }} WHERE 1=1 {% if is_incremental() %} From ec83f8a1e8258cc059be25d187fe617018178f1f Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Thu, 7 Dec 2023 09:03:26 -0500 Subject: [PATCH 12/23] add partition --- models/tokens/solana/tokens_solana_schema.yml | 1 + models/tokens/solana/tokens_solana_transfers.sql | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/models/tokens/solana/tokens_solana_schema.yml b/models/tokens/solana/tokens_solana_schema.yml index d9a6d3bffec..65822c7c8d2 100644 --- a/models/tokens/solana/tokens_solana_schema.yml +++ b/models/tokens/solana/tokens_solana_schema.yml @@ -75,6 +75,7 @@ models: get all spl and system token transfers columns: - name: block_time + - name: block_date - name: block_slot - name: action - name: token_mint_address diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index 4e5310da73d..e2bd4fe9ac6 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -1,7 +1,8 @@ {{ config( - schema = 'tokens', + schema = 'tokens_solana', alias = 'transfers', + partition_by = ['block_date'], materialized = 'incremental', file_format = 'delta', incremental_strategy = 'merge', @@ -15,6 +16,7 @@ SELECT call_block_time as block_time + , date_trunc('day', call_block_time) as block_date , call_block_slot as block_slot , action , amount @@ -71,6 +73,7 @@ UNION ALL --for the reader, note that SOL is special and can be transferred without calling the transfer instruction. It is also minted and burned without instructions. So to get balances, use daily_balances or account_activity instead of transfers. SELECT call_block_time as block_time + , date_trunc('day', call_block_time) as block_date , call_block_slot as block_slot , 'transfer' as action , lamports as amount --1e9 From 29eb2618617607c701a73a620657002e24493357 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Thu, 7 Dec 2023 09:52:02 -0500 Subject: [PATCH 13/23] remove native SOL transfers --- .../tokens/solana/tokens_solana_transfers.sql | 67 ++++++++++--------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index e2bd4fe9ac6..81d5d300a5b 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -68,40 +68,41 @@ AND {{incremental_predicate('call_block_time')}} {% endif %} -- WHERE call_block_time > now() - interval '30' day -UNION ALL +-- -- adding in SOL/native transfers seems to blow out the CI +-- UNION ALL ---for the reader, note that SOL is special and can be transferred without calling the transfer instruction. It is also minted and burned without instructions. So to get balances, use daily_balances or account_activity instead of transfers. -SELECT - call_block_time as block_time - , date_trunc('day', call_block_time) as block_date - , call_block_slot as block_slot - , 'transfer' as action - , lamports as amount --1e9 - , 'native' as token_mint_address - , account_from as from_owner - , account_to as to_owner - , account_from as from_token_account - , account_to as to_token_account - , call_tx_signer as tx_signer - , call_tx_id as tx_id - , call_outer_instruction_index as outer_instruction_index - , COALESCE(call_inner_instruction_index,0) as inner_instruction_index - , call_outer_executing_account as outer_executing_account -FROM ( - SELECT account_from, account_to, lamports, call_tx_signer, call_block_time, call_block_slot, call_tx_id, call_outer_instruction_index, call_inner_instruction_index, call_outer_executing_account - FROM {{ source('system_program_solana','system_program_call_Transfer') }} - WHERE 1=1 - {% if is_incremental() %} - AND {{incremental_predicate('call_block_time')}} - {% endif %} +-- --for the reader, note that SOL is special and can be transferred without calling the transfer instruction. It is also minted and burned without instructions. So to get balances, use daily_balances or account_activity instead of transfers. +-- SELECT +-- call_block_time as block_time +-- , date_trunc('day', call_block_time) as block_date +-- , call_block_slot as block_slot +-- , 'transfer' as action +-- , lamports as amount --1e9 +-- , 'native' as token_mint_address +-- , account_from as from_owner +-- , account_to as to_owner +-- , account_from as from_token_account +-- , account_to as to_token_account +-- , call_tx_signer as tx_signer +-- , call_tx_id as tx_id +-- , call_outer_instruction_index as outer_instruction_index +-- , COALESCE(call_inner_instruction_index,0) as inner_instruction_index +-- , call_outer_executing_account as outer_executing_account +-- FROM ( +-- SELECT account_from, account_to, lamports, call_tx_signer, call_block_time, call_block_slot, call_tx_id, call_outer_instruction_index, call_inner_instruction_index, call_outer_executing_account +-- FROM {{ source('system_program_solana','system_program_call_Transfer') }} +-- WHERE 1=1 +-- {% if is_incremental() %} +-- AND {{incremental_predicate('call_block_time')}} +-- {% endif %} - UNION ALL +-- UNION ALL - SELECT account_funding_account, account_recipient_account, lamports, call_tx_signer, call_block_time, call_block_slot, call_tx_id, call_outer_instruction_index, call_inner_instruction_index, call_outer_executing_account - FROM {{ source('system_program_solana','system_program_call_TransferWithSeed') }} - WHERE 1=1 - {% if is_incremental() %} - AND {{incremental_predicate('call_block_time')}} - {% endif %} -) +-- SELECT account_funding_account, account_recipient_account, lamports, call_tx_signer, call_block_time, call_block_slot, call_tx_id, call_outer_instruction_index, call_inner_instruction_index, call_outer_executing_account +-- FROM {{ source('system_program_solana','system_program_call_TransferWithSeed') }} +-- WHERE 1=1 +-- {% if is_incremental() %} +-- AND {{incremental_predicate('call_block_time')}} +-- {% endif %} +-- ) -- WHERE call_block_time > now() - interval '30' day From 97f812b278b2d748d2c305baa7ab0363e078f808 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Thu, 7 Dec 2023 09:57:34 -0500 Subject: [PATCH 14/23] add in native transfers model --- .../solana/tokens_solana_native_transfers.sql | 46 +++++++++++++++++++ models/tokens/solana/tokens_solana_schema.yml | 27 ++++++++++- 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 models/tokens/solana/tokens_solana_native_transfers.sql diff --git a/models/tokens/solana/tokens_solana_native_transfers.sql b/models/tokens/solana/tokens_solana_native_transfers.sql new file mode 100644 index 00000000000..33603176960 --- /dev/null +++ b/models/tokens/solana/tokens_solana_native_transfers.sql @@ -0,0 +1,46 @@ + {{ + config( + schema = 'tokens_solana', + alias = 'native_transfers', + partition_by = ['block_date'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + unique_key = ['tx_id','outer_instruction_index','inner_instruction_index', 'block_slot'], + post_hook='{{ expose_spells(\'["solana"]\', + "sector", + "tokens", + \'["ilemi"]\') }}') +}} + +--for the reader, note that SOL is special and can be transferred without calling the transfer instruction. It is also minted and burned without instructions. So to get balances, use daily_balances or account_activity instead of transfers. +SELECT + call_block_time as block_time + , date_trunc('day', call_block_time) as block_date + , call_block_slot as block_slot + , 'transfer' as action + , lamports as amount --1e9 + , 'native' as token_mint_address + , account_from as from_owner + , account_to as to_owner + , account_from as from_token_account + , account_to as to_token_account + , call_tx_signer as tx_signer + , call_tx_id as tx_id + , call_outer_instruction_index as outer_instruction_index + , COALESCE(call_inner_instruction_index,0) as inner_instruction_index + , call_outer_executing_account as outer_executing_account +FROM ( + SELECT account_from, account_to, lamports, call_tx_signer, call_block_time, call_block_slot, call_tx_id, call_outer_instruction_index, call_inner_instruction_index, call_outer_executing_account + FROM {{ source('system_program_solana','system_program_call_Transfer') }} + + UNION ALL + + SELECT account_funding_account, account_recipient_account, lamports, call_tx_signer, call_block_time, call_block_slot, call_tx_id, call_outer_instruction_index, call_inner_instruction_index, call_outer_executing_account + FROM {{ source('system_program_solana','system_program_call_TransferWithSeed') }} +) +WHERE 1=1 +{% if is_incremental() %} +AND {{incremental_predicate('call_block_time')}} +{% endif %} \ No newline at end of file diff --git a/models/tokens/solana/tokens_solana_schema.yml b/models/tokens/solana/tokens_solana_schema.yml index 65822c7c8d2..8e4c89a7820 100644 --- a/models/tokens/solana/tokens_solana_schema.yml +++ b/models/tokens/solana/tokens_solana_schema.yml @@ -72,7 +72,32 @@ models: config: tags: ['solana','transfers','erc20','nft','spl'] description: > - get all spl and system token transfers + get all spl token transfers (will add in token2022 later) + columns: + - name: block_time + - name: block_date + - name: block_slot + - name: action + - name: token_mint_address + - name: amount + - name: from_owner + - name: to_owner + - name: from_token_account + - name: to_token_account + - name: tx_signer + - name: tx_id + - name: outer_instruction_index + - name: inner_instruction_index + - name: outer_executing_account + + - name: tokens_solana_native_transfers + meta: + blockchain: solana + contributors: [ilemi] + config: + tags: ['solana','transfers','native','sol'] + description: > + get all system SOL (native) transfers. columns: - name: block_time - name: block_date From 69ad775e7d55967d87e0c7a9ab61b786d3ac2dca Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Thu, 7 Dec 2023 11:46:06 -0500 Subject: [PATCH 15/23] remove partition --- .../solana/tokens_solana_native_transfers.sql | 4 +- models/tokens/solana/tokens_solana_schema.yml | 4 +- .../tokens/solana/tokens_solana_transfers.sql | 45 ++----------------- 3 files changed, 7 insertions(+), 46 deletions(-) diff --git a/models/tokens/solana/tokens_solana_native_transfers.sql b/models/tokens/solana/tokens_solana_native_transfers.sql index 33603176960..633920c9b28 100644 --- a/models/tokens/solana/tokens_solana_native_transfers.sql +++ b/models/tokens/solana/tokens_solana_native_transfers.sql @@ -2,7 +2,7 @@ config( schema = 'tokens_solana', alias = 'native_transfers', - partition_by = ['block_date'], + -- partition_by = ['block_date'], materialized = 'incremental', file_format = 'delta', incremental_strategy = 'merge', @@ -17,7 +17,7 @@ --for the reader, note that SOL is special and can be transferred without calling the transfer instruction. It is also minted and burned without instructions. So to get balances, use daily_balances or account_activity instead of transfers. SELECT call_block_time as block_time - , date_trunc('day', call_block_time) as block_date +-- , date_trunc('day', call_block_time) as block_date , call_block_slot as block_slot , 'transfer' as action , lamports as amount --1e9 diff --git a/models/tokens/solana/tokens_solana_schema.yml b/models/tokens/solana/tokens_solana_schema.yml index 8e4c89a7820..a4c1e6d2550 100644 --- a/models/tokens/solana/tokens_solana_schema.yml +++ b/models/tokens/solana/tokens_solana_schema.yml @@ -75,7 +75,7 @@ models: get all spl token transfers (will add in token2022 later) columns: - name: block_time - - name: block_date + # - name: block_date - name: block_slot - name: action - name: token_mint_address @@ -100,7 +100,7 @@ models: get all system SOL (native) transfers. columns: - name: block_time - - name: block_date + # - name: block_date - name: block_slot - name: action - name: token_mint_address diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index 81d5d300a5b..f4dc27277fe 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -2,7 +2,7 @@ config( schema = 'tokens_solana', alias = 'transfers', - partition_by = ['block_date'], + -- partition_by = ['block_date'], materialized = 'incremental', file_format = 'delta', incremental_strategy = 'merge', @@ -16,7 +16,7 @@ SELECT call_block_time as block_time - , date_trunc('day', call_block_time) as block_date +-- , date_trunc('day', call_block_time) as block_date , call_block_slot as block_slot , action , amount @@ -66,43 +66,4 @@ WHERE 1=1 {% if is_incremental() %} AND {{incremental_predicate('call_block_time')}} {% endif %} --- WHERE call_block_time > now() - interval '30' day - --- -- adding in SOL/native transfers seems to blow out the CI --- UNION ALL - --- --for the reader, note that SOL is special and can be transferred without calling the transfer instruction. It is also minted and burned without instructions. So to get balances, use daily_balances or account_activity instead of transfers. --- SELECT --- call_block_time as block_time --- , date_trunc('day', call_block_time) as block_date --- , call_block_slot as block_slot --- , 'transfer' as action --- , lamports as amount --1e9 --- , 'native' as token_mint_address --- , account_from as from_owner --- , account_to as to_owner --- , account_from as from_token_account --- , account_to as to_token_account --- , call_tx_signer as tx_signer --- , call_tx_id as tx_id --- , call_outer_instruction_index as outer_instruction_index --- , COALESCE(call_inner_instruction_index,0) as inner_instruction_index --- , call_outer_executing_account as outer_executing_account --- FROM ( --- SELECT account_from, account_to, lamports, call_tx_signer, call_block_time, call_block_slot, call_tx_id, call_outer_instruction_index, call_inner_instruction_index, call_outer_executing_account --- FROM {{ source('system_program_solana','system_program_call_Transfer') }} --- WHERE 1=1 --- {% if is_incremental() %} --- AND {{incremental_predicate('call_block_time')}} --- {% endif %} - --- UNION ALL - --- SELECT account_funding_account, account_recipient_account, lamports, call_tx_signer, call_block_time, call_block_slot, call_tx_id, call_outer_instruction_index, call_inner_instruction_index, call_outer_executing_account --- FROM {{ source('system_program_solana','system_program_call_TransferWithSeed') }} --- WHERE 1=1 --- {% if is_incremental() %} --- AND {{incremental_predicate('call_block_time')}} --- {% endif %} --- ) --- WHERE call_block_time > now() - interval '30' day +-- WHERE call_block_time > now() - interval '30' day \ No newline at end of file From eb26dd76a83600acbb9cdd5bb9ab5355c124cd6b Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Thu, 7 Dec 2023 11:48:35 -0500 Subject: [PATCH 16/23] rem --- models/tokens/solana/tokens_solana_native_transfers.sql | 2 +- models/tokens/solana/tokens_solana_transfers.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/tokens/solana/tokens_solana_native_transfers.sql b/models/tokens/solana/tokens_solana_native_transfers.sql index 633920c9b28..c84f61b9d6c 100644 --- a/models/tokens/solana/tokens_solana_native_transfers.sql +++ b/models/tokens/solana/tokens_solana_native_transfers.sql @@ -2,7 +2,6 @@ config( schema = 'tokens_solana', alias = 'native_transfers', - -- partition_by = ['block_date'], materialized = 'incremental', file_format = 'delta', incremental_strategy = 'merge', @@ -13,6 +12,7 @@ "tokens", \'["ilemi"]\') }}') }} +-- partition_by = ['block_date'], --for the reader, note that SOL is special and can be transferred without calling the transfer instruction. It is also minted and burned without instructions. So to get balances, use daily_balances or account_activity instead of transfers. SELECT diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index f4dc27277fe..56fbdc37b7c 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -2,7 +2,6 @@ config( schema = 'tokens_solana', alias = 'transfers', - -- partition_by = ['block_date'], materialized = 'incremental', file_format = 'delta', incremental_strategy = 'merge', @@ -13,6 +12,7 @@ "tokens", \'["ilemi"]\') }}') }} +-- partition_by = ['block_date'], SELECT call_block_time as block_time From 5ac007ebbbf52f9739f590a9fd18f3e9d47f3cc6 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Thu, 7 Dec 2023 23:27:56 -0500 Subject: [PATCH 17/23] try 30 --- models/tokens/solana/tokens_solana_native_transfers.sql | 3 ++- models/tokens/solana/tokens_solana_transfers.sql | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/models/tokens/solana/tokens_solana_native_transfers.sql b/models/tokens/solana/tokens_solana_native_transfers.sql index c84f61b9d6c..5565f04ec82 100644 --- a/models/tokens/solana/tokens_solana_native_transfers.sql +++ b/models/tokens/solana/tokens_solana_native_transfers.sql @@ -43,4 +43,5 @@ FROM ( WHERE 1=1 {% if is_incremental() %} AND {{incremental_predicate('call_block_time')}} -{% endif %} \ No newline at end of file +{% endif %} +AND call_block_time > now() - interval '30' day \ No newline at end of file diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index 56fbdc37b7c..28473c7571a 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -66,4 +66,4 @@ WHERE 1=1 {% if is_incremental() %} AND {{incremental_predicate('call_block_time')}} {% endif %} --- WHERE call_block_time > now() - interval '30' day \ No newline at end of file +AND call_block_time > now() - interval '30' day \ No newline at end of file From 693468e350ce9b35b304b923fd1b528c881169e8 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Fri, 8 Dec 2023 00:32:41 -0500 Subject: [PATCH 18/23] try 300 day --- models/tokens/solana/tokens_solana_native_transfers.sql | 2 +- models/tokens/solana/tokens_solana_transfers.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/tokens/solana/tokens_solana_native_transfers.sql b/models/tokens/solana/tokens_solana_native_transfers.sql index 5565f04ec82..809d503ca70 100644 --- a/models/tokens/solana/tokens_solana_native_transfers.sql +++ b/models/tokens/solana/tokens_solana_native_transfers.sql @@ -44,4 +44,4 @@ WHERE 1=1 {% if is_incremental() %} AND {{incremental_predicate('call_block_time')}} {% endif %} -AND call_block_time > now() - interval '30' day \ No newline at end of file +AND call_block_time > now() - interval '300' day \ No newline at end of file diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index 28473c7571a..8b0201a640c 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -66,4 +66,4 @@ WHERE 1=1 {% if is_incremental() %} AND {{incremental_predicate('call_block_time')}} {% endif %} -AND call_block_time > now() - interval '30' day \ No newline at end of file +AND call_block_time > now() - interval '300' day \ No newline at end of file From d7ceadaf2b8db06b78e81580b97e47ea2cebba8b Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Fri, 8 Dec 2023 01:36:18 -0500 Subject: [PATCH 19/23] try 600 --- models/tokens/solana/tokens_solana_native_transfers.sql | 2 +- models/tokens/solana/tokens_solana_transfers.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/tokens/solana/tokens_solana_native_transfers.sql b/models/tokens/solana/tokens_solana_native_transfers.sql index 809d503ca70..5cbcb9d013e 100644 --- a/models/tokens/solana/tokens_solana_native_transfers.sql +++ b/models/tokens/solana/tokens_solana_native_transfers.sql @@ -44,4 +44,4 @@ WHERE 1=1 {% if is_incremental() %} AND {{incremental_predicate('call_block_time')}} {% endif %} -AND call_block_time > now() - interval '300' day \ No newline at end of file +AND call_block_time > now() - interval '600' day \ No newline at end of file diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index 8b0201a640c..1ed2a94c1f5 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -66,4 +66,4 @@ WHERE 1=1 {% if is_incremental() %} AND {{incremental_predicate('call_block_time')}} {% endif %} -AND call_block_time > now() - interval '300' day \ No newline at end of file +AND call_block_time > now() - interval '600' day \ No newline at end of file From 64fb7e08f89c313a2555a51f9d6a1fa87c54816d Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Fri, 8 Dec 2023 07:40:19 -0500 Subject: [PATCH 20/23] add back in partition, try all history again --- models/tokens/solana/tokens_solana_native_transfers.sql | 6 +++--- models/tokens/solana/tokens_solana_schema.yml | 2 +- models/tokens/solana/tokens_solana_transfers.sql | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/models/tokens/solana/tokens_solana_native_transfers.sql b/models/tokens/solana/tokens_solana_native_transfers.sql index 5cbcb9d013e..59f40b2eb71 100644 --- a/models/tokens/solana/tokens_solana_native_transfers.sql +++ b/models/tokens/solana/tokens_solana_native_transfers.sql @@ -6,18 +6,18 @@ file_format = 'delta', incremental_strategy = 'merge', incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + partition_by = ['block_date'], unique_key = ['tx_id','outer_instruction_index','inner_instruction_index', 'block_slot'], post_hook='{{ expose_spells(\'["solana"]\', "sector", "tokens", \'["ilemi"]\') }}') }} --- partition_by = ['block_date'], --for the reader, note that SOL is special and can be transferred without calling the transfer instruction. It is also minted and burned without instructions. So to get balances, use daily_balances or account_activity instead of transfers. SELECT call_block_time as block_time --- , date_trunc('day', call_block_time) as block_date + , cast (date_trunc('day', call_block_time) as date) as block_date , call_block_slot as block_slot , 'transfer' as action , lamports as amount --1e9 @@ -44,4 +44,4 @@ WHERE 1=1 {% if is_incremental() %} AND {{incremental_predicate('call_block_time')}} {% endif %} -AND call_block_time > now() - interval '600' day \ No newline at end of file +-- AND call_block_time > now() - interval '600' day \ No newline at end of file diff --git a/models/tokens/solana/tokens_solana_schema.yml b/models/tokens/solana/tokens_solana_schema.yml index a4c1e6d2550..ad7d48b7671 100644 --- a/models/tokens/solana/tokens_solana_schema.yml +++ b/models/tokens/solana/tokens_solana_schema.yml @@ -75,7 +75,7 @@ models: get all spl token transfers (will add in token2022 later) columns: - name: block_time - # - name: block_date + - name: block_date - name: block_slot - name: action - name: token_mint_address diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index 1ed2a94c1f5..4bd6ece5972 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -6,17 +6,17 @@ file_format = 'delta', incremental_strategy = 'merge', incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + partition_by = ['block_date'], unique_key = ['tx_id','outer_instruction_index','inner_instruction_index', 'block_slot'], post_hook='{{ expose_spells(\'["solana"]\', "sector", "tokens", \'["ilemi"]\') }}') }} --- partition_by = ['block_date'], SELECT call_block_time as block_time --- , date_trunc('day', call_block_time) as block_date + , cast (date_trunc('day', call_block_time) as date) as block_date , call_block_slot as block_slot , action , amount @@ -66,4 +66,4 @@ WHERE 1=1 {% if is_incremental() %} AND {{incremental_predicate('call_block_time')}} {% endif %} -AND call_block_time > now() - interval '600' day \ No newline at end of file +-- AND call_block_time > now() - interval '600' day \ No newline at end of file From f273fd39dec92876ab137509813242e148dce011 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Fri, 8 Dec 2023 13:57:38 -0500 Subject: [PATCH 21/23] remove partition again --- models/tokens/solana/tokens_solana_native_transfers.sql | 3 +-- models/tokens/solana/tokens_solana_schema.yml | 2 +- models/tokens/solana/tokens_solana_transfers.sql | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/models/tokens/solana/tokens_solana_native_transfers.sql b/models/tokens/solana/tokens_solana_native_transfers.sql index 59f40b2eb71..939b9ab05b4 100644 --- a/models/tokens/solana/tokens_solana_native_transfers.sql +++ b/models/tokens/solana/tokens_solana_native_transfers.sql @@ -6,7 +6,6 @@ file_format = 'delta', incremental_strategy = 'merge', incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], - partition_by = ['block_date'], unique_key = ['tx_id','outer_instruction_index','inner_instruction_index', 'block_slot'], post_hook='{{ expose_spells(\'["solana"]\', "sector", @@ -17,7 +16,7 @@ --for the reader, note that SOL is special and can be transferred without calling the transfer instruction. It is also minted and burned without instructions. So to get balances, use daily_balances or account_activity instead of transfers. SELECT call_block_time as block_time - , cast (date_trunc('day', call_block_time) as date) as block_date +-- , cast (date_trunc('day', call_block_time) as date) as block_date , call_block_slot as block_slot , 'transfer' as action , lamports as amount --1e9 diff --git a/models/tokens/solana/tokens_solana_schema.yml b/models/tokens/solana/tokens_solana_schema.yml index ad7d48b7671..a4c1e6d2550 100644 --- a/models/tokens/solana/tokens_solana_schema.yml +++ b/models/tokens/solana/tokens_solana_schema.yml @@ -75,7 +75,7 @@ models: get all spl token transfers (will add in token2022 later) columns: - name: block_time - - name: block_date + # - name: block_date - name: block_slot - name: action - name: token_mint_address diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index 4bd6ece5972..21f9e89c1d5 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -6,17 +6,17 @@ file_format = 'delta', incremental_strategy = 'merge', incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], - partition_by = ['block_date'], unique_key = ['tx_id','outer_instruction_index','inner_instruction_index', 'block_slot'], post_hook='{{ expose_spells(\'["solana"]\', "sector", "tokens", \'["ilemi"]\') }}') }} +-- partition_by = ['block_date'], SELECT call_block_time as block_time - , cast (date_trunc('day', call_block_time) as date) as block_date +-- , cast (date_trunc('day', call_block_time) as date) as block_date , call_block_slot as block_slot , action , amount From 363855a852efe9e1d43c45f4bad8a718a8b454e5 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Sun, 10 Dec 2023 07:17:03 -0500 Subject: [PATCH 22/23] remove system transfers --- .../solana/tokens_solana_native_transfers.sql | 46 ------------------- models/tokens/solana/tokens_solana_schema.yml | 25 ---------- 2 files changed, 71 deletions(-) delete mode 100644 models/tokens/solana/tokens_solana_native_transfers.sql diff --git a/models/tokens/solana/tokens_solana_native_transfers.sql b/models/tokens/solana/tokens_solana_native_transfers.sql deleted file mode 100644 index 939b9ab05b4..00000000000 --- a/models/tokens/solana/tokens_solana_native_transfers.sql +++ /dev/null @@ -1,46 +0,0 @@ - {{ - config( - schema = 'tokens_solana', - alias = 'native_transfers', - materialized = 'incremental', - file_format = 'delta', - incremental_strategy = 'merge', - incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], - unique_key = ['tx_id','outer_instruction_index','inner_instruction_index', 'block_slot'], - post_hook='{{ expose_spells(\'["solana"]\', - "sector", - "tokens", - \'["ilemi"]\') }}') -}} - ---for the reader, note that SOL is special and can be transferred without calling the transfer instruction. It is also minted and burned without instructions. So to get balances, use daily_balances or account_activity instead of transfers. -SELECT - call_block_time as block_time --- , cast (date_trunc('day', call_block_time) as date) as block_date - , call_block_slot as block_slot - , 'transfer' as action - , lamports as amount --1e9 - , 'native' as token_mint_address - , account_from as from_owner - , account_to as to_owner - , account_from as from_token_account - , account_to as to_token_account - , call_tx_signer as tx_signer - , call_tx_id as tx_id - , call_outer_instruction_index as outer_instruction_index - , COALESCE(call_inner_instruction_index,0) as inner_instruction_index - , call_outer_executing_account as outer_executing_account -FROM ( - SELECT account_from, account_to, lamports, call_tx_signer, call_block_time, call_block_slot, call_tx_id, call_outer_instruction_index, call_inner_instruction_index, call_outer_executing_account - FROM {{ source('system_program_solana','system_program_call_Transfer') }} - - UNION ALL - - SELECT account_funding_account, account_recipient_account, lamports, call_tx_signer, call_block_time, call_block_slot, call_tx_id, call_outer_instruction_index, call_inner_instruction_index, call_outer_executing_account - FROM {{ source('system_program_solana','system_program_call_TransferWithSeed') }} -) -WHERE 1=1 -{% if is_incremental() %} -AND {{incremental_predicate('call_block_time')}} -{% endif %} --- AND call_block_time > now() - interval '600' day \ No newline at end of file diff --git a/models/tokens/solana/tokens_solana_schema.yml b/models/tokens/solana/tokens_solana_schema.yml index a4c1e6d2550..ad5781a9375 100644 --- a/models/tokens/solana/tokens_solana_schema.yml +++ b/models/tokens/solana/tokens_solana_schema.yml @@ -73,31 +73,6 @@ models: tags: ['solana','transfers','erc20','nft','spl'] description: > get all spl token transfers (will add in token2022 later) - columns: - - name: block_time - # - name: block_date - - name: block_slot - - name: action - - name: token_mint_address - - name: amount - - name: from_owner - - name: to_owner - - name: from_token_account - - name: to_token_account - - name: tx_signer - - name: tx_id - - name: outer_instruction_index - - name: inner_instruction_index - - name: outer_executing_account - - - name: tokens_solana_native_transfers - meta: - blockchain: solana - contributors: [ilemi] - config: - tags: ['solana','transfers','native','sol'] - description: > - get all system SOL (native) transfers. columns: - name: block_time # - name: block_date From 2d48b4b0b01b043aaab9a8d48380b23986279daf Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Sun, 10 Dec 2023 09:08:27 -0500 Subject: [PATCH 23/23] add partition back in --- models/tokens/solana/tokens_solana_schema.yml | 2 +- models/tokens/solana/tokens_solana_transfers.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/tokens/solana/tokens_solana_schema.yml b/models/tokens/solana/tokens_solana_schema.yml index ad5781a9375..ab3b17f52e5 100644 --- a/models/tokens/solana/tokens_solana_schema.yml +++ b/models/tokens/solana/tokens_solana_schema.yml @@ -75,7 +75,7 @@ models: get all spl token transfers (will add in token2022 later) columns: - name: block_time - # - name: block_date + - name: block_date - name: block_slot - name: action - name: token_mint_address diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index 21f9e89c1d5..a28903cf3d4 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -5,6 +5,7 @@ materialized = 'incremental', file_format = 'delta', incremental_strategy = 'merge', + partition_by = ['block_date'], incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], unique_key = ['tx_id','outer_instruction_index','inner_instruction_index', 'block_slot'], post_hook='{{ expose_spells(\'["solana"]\', @@ -12,11 +13,10 @@ "tokens", \'["ilemi"]\') }}') }} --- partition_by = ['block_date'], SELECT call_block_time as block_time --- , cast (date_trunc('day', call_block_time) as date) as block_date + , cast (date_trunc('day', call_block_time) as date) as block_date , call_block_slot as block_slot , action , amount