From f9293e2043e55083c811f08f7a10579e50807a4a Mon Sep 17 00:00:00 2001 From: jeff-dude Date: Thu, 6 Jun 2024 12:47:09 -0400 Subject: [PATCH 01/14] add back changes for solana --- .../tokens/solana/tokens_solana.sources.yml | 23 ++ .../tokens/solana/tokens_solana_fungible.sql | 158 ++++++++-- models/tokens/solana/tokens_solana_schema.yml | 11 +- .../tokens/solana/tokens_solana_transfers.sql | 279 ++++++++++++++++-- 4 files changed, 415 insertions(+), 56 deletions(-) diff --git a/models/tokens/solana/tokens_solana.sources.yml b/models/tokens/solana/tokens_solana.sources.yml index 0636d8254fe..5a17103eee3 100644 --- a/models/tokens/solana/tokens_solana.sources.yml +++ b/models/tokens/solana/tokens_solana.sources.yml @@ -24,6 +24,29 @@ sources: - name: spl_token_call_burnChecked loaded_at_field: call_block_time + - name: spl_token_2022_solana + description: "spl_token 2022 decoded tables" + freshness: # default freshness + warn_after: { count: 12, period: hour } + error_after: { count: 24, period: hour } + tables: + - name: spl_token_2022_call_initializeMint + loaded_at_field: call_block_time + - name: spl_token_2022_call_initializeMint2 + loaded_at_field: call_block_time + - name: spl_token_2022_call_transferFeeExtension + loaded_at_field: call_block_time + - name: spl_token_2022_call_transferChecked + loaded_at_field: call_block_time + - name: spl_token_2022_call_mintTo + loaded_at_field: call_block_time + - name: spl_token_2022_call_mintToChecked + loaded_at_field: call_block_time + - name: spl_token_2022_call_burn + loaded_at_field: call_block_time + - name: spl_token_2022_call_burnChecked + loaded_at_field: call_block_time + - name: mpl_token_metadata_solana description: "mpl metadata decoded tables" freshness: # default freshness diff --git a/models/tokens/solana/tokens_solana_fungible.sql b/models/tokens/solana/tokens_solana_fungible.sql index 5f26d007d6c..6a66471188c 100644 --- a/models/tokens/solana/tokens_solana_fungible.sql +++ b/models/tokens/solana/tokens_solana_fungible.sql @@ -1,7 +1,7 @@ {{ config ( alias = 'fungible', - + post_hook='{{ expose_spells(\'["solana"]\', "sector", "tokens", @@ -10,26 +10,32 @@ }} -with +with tokens as ( SELECT bytearray_to_bigint(bytearray_reverse(bytearray_substring(call_data, 2, 1))) as decimals , call_data , account_mint + , token_version , call_tx_id , call_block_time + , row_number() over (partition by account_mint order by call_block_time desc) as latest FROM ( - SELECT call_data, account_mint, call_tx_id, call_block_time FROM {{ source('spl_token_solana', 'spl_token_call_initializeMint') }} - UNION ALL - SELECT call_data, account_mint, call_tx_id, call_block_time FROM {{ source('spl_token_solana', 'spl_token_call_initializeMint2') }} + SELECT call_data, account_mint, call_tx_id, call_block_time, 'spl_token' as token_version FROM {{ source('spl_token_solana', 'spl_token_call_initializeMint') }} + UNION ALL + SELECT call_data, account_mint, call_tx_id, call_block_time, 'spl_token' as token_version FROM {{ source('spl_token_solana', 'spl_token_call_initializeMint2') }} + UNION ALL + SELECT call_data, account_mint, call_tx_id, call_block_time, 'token2022' as token_version FROM {{ source('spl_token_2022_solana', 'spl_token_2022_call_initializeMint') }} + UNION ALL + SELECT call_data, account_mint, call_tx_id, call_block_time, 'token2022' as token_version FROM {{ source('spl_token_2022_solana', 'spl_token_2022_call_initializeMint2') }} ) {% if is_incremental() %} - where call_block_time >= date_trunc('day', now() - interval '7' day) + where {{ incremental_predicate('call_block_time') }} {% endif %} ) - + , metadata as ( - SELECT + SELECT meta.call_tx_id , meta.call_block_slot , meta.call_block_time @@ -38,8 +44,10 @@ with , meta.account_mint , meta.call_block_time , master.account_edition as master_edition + , metadata_program + , row_number() over (partition by meta.account_mint order by meta.call_block_time desc) as latest FROM ( - SELECT + SELECT call_tx_id , call_outer_instruction_index , call_inner_instruction_index @@ -48,9 +56,10 @@ with , json_query(createMetadataAccountArgs, 'lax $.CreateMetadataAccountArgs.data.Data') as args , account_metadata , account_mint + , call_executing_account as metadata_program FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMetadataAccount') }} - UNION ALL - SELECT + UNION ALL + SELECT call_tx_id , call_outer_instruction_index , call_inner_instruction_index @@ -59,51 +68,143 @@ with , json_query(createMetadataAccountArgsV2, 'lax $.CreateMetadataAccountArgsV2.data.DataV2') as args , account_metadata , account_mint + , call_executing_account as metadata_program FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMetadataAccountV2') }} - UNION ALL - SELECT + UNION ALL + SELECT call_tx_id , call_outer_instruction_index - , call_inner_instruction_index + , call_inner_instruction_index , call_block_slot , call_block_time , json_query(createMetadataAccountArgsV3, 'lax $.CreateMetadataAccountArgsV3.data.DataV2') as args , account_metadata , account_mint - FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMetadataAccountV3') }} - ) meta + , call_executing_account as metadata_program + FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMetadataAccountV3') }} + ) meta LEFT JOIN ( - SELECT account_mintAuthority, account_edition, account_metadata FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMasterEdition') }} + SELECT account_mintAuthority, account_edition, account_metadata FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMasterEdition') }} UNION ALL SELECT account_mintAuthority, account_edition, account_metadata FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMasterEditionV3') }} ) master ON master.account_metadata = meta.account_metadata {% if is_incremental() %} - WHERE meta.call_block_time >= date_trunc('day', now() - interval '7' day) + WHERE {{ incremental_predicate('meta.call_block_time') }} + {% endif %} + ) + + , token2022_metadata as ( + --token2022 direct metadata extension + SELECT + from_utf8(bytearray_substring(data,1+8+4,bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))))) as name + , from_utf8(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))) + 4 --start from end of name and end of length of symbol + , bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))),4))) --get length of symbol from end of name + )) as symbol + , from_utf8(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))) + 4 --end of name and end of length of symbol + + bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))),4))) + 4 --start from end of symbol and end of length of uri + , bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))) + 4 + + bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))),4))),4))) --get length of uri from end of symbol + )) as uri + , tx_id as metadata_tx + , account_arguments[3] as account_mint + , block_time + , executing_account as metadata_program + , row_number() over (partition by account_arguments[3] order by block_time desc) as latest + FROM {{ source('solana','instruction_calls') }} + WHERE executing_account = 'TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb' + AND bytearray_substring(data,1,1) = 0xd2 --deal with updateField later 0xdd + AND tx_success + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} + {% endif %} + ) + + , token_metadata_other as ( + --some other metadata program (idk the owner) + SELECT + from_utf8(bytearray_substring(data,1+1+4,bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))))) as name + , from_utf8(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))) + 4 --start from end of name and end of length of symbol + , bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))),4))) --get length of symbol from end of name + )) as symbol + , from_utf8(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))) + 4 --end of name and end of length of symbol + + bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))),4))) + 4 --start from end of symbol and end of length of uri + , bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))) + 4 + + bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))),4))),4))) --get length of uri from end of symbol + )) as uri + , tx_id as metadata_tx + , account_arguments[2] as account_mint + , block_time + , executing_account as metadata_program + , row_number() over (partition by account_arguments[2] order by block_time desc) as latest + FROM {{ source('solana','instruction_calls') }} + WHERE executing_account = 'META4s4fSmpkTbZoUsgC1oBnWB31vQcmnN8giPw51Zu' + AND bytearray_substring(data,1,1) = 0x21 + AND tx_success + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} {% endif %} ) SELECT tk.account_mint as token_mint_address , tk.decimals - , trim(json_value(args, 'strict $.name'))as name - , trim(json_value(args, 'strict $.symbol')) as symbol - , trim(json_value(args, 'strict $.uri')) as token_uri + , coalesce(m22.name,mo.name,trim(json_value(args, 'strict $.name'))) as name + , coalesce(m22.symbol,mo.symbol,trim(json_value(args, 'strict $.symbol'))) as symbol + , coalesce(m22.uri,mo.uri,trim(json_value(args, 'strict $.uri'))) as token_uri , tk.call_block_time as created_at + , coalesce(m22.metadata_program,mo.metadata_program,m.metadata_program) as metadata_program + , tk.token_version + , tk.call_tx_id as init_tx FROM tokens tk -LEFT JOIN metadata m ON tk.account_mint = m.account_mint +LEFT JOIN token2022_metadata m22 ON tk.account_mint = m22.account_mint AND m22.latest = 1 +LEFT JOIN token_metadata_other mo ON tk.account_mint = mo.account_mint AND mo.latest = 1 +LEFT JOIN metadata m ON tk.account_mint = m.account_mint AND m.latest = 1 WHERE m.master_edition is null +AND tk.latest = 1 + +UNION ALL + +--token2022 wrapped sol https://solscan.io/tx/2L1o7sDMCMJ6PYqfNrnY6ozJC1DEx61pRYiLdfCCggxw81naQXsmHKDLn6EhJXmDmDSQ2eCKjUMjZAQuUsyNnYUv +SELECT + trim(token_mint_address) as token_mint_address + , decimals + , trim(name) as name + , trim(symbol) as symbol + , token_uri + , cast(created_at as timestamp) created_at + , metadata_program + , token_version + , init_tx +FROM +( + VALUES +( + '9pan9bMn5HatX4EJdBwg9VgCa7Uz5HL8N1m5D3NdXejP', + 9, + 'wrapped SOL', + 'SOL', + null, + '2023-08-02 00:00:00', + null, + 'token2022', + '2L1o7sDMCMJ6PYqfNrnY6ozJC1DEx61pRYiLdfCCggxw81naQXsmHKDLn6EhJXmDmDSQ2eCKjUMjZAQuUsyNnYUv' +) +) AS temp_table (token_mint_address, decimals, name, symbol, token_uri, created_at, metadata_program, token_version, init_tx) UNION ALL ---wrapped sol is special and doesn't have a init tx (that I can find) -SELECT +--old wrapped sol is special and doesn't have a init tx (that I can find) +SELECT trim(token_mint_address) as token_mint_address , decimals , trim(name) as name , trim(symbol) as symbol , token_uri , cast(created_at as timestamp) created_at -FROM + , metadata_program + , token_version + , init_tx +FROM ( VALUES ( @@ -112,6 +213,9 @@ FROM 'wrapped SOL', 'SOL', null, - '2021-01-31 00:00:00' + '2021-01-31 00:00:00', + null, + 'spl_token', + null ) -) AS temp_table (token_mint_address, decimals, name, symbol, token_uri, created_at) +) AS temp_table (token_mint_address, decimals, name, symbol, token_uri, created_at, metadata_program, token_version, init_tx) diff --git a/models/tokens/solana/tokens_solana_schema.yml b/models/tokens/solana/tokens_solana_schema.yml index ab3b17f52e5..29ac7ed40e8 100644 --- a/models/tokens/solana/tokens_solana_schema.yml +++ b/models/tokens/solana/tokens_solana_schema.yml @@ -19,8 +19,14 @@ models: description: "token symbol" - name: decimals description: "Number of decimals, refers to how divisible a token can be" + - name: metadata_program + description: program used for creating token metadata + - name: token_version + description: version of the token program used (spl_token, token2022) - name: created_at description: token mint created at + - name: init_tx + description: "transaction that initialized the mint" - name: tokens_solana_nft meta: @@ -72,7 +78,7 @@ models: config: tags: ['solana','transfers','erc20','nft','spl'] description: > - get all spl token transfers (will add in token2022 later) + get all spl token transfers columns: - name: block_time - name: block_date @@ -80,10 +86,13 @@ models: - name: action - name: token_mint_address - name: amount + - name: fee + - name: token_version - name: from_owner - name: to_owner - name: from_token_account - name: to_token_account + - name: token_version - name: tx_signer - name: tx_id - name: outer_instruction_index diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index a28903cf3d4..2bb1e5600d6 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -14,56 +14,279 @@ \'["ilemi"]\') }}') }} -SELECT - call_block_time as block_time - , cast (date_trunc('day', call_block_time) as date) as block_date - , call_block_slot as block_slot - , action - , amount - , COALESCE(tk_s.token_mint_address, tk_d.token_mint_address) as token_mint_address - , tk_s.token_balance_owner as from_owner - , tk_d.token_balance_owner as to_owner - , account_source as from_token_account - , account_destination as to_token_account - , call_tx_signer as tx_signer - , call_tx_id as tx_id - , call_outer_instruction_index as outer_instruction_index - , COALESCE(call_inner_instruction_index,0) as inner_instruction_index - , call_outer_executing_account as outer_executing_account -FROM ( - SELECT account_source, account_destination, amount, call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer, 'transfer' as action, call_outer_instruction_index, call_inner_instruction_index +with +token2022_fee_state as ( + --we need the fee basis points and maximum fee for token2022 transfers because the fee amount is not emitted in transferChecked + SELECT + call_account_arguments[1] as account_mint + , try(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data, + 1+1+1+1+1+case when bytearray_substring(call_data,1+1+1,1) = 0x01 and bytearray_substring(call_data,1+1+1+32+1,1) = 0x01 + then 64 + when bytearray_substring(call_data,1+1+1,1) = 0x01 and bytearray_substring(call_data,1+1+1+32+1,1) = 0x00 + then 32 + when bytearray_substring(call_data,1+1+1,1) = 0x00 and bytearray_substring(call_data,1+1+1+1,1) = 0x01 + then 32 + when bytearray_substring(call_data,1+1+1,1) = 0x00 and bytearray_substring(call_data,1+1+1+1,1) = 0x00 + then 0 + end --variations of COPTION enums for first two arguments + ,2)))) as fee_basis + , try(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data, + 1+1+1+1+1+case when bytearray_substring(call_data,1+1+1,1) = 0x01 and bytearray_substring(call_data,1+1+1+32+1,1) = 0x01 + then 64 + when bytearray_substring(call_data,1+1+1,1) = 0x01 and bytearray_substring(call_data,1+1+1+32+1,1) = 0x00 + then 32 + when bytearray_substring(call_data,1+1+1,1) = 0x00 and bytearray_substring(call_data,1+1+1+1,1) = 0x01 + then 32 + when bytearray_substring(call_data,1+1+1,1) = 0x00 and bytearray_substring(call_data,1+1+1+1,1) = 0x00 + then 0 + end + +2 + ,16)))) as fee_maximum + , call_block_time as fee_time + FROM {{ source('spl_token_2022_solana','spl_token_2022_call_transferFeeExtension') }} + WHERE bytearray_substring(call_data,1+1,1) = 0x00 --https://github.com/solana-labs/solana-program-library/blob/8f50c6fabc6ec87ada229e923030381f573e0aed/token/program-2022/src/extension/transfer_fee/instruction.rs#L38 + UNION ALL + SELECT + call_account_arguments[1] as account_mint + , try(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data, + 1+1+1,2)))) as fee_basis + , try(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data, + 1+1+1+2,16)))) as fee_maximum + , call_block_time as fee_time + FROM {{ source('spl_token_2022_solana','spl_token_2022_call_transferFeeExtension') }} + WHERE bytearray_substring(call_data,1+1,1) = 0x05 --https://github.com/solana-labs/solana-program-library/blob/8f50c6fabc6ec87ada229e923030381f573e0aed/token/program-2022/src/extension/transfer_fee/instruction.rs#L147 +) + +, base as ( + SELECT + account_source, account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'transfer' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'spl_token' as token_version FROM {{ source('spl_token_solana','spl_token_call_transfer') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} UNION ALL - SELECT account_source, account_destination, amount, call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer, 'transfer' as action, call_outer_instruction_index, call_inner_instruction_index + SELECT + account_source, account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'transfer' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'spl_token' as token_version FROM {{ source('spl_token_solana','spl_token_call_transferChecked') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} UNION ALL - SELECT null, account_account as account_destination, amount, call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer, 'mint' as action, call_outer_instruction_index, call_inner_instruction_index + SELECT + null as account_source, account_account as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'mint' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'spl_token' as token_version FROM {{ source('spl_token_solana','spl_token_call_mintTo') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} UNION ALL - SELECT null, account_account as account_destination, amount, call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer, 'mint' as action, call_outer_instruction_index, call_inner_instruction_index + SELECT + null as account_source, account_account as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'mint' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'spl_token' as token_version FROM {{ source('spl_token_solana','spl_token_call_mintToChecked') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} UNION ALL - SELECT account_account as account_source, null, amount, call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer, 'burn' as action, call_outer_instruction_index, call_inner_instruction_index + SELECT + account_account as account_source, null as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'burn' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'spl_token' as token_version FROM {{ source('spl_token_solana','spl_token_call_burn') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} UNION ALL - SELECT account_account as account_source, null, amount, call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer, 'burn' as action, call_outer_instruction_index, call_inner_instruction_index + SELECT + account_account as account_source, null as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'burn' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'spl_token' as token_version FROM {{ source('spl_token_solana','spl_token_call_burnChecked') }} -) tr + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + --token2022. Most mint and account extensions still use the parent transferChecked instruction, hooks are excecuted after and interest-bearing is precalculated. + UNION ALL + + SELECT + account_source, account_destination, amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , action + , call_outer_instruction_index, call_inner_instruction_index + , fee + , token_version + FROM ( + SELECT + account_source, account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount --note that interestbearing mints have a different amount methodology, to add later + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'transfer' as action + , call_outer_instruction_index, call_inner_instruction_index + , least( + cast(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as double) + *cast(f.fee_basis as double)/10000 + ,f.fee_maximum) as fee --we want to take the percent fee on total amount, but not exceed the maximum fee + , 'token2022' as token_version + , f.fee_time + , row_number() over (partition by tr.call_tx_id, tr.call_outer_instruction_index, tr.call_inner_instruction_index order by f.fee_time desc) as latest_fee + FROM {{ source('spl_token_2022_solana','spl_token_2022_call_transferChecked') }} tr + LEFT JOIN token2022_fee_state f ON tr.account_tokenMint = f.account_mint AND tr.call_block_time >= f.fee_time + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('tr.call_block_time')}} + {% endif %} + ) WHERE latest_fee = 1 + + UNION ALL + + SELECT + null as account_source, account_mintTo as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'mint' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'token2022' as token_version + FROM {{ source('spl_token_2022_solana','spl_token_2022_call_mintTo') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT + null as account_source, account_mintTo as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'mint' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'token2022' as token_version + FROM {{ source('spl_token_2022_solana','spl_token_2022_call_mintToChecked') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT + account_burnAccount as account_source, null as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'burn' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'token2022' as token_version + FROM {{ source('spl_token_2022_solana','spl_token_2022_call_burn') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT + account_burnAccount as account_source, null as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'burn' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'token2022' as token_version + FROM {{ source('spl_token_2022_solana','spl_token_2022_call_burnChecked') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + --token2022 transferFeeExtension has some extra complications. It's the only extension with its own transferChecked wrapper (confidential transfers will have this too) + UNION ALL + + SELECT + call_account_arguments[1] as account_source, call_account_arguments[3] as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+2,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'transfer' as action + , call_outer_instruction_index, call_inner_instruction_index + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data, 1+2+8+1,8))) as fee + , 'token2022' as token_version + FROM {{ source('spl_token_2022_solana','spl_token_2022_call_transferFeeExtension') }} + WHERE bytearray_substring(call_data,1,2) = 0x1a01 --https://github.com/solana-labs/solana-program-library/blob/8f50c6fabc6ec87ada229e923030381f573e0aed/token/program-2022/src/extension/transfer_fee/instruction.rs#L284 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} +) + +SELECT + call_block_time as block_time + , cast (date_trunc('day', call_block_time) as date) as block_date + , call_block_slot as block_slot + , action + , amount + , fee + , COALESCE(tk_s.token_mint_address, tk_d.token_mint_address) as token_mint_address + , tk_s.token_balance_owner as from_owner + , tk_d.token_balance_owner as to_owner + , account_source as from_token_account + , account_destination as to_token_account + , token_version + , call_tx_signer as tx_signer + , call_tx_id as tx_id + , call_outer_instruction_index as outer_instruction_index + , COALESCE(call_inner_instruction_index,0) as inner_instruction_index + , call_outer_executing_account as outer_executing_account +FROM base tr --get token and accounts LEFT JOIN {{ ref('solana_utils_token_accounts') }} tk_s ON tk_s.address = tr.account_source LEFT JOIN {{ ref('solana_utils_token_accounts') }} tk_d ON tk_d.address = tr.account_destination WHERE 1=1 -{% if is_incremental() %} -AND {{incremental_predicate('call_block_time')}} -{% endif %} --- AND call_block_time > now() - interval '600' day \ No newline at end of file +-- AND call_block_time > now() - interval '90' day --for faster CI testing \ No newline at end of file From b1ebdd616b2450b636686f42599fe0e839903283 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Fri, 7 Jun 2024 07:20:07 -0400 Subject: [PATCH 02/14] moving --- .../tokens/solana/tokens_solana_transfers.sql | 47 ++----------------- 1 file changed, 3 insertions(+), 44 deletions(-) diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index 2bb1e5600d6..bda951b85eb 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -14,50 +14,9 @@ \'["ilemi"]\') }}') }} -with -token2022_fee_state as ( - --we need the fee basis points and maximum fee for token2022 transfers because the fee amount is not emitted in transferChecked - SELECT - call_account_arguments[1] as account_mint - , try(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data, - 1+1+1+1+1+case when bytearray_substring(call_data,1+1+1,1) = 0x01 and bytearray_substring(call_data,1+1+1+32+1,1) = 0x01 - then 64 - when bytearray_substring(call_data,1+1+1,1) = 0x01 and bytearray_substring(call_data,1+1+1+32+1,1) = 0x00 - then 32 - when bytearray_substring(call_data,1+1+1,1) = 0x00 and bytearray_substring(call_data,1+1+1+1,1) = 0x01 - then 32 - when bytearray_substring(call_data,1+1+1,1) = 0x00 and bytearray_substring(call_data,1+1+1+1,1) = 0x00 - then 0 - end --variations of COPTION enums for first two arguments - ,2)))) as fee_basis - , try(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data, - 1+1+1+1+1+case when bytearray_substring(call_data,1+1+1,1) = 0x01 and bytearray_substring(call_data,1+1+1+32+1,1) = 0x01 - then 64 - when bytearray_substring(call_data,1+1+1,1) = 0x01 and bytearray_substring(call_data,1+1+1+32+1,1) = 0x00 - then 32 - when bytearray_substring(call_data,1+1+1,1) = 0x00 and bytearray_substring(call_data,1+1+1+1,1) = 0x01 - then 32 - when bytearray_substring(call_data,1+1+1,1) = 0x00 and bytearray_substring(call_data,1+1+1+1,1) = 0x00 - then 0 - end - +2 - ,16)))) as fee_maximum - , call_block_time as fee_time - FROM {{ source('spl_token_2022_solana','spl_token_2022_call_transferFeeExtension') }} - WHERE bytearray_substring(call_data,1+1,1) = 0x00 --https://github.com/solana-labs/solana-program-library/blob/8f50c6fabc6ec87ada229e923030381f573e0aed/token/program-2022/src/extension/transfer_fee/instruction.rs#L38 - UNION ALL - SELECT - call_account_arguments[1] as account_mint - , try(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data, - 1+1+1,2)))) as fee_basis - , try(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data, - 1+1+1+2,16)))) as fee_maximum - , call_block_time as fee_time - FROM {{ source('spl_token_2022_solana','spl_token_2022_call_transferFeeExtension') }} - WHERE bytearray_substring(call_data,1+1,1) = 0x05 --https://github.com/solana-labs/solana-program-library/blob/8f50c6fabc6ec87ada229e923030381f573e0aed/token/program-2022/src/extension/transfer_fee/instruction.rs#L147 -) -, base as ( +WITH +base as ( SELECT account_source, account_destination , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount @@ -177,7 +136,7 @@ token2022_fee_state as ( , f.fee_time , row_number() over (partition by tr.call_tx_id, tr.call_outer_instruction_index, tr.call_inner_instruction_index order by f.fee_time desc) as latest_fee FROM {{ source('spl_token_2022_solana','spl_token_2022_call_transferChecked') }} tr - LEFT JOIN token2022_fee_state f ON tr.account_tokenMint = f.account_mint AND tr.call_block_time >= f.fee_time + LEFT JOIN {{ ref('tokens_solan_fees_history') }} f ON tr.account_tokenMint = f.account_mint AND tr.call_block_time >= f.fee_time WHERE 1=1 {% if is_incremental() %} AND {{incremental_predicate('tr.call_block_time')}} From 7def82d451b365c155bdb158346e3183d13e105f Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Fri, 7 Jun 2024 07:20:26 -0400 Subject: [PATCH 03/14] removing --- .../tokens/solana/tokens_solana_fungible.sql | 158 +++--------------- 1 file changed, 27 insertions(+), 131 deletions(-) diff --git a/models/tokens/solana/tokens_solana_fungible.sql b/models/tokens/solana/tokens_solana_fungible.sql index 6a66471188c..5f26d007d6c 100644 --- a/models/tokens/solana/tokens_solana_fungible.sql +++ b/models/tokens/solana/tokens_solana_fungible.sql @@ -1,7 +1,7 @@ {{ config ( alias = 'fungible', - + post_hook='{{ expose_spells(\'["solana"]\', "sector", "tokens", @@ -10,32 +10,26 @@ }} -with +with tokens as ( SELECT bytearray_to_bigint(bytearray_reverse(bytearray_substring(call_data, 2, 1))) as decimals , call_data , account_mint - , token_version , call_tx_id , call_block_time - , row_number() over (partition by account_mint order by call_block_time desc) as latest FROM ( - SELECT call_data, account_mint, call_tx_id, call_block_time, 'spl_token' as token_version FROM {{ source('spl_token_solana', 'spl_token_call_initializeMint') }} - UNION ALL - SELECT call_data, account_mint, call_tx_id, call_block_time, 'spl_token' as token_version FROM {{ source('spl_token_solana', 'spl_token_call_initializeMint2') }} - UNION ALL - SELECT call_data, account_mint, call_tx_id, call_block_time, 'token2022' as token_version FROM {{ source('spl_token_2022_solana', 'spl_token_2022_call_initializeMint') }} - UNION ALL - SELECT call_data, account_mint, call_tx_id, call_block_time, 'token2022' as token_version FROM {{ source('spl_token_2022_solana', 'spl_token_2022_call_initializeMint2') }} + SELECT call_data, account_mint, call_tx_id, call_block_time FROM {{ source('spl_token_solana', 'spl_token_call_initializeMint') }} + UNION ALL + SELECT call_data, account_mint, call_tx_id, call_block_time FROM {{ source('spl_token_solana', 'spl_token_call_initializeMint2') }} ) {% if is_incremental() %} - where {{ incremental_predicate('call_block_time') }} + where call_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} ) - + , metadata as ( - SELECT + SELECT meta.call_tx_id , meta.call_block_slot , meta.call_block_time @@ -44,10 +38,8 @@ with , meta.account_mint , meta.call_block_time , master.account_edition as master_edition - , metadata_program - , row_number() over (partition by meta.account_mint order by meta.call_block_time desc) as latest FROM ( - SELECT + SELECT call_tx_id , call_outer_instruction_index , call_inner_instruction_index @@ -56,10 +48,9 @@ with , json_query(createMetadataAccountArgs, 'lax $.CreateMetadataAccountArgs.data.Data') as args , account_metadata , account_mint - , call_executing_account as metadata_program FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMetadataAccount') }} - UNION ALL - SELECT + UNION ALL + SELECT call_tx_id , call_outer_instruction_index , call_inner_instruction_index @@ -68,143 +59,51 @@ with , json_query(createMetadataAccountArgsV2, 'lax $.CreateMetadataAccountArgsV2.data.DataV2') as args , account_metadata , account_mint - , call_executing_account as metadata_program FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMetadataAccountV2') }} - UNION ALL - SELECT + UNION ALL + SELECT call_tx_id , call_outer_instruction_index - , call_inner_instruction_index + , call_inner_instruction_index , call_block_slot , call_block_time , json_query(createMetadataAccountArgsV3, 'lax $.CreateMetadataAccountArgsV3.data.DataV2') as args , account_metadata , account_mint - , call_executing_account as metadata_program - FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMetadataAccountV3') }} - ) meta + FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMetadataAccountV3') }} + ) meta LEFT JOIN ( - SELECT account_mintAuthority, account_edition, account_metadata FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMasterEdition') }} + SELECT account_mintAuthority, account_edition, account_metadata FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMasterEdition') }} UNION ALL SELECT account_mintAuthority, account_edition, account_metadata FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMasterEditionV3') }} ) master ON master.account_metadata = meta.account_metadata {% if is_incremental() %} - WHERE {{ incremental_predicate('meta.call_block_time') }} - {% endif %} - ) - - , token2022_metadata as ( - --token2022 direct metadata extension - SELECT - from_utf8(bytearray_substring(data,1+8+4,bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))))) as name - , from_utf8(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))) + 4 --start from end of name and end of length of symbol - , bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))),4))) --get length of symbol from end of name - )) as symbol - , from_utf8(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))) + 4 --end of name and end of length of symbol - + bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))),4))) + 4 --start from end of symbol and end of length of uri - , bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))) + 4 - + bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))),4))),4))) --get length of uri from end of symbol - )) as uri - , tx_id as metadata_tx - , account_arguments[3] as account_mint - , block_time - , executing_account as metadata_program - , row_number() over (partition by account_arguments[3] order by block_time desc) as latest - FROM {{ source('solana','instruction_calls') }} - WHERE executing_account = 'TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb' - AND bytearray_substring(data,1,1) = 0xd2 --deal with updateField later 0xdd - AND tx_success - {% if is_incremental() %} - AND {{ incremental_predicate('block_time') }} - {% endif %} - ) - - , token_metadata_other as ( - --some other metadata program (idk the owner) - SELECT - from_utf8(bytearray_substring(data,1+1+4,bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))))) as name - , from_utf8(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))) + 4 --start from end of name and end of length of symbol - , bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))),4))) --get length of symbol from end of name - )) as symbol - , from_utf8(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))) + 4 --end of name and end of length of symbol - + bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))),4))) + 4 --start from end of symbol and end of length of uri - , bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))) + 4 - + bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))),4))),4))) --get length of uri from end of symbol - )) as uri - , tx_id as metadata_tx - , account_arguments[2] as account_mint - , block_time - , executing_account as metadata_program - , row_number() over (partition by account_arguments[2] order by block_time desc) as latest - FROM {{ source('solana','instruction_calls') }} - WHERE executing_account = 'META4s4fSmpkTbZoUsgC1oBnWB31vQcmnN8giPw51Zu' - AND bytearray_substring(data,1,1) = 0x21 - AND tx_success - {% if is_incremental() %} - AND {{ incremental_predicate('block_time') }} + WHERE meta.call_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} ) SELECT tk.account_mint as token_mint_address , tk.decimals - , coalesce(m22.name,mo.name,trim(json_value(args, 'strict $.name'))) as name - , coalesce(m22.symbol,mo.symbol,trim(json_value(args, 'strict $.symbol'))) as symbol - , coalesce(m22.uri,mo.uri,trim(json_value(args, 'strict $.uri'))) as token_uri + , trim(json_value(args, 'strict $.name'))as name + , trim(json_value(args, 'strict $.symbol')) as symbol + , trim(json_value(args, 'strict $.uri')) as token_uri , tk.call_block_time as created_at - , coalesce(m22.metadata_program,mo.metadata_program,m.metadata_program) as metadata_program - , tk.token_version - , tk.call_tx_id as init_tx FROM tokens tk -LEFT JOIN token2022_metadata m22 ON tk.account_mint = m22.account_mint AND m22.latest = 1 -LEFT JOIN token_metadata_other mo ON tk.account_mint = mo.account_mint AND mo.latest = 1 -LEFT JOIN metadata m ON tk.account_mint = m.account_mint AND m.latest = 1 +LEFT JOIN metadata m ON tk.account_mint = m.account_mint WHERE m.master_edition is null -AND tk.latest = 1 - -UNION ALL - ---token2022 wrapped sol https://solscan.io/tx/2L1o7sDMCMJ6PYqfNrnY6ozJC1DEx61pRYiLdfCCggxw81naQXsmHKDLn6EhJXmDmDSQ2eCKjUMjZAQuUsyNnYUv -SELECT - trim(token_mint_address) as token_mint_address - , decimals - , trim(name) as name - , trim(symbol) as symbol - , token_uri - , cast(created_at as timestamp) created_at - , metadata_program - , token_version - , init_tx -FROM -( - VALUES -( - '9pan9bMn5HatX4EJdBwg9VgCa7Uz5HL8N1m5D3NdXejP', - 9, - 'wrapped SOL', - 'SOL', - null, - '2023-08-02 00:00:00', - null, - 'token2022', - '2L1o7sDMCMJ6PYqfNrnY6ozJC1DEx61pRYiLdfCCggxw81naQXsmHKDLn6EhJXmDmDSQ2eCKjUMjZAQuUsyNnYUv' -) -) AS temp_table (token_mint_address, decimals, name, symbol, token_uri, created_at, metadata_program, token_version, init_tx) UNION ALL ---old wrapped sol is special and doesn't have a init tx (that I can find) -SELECT +--wrapped sol is special and doesn't have a init tx (that I can find) +SELECT trim(token_mint_address) as token_mint_address , decimals , trim(name) as name , trim(symbol) as symbol , token_uri , cast(created_at as timestamp) created_at - , metadata_program - , token_version - , init_tx -FROM +FROM ( VALUES ( @@ -213,9 +112,6 @@ FROM 'wrapped SOL', 'SOL', null, - '2021-01-31 00:00:00', - null, - 'spl_token', - null + '2021-01-31 00:00:00' ) -) AS temp_table (token_mint_address, decimals, name, symbol, token_uri, created_at, metadata_program, token_version, init_tx) +) AS temp_table (token_mint_address, decimals, name, symbol, token_uri, created_at) From 042b5afe149f09ebc48625c78b943e4699fa2229 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Fri, 7 Jun 2024 07:20:59 -0400 Subject: [PATCH 04/14] removing --- models/tokens/solana/tokens_solana_schema.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/models/tokens/solana/tokens_solana_schema.yml b/models/tokens/solana/tokens_solana_schema.yml index 29ac7ed40e8..97871a6c0e4 100644 --- a/models/tokens/solana/tokens_solana_schema.yml +++ b/models/tokens/solana/tokens_solana_schema.yml @@ -19,14 +19,8 @@ models: description: "token symbol" - name: decimals description: "Number of decimals, refers to how divisible a token can be" - - name: metadata_program - description: program used for creating token metadata - - name: token_version - description: version of the token program used (spl_token, token2022) - name: created_at description: token mint created at - - name: init_tx - description: "transaction that initialized the mint" - name: tokens_solana_nft meta: From 11254fbad9d1d6542fadd3c2a05ebd718b24e29b Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Fri, 7 Jun 2024 07:22:36 -0400 Subject: [PATCH 05/14] sources removed --- .../tokens/solana/tokens_solana.sources.yml | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/models/tokens/solana/tokens_solana.sources.yml b/models/tokens/solana/tokens_solana.sources.yml index 5a17103eee3..0636d8254fe 100644 --- a/models/tokens/solana/tokens_solana.sources.yml +++ b/models/tokens/solana/tokens_solana.sources.yml @@ -24,29 +24,6 @@ sources: - name: spl_token_call_burnChecked loaded_at_field: call_block_time - - name: spl_token_2022_solana - description: "spl_token 2022 decoded tables" - freshness: # default freshness - warn_after: { count: 12, period: hour } - error_after: { count: 24, period: hour } - tables: - - name: spl_token_2022_call_initializeMint - loaded_at_field: call_block_time - - name: spl_token_2022_call_initializeMint2 - loaded_at_field: call_block_time - - name: spl_token_2022_call_transferFeeExtension - loaded_at_field: call_block_time - - name: spl_token_2022_call_transferChecked - loaded_at_field: call_block_time - - name: spl_token_2022_call_mintTo - loaded_at_field: call_block_time - - name: spl_token_2022_call_mintToChecked - loaded_at_field: call_block_time - - name: spl_token_2022_call_burn - loaded_at_field: call_block_time - - name: spl_token_2022_call_burnChecked - loaded_at_field: call_block_time - - name: mpl_token_metadata_solana description: "mpl metadata decoded tables" freshness: # default freshness From 862713f43bf322fbb3a6a7e4616e78abc54e1d33 Mon Sep 17 00:00:00 2001 From: Alan Ghobadi Date: Fri, 7 Jun 2024 15:22:05 +0200 Subject: [PATCH 06/14] Fix name --- models/tokens/solana/tokens_solana_transfers.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index bda951b85eb..a9d318a2321 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -136,8 +136,8 @@ base as ( , f.fee_time , row_number() over (partition by tr.call_tx_id, tr.call_outer_instruction_index, tr.call_inner_instruction_index order by f.fee_time desc) as latest_fee FROM {{ source('spl_token_2022_solana','spl_token_2022_call_transferChecked') }} tr - LEFT JOIN {{ ref('tokens_solan_fees_history') }} f ON tr.account_tokenMint = f.account_mint AND tr.call_block_time >= f.fee_time - WHERE 1=1 + LEFT JOIN {{ ref('tokens_solana_fees_history') }} f ON tr.account_tokenMint = f.account_mint AND tr.call_block_time >= f.fee_time + WHERE 1=1 {% if is_incremental() %} AND {{incremental_predicate('tr.call_block_time')}} {% endif %} From 19c5f987d982c1167475d75c6d73eae5dd367cf7 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Fri, 7 Jun 2024 07:06:44 -0700 Subject: [PATCH 07/14] Update tokens_solana_transfers.sql --- models/tokens/solana/tokens_solana_transfers.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index a9d318a2321..c044e4222b3 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -245,7 +245,7 @@ SELECT , call_outer_executing_account as outer_executing_account FROM base tr --get token and accounts -LEFT JOIN {{ ref('solana_utils_token_accounts') }} tk_s ON tk_s.address = tr.account_source -LEFT JOIN {{ ref('solana_utils_token_accounts') }} tk_d ON tk_d.address = tr.account_destination +INNER JOIN {{ ref('solana_utils_token_accounts') }} tk_s ON tk_s.address = tr.account_source +INNER JOIN {{ ref('solana_utils_token_accounts') }} tk_d ON tk_d.address = tr.account_destination WHERE 1=1 -- AND call_block_time > now() - interval '90' day --for faster CI testing \ No newline at end of file From e8f56f152794db2c6d25bfda6785cb796b2a214f Mon Sep 17 00:00:00 2001 From: Alan Ghobadi Date: Fri, 7 Jun 2024 16:58:51 +0200 Subject: [PATCH 08/14] Try splitting up --- .../solana/tokens_solana_spl_transfers.sql | 127 +++++++++ .../tokens_solana_token22_spl_transfers.sql | 146 ++++++++++ .../tokens/solana/tokens_solana_transfers.sql | 263 ++++-------------- 3 files changed, 320 insertions(+), 216 deletions(-) create mode 100644 models/tokens/solana/tokens_solana_spl_transfers.sql create mode 100644 models/tokens/solana/tokens_solana_token22_spl_transfers.sql diff --git a/models/tokens/solana/tokens_solana_spl_transfers.sql b/models/tokens/solana/tokens_solana_spl_transfers.sql new file mode 100644 index 00000000000..06f1fbfd692 --- /dev/null +++ b/models/tokens/solana/tokens_solana_spl_transfers.sql @@ -0,0 +1,127 @@ + {{ + config( + schema = 'tokens_solana', + alias = 'spl_transfers', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + partition_by = ['block_date'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + unique_key = ['tx_id','outer_instruction_index','inner_instruction_index', 'block_slot'] + ) +}} + + +WITH +base as ( + SELECT + account_source, account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'transfer' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'spl_token' as token_version + FROM {{ source('spl_token_solana','spl_token_call_transfer') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT + account_source, account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'transfer' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'spl_token' as token_version + FROM {{ source('spl_token_solana','spl_token_call_transferChecked') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT + null as account_source, account_account as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'mint' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'spl_token' as token_version + FROM {{ source('spl_token_solana','spl_token_call_mintTo') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT + null as account_source, account_account as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'mint' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'spl_token' as token_version + FROM {{ source('spl_token_solana','spl_token_call_mintToChecked') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT + account_account as account_source, null as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'burn' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'spl_token' as token_version + FROM {{ source('spl_token_solana','spl_token_call_burn') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT + account_account as account_source, null as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'burn' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'spl_token' as token_version + FROM {{ source('spl_token_solana','spl_token_call_burnChecked') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} +) + +SELECT + call_block_time as block_time + , cast (date_trunc('day', call_block_time) as date) as block_date + , call_block_slot as block_slot + , action + , amount + , fee + , account_source as from_token_account + , account_destination as to_token_account + , token_version + , call_tx_signer as tx_signer + , call_tx_id as tx_id + , call_outer_instruction_index as outer_instruction_index + , COALESCE(call_inner_instruction_index,0) as inner_instruction_index + , call_outer_executing_account as outer_executing_account +FROM base tr \ No newline at end of file diff --git a/models/tokens/solana/tokens_solana_token22_spl_transfers.sql b/models/tokens/solana/tokens_solana_token22_spl_transfers.sql new file mode 100644 index 00000000000..d4efb921d86 --- /dev/null +++ b/models/tokens/solana/tokens_solana_token22_spl_transfers.sql @@ -0,0 +1,146 @@ + {{ + config( + schema = 'tokens_solana', + alias = 'token22_spl_transfers', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + partition_by = ['block_date'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + unique_key = ['tx_id','outer_instruction_index','inner_instruction_index', 'block_slot'] +}} + + +WITH +base as ( + --token2022. Most mint and account extensions still use the parent transferChecked instruction, hooks are excecuted after and interest-bearing is precalculated. + UNION ALL + + SELECT + account_source, account_destination, amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , action + , call_outer_instruction_index, call_inner_instruction_index + , fee + , token_version + FROM ( + SELECT + account_source, account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount --note that interestbearing mints have a different amount methodology, to add later + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'transfer' as action + , call_outer_instruction_index, call_inner_instruction_index + , least( + cast(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as double) + *cast(f.fee_basis as double)/10000 + ,f.fee_maximum) as fee --we want to take the percent fee on total amount, but not exceed the maximum fee + , 'token2022' as token_version + , f.fee_time + , row_number() over (partition by tr.call_tx_id, tr.call_outer_instruction_index, tr.call_inner_instruction_index order by f.fee_time desc) as latest_fee + FROM {{ source('spl_token_2022_solana','spl_token_2022_call_transferChecked') }} tr + LEFT JOIN {{ ref('tokens_solana_fees_history') }} f ON tr.account_tokenMint = f.account_mint AND tr.call_block_time >= f.fee_time + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('tr.call_block_time')}} + {% endif %} + ) WHERE latest_fee = 1 + + UNION ALL + + SELECT + null as account_source, account_mintTo as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'mint' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'token2022' as token_version + FROM {{ source('spl_token_2022_solana','spl_token_2022_call_mintTo') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT + null as account_source, account_mintTo as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'mint' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'token2022' as token_version + FROM {{ source('spl_token_2022_solana','spl_token_2022_call_mintToChecked') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT + account_burnAccount as account_source, null as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'burn' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'token2022' as token_version + FROM {{ source('spl_token_2022_solana','spl_token_2022_call_burn') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT + account_burnAccount as account_source, null as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'burn' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'token2022' as token_version + FROM {{ source('spl_token_2022_solana','spl_token_2022_call_burnChecked') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + --token2022 transferFeeExtension has some extra complications. It's the only extension with its own transferChecked wrapper (confidential transfers will have this too) + UNION ALL + + SELECT + call_account_arguments[1] as account_source, call_account_arguments[3] as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+2,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'transfer' as action + , call_outer_instruction_index, call_inner_instruction_index + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data, 1+2+8+1,8))) as fee + , 'token2022' as token_version + FROM {{ source('spl_token_2022_solana','spl_token_2022_call_transferFeeExtension') }} + WHERE bytearray_substring(call_data,1,2) = 0x1a01 --https://github.com/solana-labs/solana-program-library/blob/8f50c6fabc6ec87ada229e923030381f573e0aed/token/program-2022/src/extension/transfer_fee/instruction.rs#L284 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} +) + +SELECT + call_block_time as block_time + , cast (date_trunc('day', call_block_time) as date) as block_date + , call_block_slot as block_slot + , action + , amount + , fee + , account_source as from_token_account + , account_destination as to_token_account + , token_version + , call_tx_signer as tx_signer + , call_tx_id as tx_id + , call_outer_instruction_index as outer_instruction_index + , COALESCE(call_inner_instruction_index,0) as inner_instruction_index + , call_outer_executing_account as outer_executing_account +FROM base tr +-- AND call_block_time > now() - interval '90' day --for faster CI testing \ No newline at end of file diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index c044e4222b3..228ba1c332d 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -16,214 +16,45 @@ WITH -base as ( - SELECT - account_source, account_destination - , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount - , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer - , 'transfer' as action - , call_outer_instruction_index, call_inner_instruction_index - , null as fee - , 'spl_token' as token_version - FROM {{ source('spl_token_solana','spl_token_call_transfer') }} - WHERE 1=1 - {% if is_incremental() %} - AND {{incremental_predicate('call_block_time')}} - {% endif %} - - UNION ALL - - SELECT - account_source, account_destination - , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount - , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer - , 'transfer' as action - , call_outer_instruction_index, call_inner_instruction_index - , null as fee - , 'spl_token' as token_version - FROM {{ source('spl_token_solana','spl_token_call_transferChecked') }} - WHERE 1=1 - {% if is_incremental() %} - AND {{incremental_predicate('call_block_time')}} - {% endif %} - - UNION ALL - - SELECT - null as account_source, account_account as account_destination - , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount - , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer - , 'mint' as action - , call_outer_instruction_index, call_inner_instruction_index - , null as fee - , 'spl_token' as token_version - FROM {{ source('spl_token_solana','spl_token_call_mintTo') }} - WHERE 1=1 - {% if is_incremental() %} - AND {{incremental_predicate('call_block_time')}} - {% endif %} - - UNION ALL - - SELECT - null as account_source, account_account as account_destination - , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount - , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer - , 'mint' as action - , call_outer_instruction_index, call_inner_instruction_index - , null as fee - , 'spl_token' as token_version - FROM {{ source('spl_token_solana','spl_token_call_mintToChecked') }} - WHERE 1=1 - {% if is_incremental() %} - AND {{incremental_predicate('call_block_time')}} - {% endif %} - - UNION ALL - - SELECT - account_account as account_source, null as account_destination - , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount - , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer - , 'burn' as action - , call_outer_instruction_index, call_inner_instruction_index - , null as fee - , 'spl_token' as token_version - FROM {{ source('spl_token_solana','spl_token_call_burn') }} - WHERE 1=1 - {% if is_incremental() %} - AND {{incremental_predicate('call_block_time')}} - {% endif %} - - UNION ALL - - SELECT - account_account as account_source, null as account_destination - , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount - , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer - , 'burn' as action - , call_outer_instruction_index, call_inner_instruction_index - , null as fee - , 'spl_token' as token_version - FROM {{ source('spl_token_solana','spl_token_call_burnChecked') }} - WHERE 1=1 - {% if is_incremental() %} - AND {{incremental_predicate('call_block_time')}} - {% endif %} - - --token2022. Most mint and account extensions still use the parent transferChecked instruction, hooks are excecuted after and interest-bearing is precalculated. - UNION ALL - - SELECT - account_source, account_destination, amount - , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer - , action - , call_outer_instruction_index, call_inner_instruction_index - , fee - , token_version - FROM ( - SELECT - account_source, account_destination - , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount --note that interestbearing mints have a different amount methodology, to add later - , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer - , 'transfer' as action - , call_outer_instruction_index, call_inner_instruction_index - , least( - cast(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as double) - *cast(f.fee_basis as double)/10000 - ,f.fee_maximum) as fee --we want to take the percent fee on total amount, but not exceed the maximum fee - , 'token2022' as token_version - , f.fee_time - , row_number() over (partition by tr.call_tx_id, tr.call_outer_instruction_index, tr.call_inner_instruction_index order by f.fee_time desc) as latest_fee - FROM {{ source('spl_token_2022_solana','spl_token_2022_call_transferChecked') }} tr - LEFT JOIN {{ ref('tokens_solana_fees_history') }} f ON tr.account_tokenMint = f.account_mint AND tr.call_block_time >= f.fee_time - WHERE 1=1 - {% if is_incremental() %} - AND {{incremental_predicate('tr.call_block_time')}} - {% endif %} - ) WHERE latest_fee = 1 - - UNION ALL - - SELECT - null as account_source, account_mintTo as account_destination - , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount - , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer - , 'mint' as action - , call_outer_instruction_index, call_inner_instruction_index - , null as fee - , 'token2022' as token_version - FROM {{ source('spl_token_2022_solana','spl_token_2022_call_mintTo') }} - WHERE 1=1 - {% if is_incremental() %} - AND {{incremental_predicate('call_block_time')}} - {% endif %} - - UNION ALL - - SELECT - null as account_source, account_mintTo as account_destination - , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount - , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer - , 'mint' as action - , call_outer_instruction_index, call_inner_instruction_index - , null as fee - , 'token2022' as token_version - FROM {{ source('spl_token_2022_solana','spl_token_2022_call_mintToChecked') }} - WHERE 1=1 - {% if is_incremental() %} - AND {{incremental_predicate('call_block_time')}} - {% endif %} - - UNION ALL - - SELECT - account_burnAccount as account_source, null as account_destination - , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount - , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer - , 'burn' as action - , call_outer_instruction_index, call_inner_instruction_index - , null as fee - , 'token2022' as token_version - FROM {{ source('spl_token_2022_solana','spl_token_2022_call_burn') }} - WHERE 1=1 - {% if is_incremental() %} - AND {{incremental_predicate('call_block_time')}} - {% endif %} - - UNION ALL - - SELECT - account_burnAccount as account_source, null as account_destination - , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount - , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer - , 'burn' as action - , call_outer_instruction_index, call_inner_instruction_index - , null as fee - , 'token2022' as token_version - FROM {{ source('spl_token_2022_solana','spl_token_2022_call_burnChecked') }} - WHERE 1=1 - {% if is_incremental() %} - AND {{incremental_predicate('call_block_time')}} - {% endif %} - - --token2022 transferFeeExtension has some extra complications. It's the only extension with its own transferChecked wrapper (confidential transfers will have this too) - UNION ALL - - SELECT - call_account_arguments[1] as account_source, call_account_arguments[3] as account_destination - , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+2,8))) as amount - , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer - , 'transfer' as action - , call_outer_instruction_index, call_inner_instruction_index - , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data, 1+2+8+1,8))) as fee - , 'token2022' as token_version - FROM {{ source('spl_token_2022_solana','spl_token_2022_call_transferFeeExtension') }} - WHERE bytearray_substring(call_data,1,2) = 0x1a01 --https://github.com/solana-labs/solana-program-library/blob/8f50c6fabc6ec87ada229e923030381f573e0aed/token/program-2022/src/extension/transfer_fee/instruction.rs#L284 - {% if is_incremental() %} - AND {{incremental_predicate('call_block_time')}} - {% endif %} -) +base as ( + SELECT + block_time, + , call_block_slot as block_slot + , action + , amount + , fee + , from_token_account + , to_token_account + , token_version + , tx_signer + , tx_id + , outer_instruction_index + , inner_instruction_index + , outer_executing_account +FROM {{ ref('tokens_solana_spl_transfers') }} +{% if is_incremental() %} +WHERE {{incremental_predicate('block_time')}} +{% endif %} +UNION ALL + SELECT + block_time, + , call_block_slot as block_slot + , action + , amount + , fee + , from_token_account + , to_token_account + , token_version + , tx_signer + , tx_id + , outer_instruction_index + , inner_instruction_index + , outer_executing_account +FROM {{ ref('tokens_solana_token22_spl_transfers') }} +{% if is_incremental() %} +WHERE {{incremental_predicate('block_time')}} +{% endif %} +) SELECT call_block_time as block_time @@ -235,17 +66,17 @@ SELECT , COALESCE(tk_s.token_mint_address, tk_d.token_mint_address) as token_mint_address , tk_s.token_balance_owner as from_owner , tk_d.token_balance_owner as to_owner - , account_source as from_token_account - , account_destination as to_token_account + , from_token_account + , to_token_account , token_version - , call_tx_signer as tx_signer - , call_tx_id as tx_id - , call_outer_instruction_index as outer_instruction_index - , COALESCE(call_inner_instruction_index,0) as inner_instruction_index - , call_outer_executing_account as outer_executing_account + , tx_signer + , tx_id + , outer_instruction_index + , inner_instruction_index + , outer_executing_account FROM base tr --get token and accounts -INNER JOIN {{ ref('solana_utils_token_accounts') }} tk_s ON tk_s.address = tr.account_source +INNER JOIN {{ ref('solana_utils_token_accounts') }} tk_s ON tk_s.address = tr.account_source INNER JOIN {{ ref('solana_utils_token_accounts') }} tk_d ON tk_d.address = tr.account_destination WHERE 1=1 -- AND call_block_time > now() - interval '90' day --for faster CI testing \ No newline at end of file From d75c3d2c548c0d78b8602699cedca09e4945dd4c Mon Sep 17 00:00:00 2001 From: Alan Ghobadi Date: Fri, 7 Jun 2024 17:09:40 +0200 Subject: [PATCH 09/14] Fxi typo --- models/tokens/solana/tokens_solana_token22_spl_transfers.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/tokens/solana/tokens_solana_token22_spl_transfers.sql b/models/tokens/solana/tokens_solana_token22_spl_transfers.sql index d4efb921d86..78b23717480 100644 --- a/models/tokens/solana/tokens_solana_token22_spl_transfers.sql +++ b/models/tokens/solana/tokens_solana_token22_spl_transfers.sql @@ -8,6 +8,7 @@ partition_by = ['block_date'], incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], unique_key = ['tx_id','outer_instruction_index','inner_instruction_index', 'block_slot'] + ) }} From 8a53bdb7e5bc6c6119c8dc01dc6820a545c9f542 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Fri, 7 Jun 2024 12:09:07 -0400 Subject: [PATCH 10/14] fix union --- models/tokens/solana/tokens_solana_token22_spl_transfers.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/models/tokens/solana/tokens_solana_token22_spl_transfers.sql b/models/tokens/solana/tokens_solana_token22_spl_transfers.sql index 78b23717480..9ebc9bff9cb 100644 --- a/models/tokens/solana/tokens_solana_token22_spl_transfers.sql +++ b/models/tokens/solana/tokens_solana_token22_spl_transfers.sql @@ -15,8 +15,6 @@ WITH base as ( --token2022. Most mint and account extensions still use the parent transferChecked instruction, hooks are excecuted after and interest-bearing is precalculated. - UNION ALL - SELECT account_source, account_destination, amount , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer From 4afc64e5f5ff66532114b4ad17129568167639bb Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Fri, 7 Jun 2024 12:51:49 -0400 Subject: [PATCH 11/14] typeing --- models/tokens/solana/tokens_solana_spl_transfers.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/models/tokens/solana/tokens_solana_spl_transfers.sql b/models/tokens/solana/tokens_solana_spl_transfers.sql index 06f1fbfd692..800024c1a26 100644 --- a/models/tokens/solana/tokens_solana_spl_transfers.sql +++ b/models/tokens/solana/tokens_solana_spl_transfers.sql @@ -20,7 +20,7 @@ base as ( , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer , 'transfer' as action , call_outer_instruction_index, call_inner_instruction_index - , null as fee + , cast(null as double) as fee , 'spl_token' as token_version FROM {{ source('spl_token_solana','spl_token_call_transfer') }} WHERE 1=1 @@ -36,7 +36,7 @@ base as ( , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer , 'transfer' as action , call_outer_instruction_index, call_inner_instruction_index - , null as fee + , cast(null as double) as fee , 'spl_token' as token_version FROM {{ source('spl_token_solana','spl_token_call_transferChecked') }} WHERE 1=1 @@ -52,7 +52,7 @@ base as ( , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer , 'mint' as action , call_outer_instruction_index, call_inner_instruction_index - , null as fee + , cast(null as double) as fee , 'spl_token' as token_version FROM {{ source('spl_token_solana','spl_token_call_mintTo') }} WHERE 1=1 @@ -68,7 +68,7 @@ base as ( , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer , 'mint' as action , call_outer_instruction_index, call_inner_instruction_index - , null as fee + , cast(null as double) as fee , 'spl_token' as token_version FROM {{ source('spl_token_solana','spl_token_call_mintToChecked') }} WHERE 1=1 @@ -84,7 +84,7 @@ base as ( , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer , 'burn' as action , call_outer_instruction_index, call_inner_instruction_index - , null as fee + , cast(null as double) as fee , 'spl_token' as token_version FROM {{ source('spl_token_solana','spl_token_call_burn') }} WHERE 1=1 @@ -100,7 +100,7 @@ base as ( , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer , 'burn' as action , call_outer_instruction_index, call_inner_instruction_index - , null as fee + , cast(null as double) as fee , 'spl_token' as token_version FROM {{ source('spl_token_solana','spl_token_call_burnChecked') }} WHERE 1=1 From 990d035bbb23f7fa55cf84f74aeaf9dc17a27832 Mon Sep 17 00:00:00 2001 From: Alan Ghobadi Date: Mon, 10 Jun 2024 10:31:51 +0200 Subject: [PATCH 12/14] Try another split --- .../solana/tokens_solana_spl_transfers.sql | 38 ++++++++++--------- ...ens_solana_spl_transfers_call_transfer.sql | 33 ++++++++++++++++ 2 files changed, 54 insertions(+), 17 deletions(-) create mode 100644 models/tokens/solana/tokens_solana_spl_transfers_call_transfer.sql diff --git a/models/tokens/solana/tokens_solana_spl_transfers.sql b/models/tokens/solana/tokens_solana_spl_transfers.sql index 800024c1a26..7ad07021765 100644 --- a/models/tokens/solana/tokens_solana_spl_transfers.sql +++ b/models/tokens/solana/tokens_solana_spl_transfers.sql @@ -14,22 +14,6 @@ WITH base as ( - SELECT - account_source, account_destination - , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount - , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer - , 'transfer' as action - , call_outer_instruction_index, call_inner_instruction_index - , cast(null as double) as fee - , 'spl_token' as token_version - FROM {{ source('spl_token_solana','spl_token_call_transfer') }} - WHERE 1=1 - {% if is_incremental() %} - AND {{incremental_predicate('call_block_time')}} - {% endif %} - - UNION ALL - SELECT account_source, account_destination , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount @@ -124,4 +108,24 @@ SELECT , call_outer_instruction_index as outer_instruction_index , COALESCE(call_inner_instruction_index,0) as inner_instruction_index , call_outer_executing_account as outer_executing_account -FROM base tr \ No newline at end of file +FROM base +UNION ALL +SELECT + block_time + , block_date + , block_slot + , action + , amount + , fee + , from_token_account + , to_token_account + , token_version + , tx_signer + , tx_id + , outer_instruction_index + , inner_instruction_index + , outer_executing_account + FROM {{ref'spl_transfers_call_transfer')}} +{% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} \ No newline at end of file diff --git a/models/tokens/solana/tokens_solana_spl_transfers_call_transfer.sql b/models/tokens/solana/tokens_solana_spl_transfers_call_transfer.sql new file mode 100644 index 00000000000..cc319990015 --- /dev/null +++ b/models/tokens/solana/tokens_solana_spl_transfers_call_transfer.sql @@ -0,0 +1,33 @@ + {{ + config( + schema = 'tokens_solana', + alias = 'spl_transfers_call_transfer', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + partition_by = ['block_date'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + unique_key = ['tx_id','outer_instruction_index','inner_instruction_index', 'block_slot'] + ) +}} + +SELECT + call_block_time as block_time + , cast (date_trunc('day', call_block_time) as date) as block_date + , call_block_slot as block_slot + , 'transfer' as action + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , cast(null as double) as fee + , account_source as from_token_account + , account_destination as to_token_account + , 'spl_token' as token_version + , call_tx_signer as tx_signer + , call_tx_id as tx_id + , call_outer_instruction_index as outer_instruction_index + , COALESCE(call_inner_instruction_index,0) as inner_instruction_index + , call_outer_executing_account as outer_executing_account +FROM {{ source('spl_token_solana','spl_token_call_transfer') }} +WHERE 1=1 +{% if is_incremental() %} +AND {{incremental_predicate('call_block_time')}} +{% endif %} From a567bb702b0eb58b795f6d6ed9d0696e017d8b7c Mon Sep 17 00:00:00 2001 From: Alan Ghobadi Date: Mon, 10 Jun 2024 10:36:31 +0200 Subject: [PATCH 13/14] Fix typo --- models/tokens/solana/tokens_solana_spl_transfers.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/tokens/solana/tokens_solana_spl_transfers.sql b/models/tokens/solana/tokens_solana_spl_transfers.sql index 7ad07021765..0f4af81972d 100644 --- a/models/tokens/solana/tokens_solana_spl_transfers.sql +++ b/models/tokens/solana/tokens_solana_spl_transfers.sql @@ -125,7 +125,7 @@ SELECT , outer_instruction_index , inner_instruction_index , outer_executing_account - FROM {{ref'spl_transfers_call_transfer')}} + FROM {{ref('spl_transfers_call_transfer')}} {% if is_incremental() %} AND {{incremental_predicate('call_block_time')}} {% endif %} \ No newline at end of file From 88f064906a41374d0106d8d2027b671236a26c70 Mon Sep 17 00:00:00 2001 From: Alan Ghobadi Date: Mon, 10 Jun 2024 10:43:48 +0200 Subject: [PATCH 14/14] Use correct path --- models/tokens/solana/tokens_solana_spl_transfers.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/tokens/solana/tokens_solana_spl_transfers.sql b/models/tokens/solana/tokens_solana_spl_transfers.sql index 0f4af81972d..2f788049ac5 100644 --- a/models/tokens/solana/tokens_solana_spl_transfers.sql +++ b/models/tokens/solana/tokens_solana_spl_transfers.sql @@ -125,7 +125,7 @@ SELECT , outer_instruction_index , inner_instruction_index , outer_executing_account - FROM {{ref('spl_transfers_call_transfer')}} + FROM {{ref('tokens_solana_spl_transfers_call_transfer')}} {% if is_incremental() %} AND {{incremental_predicate('call_block_time')}} {% endif %} \ No newline at end of file