Skip to content

Commit

Permalink
NFT Restructuring (#3357)
Browse files Browse the repository at this point in the history
* Restructure Cryptopunks to new NFT trades setup (#3286)

* cryptopunks restructuring

* add to trades_beta

* fix join logic

* fix call reference

* fix bid pricing

* fix bid buyer

* syntax

* fix

* allow 0.1% error and fix seed value

* Restructure Sudoswap to new NFT trades setup (#3329)

* sudoswap restructuring

* syntax fix

* add seed test

* Restructure collectionswap to new NFT trades setup (#3330)

* restructure collectionswap

* syntax fix

* bugfix

* use price_raw

* add test seed

* check-seed

* Restructure looksrare to new NFT trades setup (#3332)

* restructure looksrare

* bugfix

* bugfix

* bugfix

* add seed tests

* fix seed

* seed schema

* cleanup

* lr seed schema

* NFT trades backwards compatibility layer (#3350)

* compatibility layer

* typo

* test passes with no results

* fix setup

* fix

* add fee percentages

* nest test

* move from string to varchar

* allow the ignore of 1 punk sale in the rowcount test

* allow the ignore of 1 punk sale in the rowcount test

* allow the ignore of 1 punk sale in the rowcount test
  • Loading branch information
0xRobin authored May 17, 2023
1 parent 3ffc84a commit f68e658
Show file tree
Hide file tree
Showing 21 changed files with 1,017 additions and 5 deletions.
2 changes: 2 additions & 0 deletions macros/models/_sector/nft/enrich_trades.sql
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ SELECT
base.price_raw/pow(10,coalesce(erc20.decimals,18))*p.price as price_usd,
base.platform_fee_amount_raw/pow(10,coalesce(erc20.decimals,18))*p.price as platform_fee_amount_usd,
base.royalty_fee_amount_raw/pow(10,coalesce(erc20.decimals,18))*p.price as royalty_fee_amount_usd,
coalesce(cast(100*base.platform_fee_amount_raw/base.price_raw as double),cast(0.0 as double)) as platform_fee_percentage,
coalesce(cast(100*base.royalty_fee_amount_raw/base.price_raw as double),cast(0.0 as double)) as royalty_fee_percentage,
coalesce(agg1.contract_address,agg2.contract_address) as aggregator_address,
{% if aggregator_markers != null %}
coalesce(agg_mark.aggregator_name, agg1.name, agg2.name) as aggregator_name
Expand Down
44 changes: 44 additions & 0 deletions macros/models/_sector/nft/port_to_old_schema.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
{% macro port_to_old_schema(model) %}

SELECT
blockchain,
project,
project_version as version,
block_date,
block_time,
nft_token_id as token_id,
nft_collection as collection,
price_usd as amount_usd,
nft_standard as token_standard,
trade_type,
nft_amount as number_of_items,
trade_category,
'Trade' as evt_type,
seller,
buyer,
price as amount_original,
price_raw as amount_raw,
currency_symbol,
currency_contract,
nft_contract_address,
project_contract_address,
aggregator_name,
aggregator_address,
tx_hash,
block_number,
tx_from,
tx_to,
platform_fee_amount_raw,
platform_fee_amount,
platform_fee_amount_usd,
platform_fee_percentage,
royalty_fee_address as royalty_fee_receive_address,
currency_symbol as royalty_fee_currency_symbol,
royalty_fee_amount_raw,
royalty_fee_amount,
royalty_fee_amount_usd,
royalty_fee_percentage,
concat(cast(block_number as varchar(10)),'-',cast(tx_hash as varchar(42)),'-',cast(sub_tx_trade_id as varchar(10))) as unique_trade_id
FROM {{ model }}

{% endmacro %}
2 changes: 1 addition & 1 deletion models/_sector/nft/trades/MIGRATION.md
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ The migration for each protocol consists of 3 parts.
|-------------|:-------:|-------------------:|:--------:|:---------:|:------:|
| archipelago | v1 | 561 | [x] | [x] | |
| blur | v1 | 3,067,180 | [x] | [x] | |
| cryptopunks | v1 | 23,054 | | | |
| cryptopunks | v1 | 23,054 | [x] | [x] | |
| element | v1 | 106,654 | | | |
| foundation | v1 | 137,246 | [x] | [x] | |
| looksrare | v1 | 401,647 | | | |
Expand Down
145 changes: 145 additions & 0 deletions models/_sector/nft/trades/ethereum/platforms/_schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,37 @@ models:
- platform_fee_amount_raw
- royalty_fee_amount_raw

- name: cryptopunks_ethereum_base_trades
meta:
blockchain: ethereum
sector: nft
project: cyptopunks
contributors: [ 'cat', '0xRob' ]
config:
tags: [ 'ethereum', 'nft', 'trades', 'cyptopunks' ]
description: "cyptopunks base trades"
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns: [ 'block_number','tx_hash','sub_tx_trade_id' ]
- equal_rowcount_with_sources:
error_if: ">1"
evt_sources:
- source('cryptopunks_ethereum','CryptoPunksMarket_evt_PunkBought')
- check_seed:
seed_file: ref('cryptopunks_ethereum_base_trades_seed')
match_columns:
- block_number
- tx_hash
- sub_tx_trade_id
check_columns:
- buyer
- seller
- nft_contract_address
- nft_token_id
- price_raw
- platform_fee_amount_raw
- royalty_fee_amount_raw

- name: blur_ethereum_base_trades
meta:
blockchain: ethereum
Expand Down Expand Up @@ -183,6 +214,67 @@ models:
- platform_fee_amount_raw
- royalty_fee_amount_raw

- name: looksrare_v1_ethereum_base_trades
meta:
blockchain: ethereum
sector: nft
project: looksrare
contributors: [ '0xRob','soispoke', 'hildobby', 'denze' ]
config:
tags: [ 'ethereum', 'nft', 'trades', 'looksrare' ]
description: "looksrare v1 base trades"
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns: [ 'block_number','tx_hash','sub_tx_trade_id' ]
- equal_rowcount_with_sources:
evt_sources:
- source('looksrare_ethereum','LooksRareExchange_evt_TakerAsk')
- source('looksrare_ethereum','LooksRareExchange_evt_TakerBid')
- check_seed:
seed_file: ref('looksrare_v1_ethereum_base_trades_seed')
match_columns:
- block_number
- tx_hash
- sub_tx_trade_id
check_columns:
- buyer
- seller
- nft_contract_address
- nft_token_id
- price_raw
- platform_fee_amount_raw
- royalty_fee_amount_raw

- name: looksrare_v2_ethereum_base_trades
meta:
blockchain: ethereum
sector: nft
project: looksrare
contributors: [ '0xRob','soispoke', 'hildobby', 'denze' ]
config:
tags: [ 'ethereum', 'nft', 'trades', 'looksrare' ]
description: "looksrare v2 base trades"
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns: [ 'block_number','tx_hash','sub_tx_trade_id' ]
- equal_rowcount_with_sources:
evt_sources:
- source('looksrare_v2_ethereum','LooksRareProtocol_evt_TakerAsk')
- source('looksrare_v2_ethereum','LooksRareProtocol_evt_TakerBid')
- check_seed:
seed_file: ref('looksrare_v2_ethereum_base_trades_seed')
match_columns:
- block_number
- tx_hash
- sub_tx_trade_id
check_columns:
- buyer
- seller
- nft_contract_address
- nft_token_id
- price_raw
- platform_fee_amount_raw
- royalty_fee_amount_raw
- name: zora_v1_ethereum_base_trades
meta:
blockchain: ethereum
Expand Down Expand Up @@ -280,3 +372,56 @@ models:
- platform_fee_amount_raw
- royalty_fee_amount_raw

- name: sudoswap_ethereum_base_trades
meta:
blockchain: ethereum
sector: nft
project: sudoswap
contributors: [ '0xRob','ilemi' ]
config:
tags: [ 'ethereum', 'nft', 'trades', 'sudoswap' ]
description: "sudoswap base trades"
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns: [ 'block_number','tx_hash','sub_tx_trade_id' ]
- check_seed:
seed_file: ref('sudoswap_ethereum_base_trades_seed')
match_columns:
- block_number
- tx_hash
- sub_tx_trade_id
check_columns:
- buyer
- seller
- nft_contract_address
- nft_token_id
- price_raw
- platform_fee_amount_raw
- royalty_fee_amount_raw

- name: collectionswap_ethereum_base_trades
meta:
blockchain: ethereum
sector: nft
project: collectionswap
contributors: [ '0xRob' ]
config:
tags: [ 'ethereum', 'nft', 'trades', 'collectionswap' ]
description: "collectionswap base trades"
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns: [ 'block_number','tx_hash','sub_tx_trade_id' ]
- check_seed:
seed_file: ref('collectionswap_ethereum_base_trades_seed')
match_columns:
- block_number
- tx_hash
- sub_tx_trade_id
check_columns:
- buyer
- seller
- nft_contract_address
- nft_token_id
- price_raw
- platform_fee_amount_raw
- royalty_fee_amount_raw
3 changes: 3 additions & 0 deletions models/_sector/nft/trades/ethereum/platforms/_sources.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,7 @@ version: 2
sources:
- name: archipelago_ethereum
- name: superrare_ethereum
- name: cryptopunks_ethereum
tables:
- name: CryptoPunksMarket_call_acceptBidForPunk
# sources are already defined in the "older" directories and can't be defined twice. In the end we should migrate them to here.
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
{{ config(
schema = 'collectionswap_ethereum',
alias ='base_trades',
partition_by = ['block_date'],
materialized = 'incremental',
file_format = 'delta',
incremental_strategy = 'merge',
unique_key = ['block_number','tx_hash','sub_tx_trade_id'],
)
}}

{%- set project_start_date = '2023-03-29' %}

WITH
raw_trades as (
select *
, row_number() over (partition by tx_hash order by evt_index asc, sub_order_id asc) as sub_tx_trade_id
from(
select
block_number, block_time, evt_index, tx_hash, buyer, seller,
posexplode(nft_id_array) as (sub_order_id, nft_token_id),
cast(1 as int) as nft_amount,
price_raw/number_of_items as price_raw,
platform_fee_amount_raw/number_of_items as platform_fee_amount_raw,
royalty_fee_amount_raw/number_of_items as royalty_fee_amount_raw,
trade_fee_amount_raw/number_of_items as trade_fee_amount_raw,
royalty_fee_address,
project_contract_address,
number_of_items,
'secondary' as trade_type,
trade_category
from(
select
evt_block_number as block_number
,evt_block_time as block_time
,evt_index
,evt_tx_hash as tx_hash
,null as buyer
,contract_address as seller
,'Buy' as trade_category
,nftIds as nft_id_array
,cardinality(nftIds) as number_of_items
,cast(outputAmount as decimal(38)) as price_raw
,cast(protocolFee as decimal(38)) as platform_fee_amount_raw
,get_json_object(royaltyDue[0], '$.amount') as royalty_fee_amount_raw
,get_json_object(royaltyDue[0], '$.recipient') as royalty_fee_address
,cast(tradeFee as decimal(38)) as trade_fee_amount_raw
,contract_address as project_contract_address
from {{ source('collectionswap_ethereum','CollectionPool_evt_SwapNFTOutPool') }} e
{% if is_incremental() %}
WHERE evt_block_time >= date_trunc("day", now() - interval '1 week')
{% else %}
WHERE evt_block_time >= '{{project_start_date}}'
{% endif %}
union all
select
evt_block_number as block_number
,evt_block_time as block_time
,evt_index
,evt_tx_hash as tx_hash
,contract_address as buyer
,null as seller
,'Sell' as trade_category
,nftIds as nft_id_array
,cardinality(nftIds) as number_of_items
,cast(inputAmount + protocolFee + cast(get_json_object(royaltyDue[0], '$.amount') as decimal(38)) as decimal(38)) as price_raw
,cast(protocolFee as decimal(38)) as platform_fee_amount_raw
,get_json_object(royaltyDue[0], '$.amount') as royalty_fee_amount_raw
,get_json_object(royaltyDue[0], '$.recipient') as royalty_fee_address
,cast(tradeFee as decimal(38)) as trade_fee_amount_raw
,contract_address as project_contract_address
from {{ source('collectionswap_ethereum','CollectionPool_evt_SwapNFTInPool') }} e
{% if is_incremental() %}
WHERE evt_block_time >= date_trunc("day", now() - interval '1 week')
{% else %}
WHERE evt_block_time >= '{{project_start_date}}'
{% endif %}
)
)
),

base_trades as (
select
t.*,
p.nft_contract_address,
p.token_address as currency_contract
from raw_trades t
left join {{ ref('collectionswap_ethereum_pools') }} p
on t.project_contract_address = p.pool_address
)

-- results
SELECT
date_trunc('day',block_time ) as block_date
, block_time
, block_number
, tx_hash
, project_contract_address
, buyer
, seller
, nft_contract_address
, nft_token_id
, nft_amount
, trade_type
, trade_category
, currency_contract
, cast(price_raw as decimal(38)) as price_raw
, cast(platform_fee_amount_raw as decimal(38)) as platform_fee_amount_raw
, cast(royalty_fee_amount_raw as decimal(38)) as royalty_fee_amount_raw
, cast(null as varchar(1)) as platform_fee_address
, royalty_fee_address
, sub_tx_trade_id
FROM base_trades


Loading

0 comments on commit f68e658

Please sign in to comment.