flip/flow-models
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/flow-models.git synced 2026-02-06 13:06:59 +00:00
Code Issues Actions 29 Packages Projects Releases Wiki Activity

AN-1378/nft secondary market sales (#28)

Browse Source 
* nft purchase txs v1

* v2 and tokenflow

* v3 and v4

* comments no changes

* blocto transactions

* nfts

* silver models

* topshot tokenflow

* silver nft sales

* small tweak

* excl outlier
This commit is contained in:
Jack Forgash 2022-06-09 10:44:52 -06:00 committed by GitHub
parent 1eaa67e407
commit f9ec13fa9d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 743 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
models/core/core__fact_nft_sales.sql
View File

@ -23,9 +23,13 @@ gold_nfts AS (
seller,
price,
currency,
tx_succeeded
tx_succeeded,
tokenflow,
counterparties
FROM
silver_nfts
WHERE
tx_id != '8620792f30d607a35eb5a7ffe6ea2a088d448f1b706e8585ca8ae8697655e6fa'
)
SELECT
*

21
models/core/core__fact_nft_sales.yml
View File

@ -94,6 +94,7 @@ models:
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: currency
description: "{{ doc('currency') }}"
@ -111,3 +112,23 @@ models:
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- BOOLEAN
- name: tokenflow
description: "{{ doc('tokenflow') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- ARRAY
- OBJECT
- VARIANT
- name: counterparties
description: "{{ doc('counterparties') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- ARRAY
- OBJECT
- VARIANT

5
models/descriptions/counterparties.md Normal file
View File

@ -0,0 +1,5 @@
{% docs counterparties %}
An array containing the addresses interacted with during the tokenflow of the transaction. This contains all payers and recipients of tokens within the transaction.
{% enddocs %}

5
models/descriptions/event_data_listing.md Normal file
View File

@ -0,0 +1,5 @@
{% docs event_data_listing %}
The raw event data from the ListingCompleted event.
{% enddocs %}

2
models/descriptions/marketplace.md
View File

@ -1,5 +1,5 @@
{% docs marketplace %}
Contract address for the marketplace where the transaction occurred.
Contract address for the marketplace where the transaction occurred. Flow uses a general-purpose contract at `A.4eb8a10cb9f87357.NFTStorefront` for a significant number of sales. This column does not necessarily indicate the website or specific platform on which the sale occurred.
{% enddocs %}

5
models/descriptions/num_steps.md Normal file
View File

@ -0,0 +1,5 @@
{% docs num_steps %}
The number of steps (events) taken by tokens within the transaction. This includes only tokenflow steps, and is not the count of total events within the transaction.
{% enddocs %}

5
models/descriptions/step_action.md Normal file
View File

@ -0,0 +1,5 @@
{% docs step_action %}
An array containing the actions taken at index `n` within the tokenflow.
{% enddocs %}

5
models/descriptions/step_data.md Normal file
View File

@ -0,0 +1,5 @@
{% docs step_data %}
An array containing the data passed at index `n` within the tokenflow.
{% enddocs %}

5
models/descriptions/tokenflow.md Normal file
View File

@ -0,0 +1,5 @@
{% docs tokenflow %}
An array of events tracking the flow of tokens, in event_index order, for a transaction. This is constructed from the following events within a transaction: `TokensDeposited`, `TokensWithdrawn`, `ForwardedDeposit`.
{% enddocs %}

50
models/silver/silver__nft_sales.sql
View File

@ -16,8 +16,56 @@ WITH topshot AS (
WHERE
_ingested_at :: DATE >= CURRENT_DATE - 2
{% endif %}
),
secondary AS (
SELECT
*
FROM
{{ ref('silver__nft_transactions_secondary_market') }}
{% if is_incremental() %}
WHERE
_ingested_at :: DATE >= CURRENT_DATE - 2
{% endif %}
),
combo AS (
SELECT
tx_id,
block_height,
block_timestamp,
marketplace,
nft_collection,
nft_id,
buyer,
seller,
price,
currency,
tx_succeeded,
_ingested_at,
tokenflow,
counterparties
FROM
topshot
UNION
SELECT
tx_id,
block_height,
block_timestamp,
marketplace,
nft_collection,
nft_id,
buyer,
seller,
price,
currency,
tx_succeeded,
_ingested_at,
tokenflow,
counterparties
FROM
secondary
)
SELECT
*
FROM
topshot
combo

24
models/silver/silver__nft_sales.yml
View File

@ -4,7 +4,7 @@ version: 2
models:
- name: silver__nft_sales
description: |-
NFT market sales on the Flow blockchain.
NFT market sales on the Flow blockchain. This table will only contain successful transactions, as failed transactions will not have the requisite events to determine it was an attempted NFT purchase.
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
@ -118,4 +118,24 @@ models:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_NTZ
- TIMESTAMP_NTZ
- name: tokenflow
description: "{{ doc('tokenflow') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- ARRAY
- OBJECT
- VARIANT
- name: counterparties
description: "{{ doc('counterparties') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- ARRAY
- OBJECT
- VARIANT

61
models/silver/silver__nft_topshot_sales.sql
View File

@ -24,7 +24,7 @@ moment_data AS (
tx_id,
event_contract :: STRING AS marketplace,
event_data :id :: STRING AS nft_id,
event_data :price :: NUMBER AS price,
event_data :price :: DOUBLE AS price,
event_data :seller :: STRING AS seller,
tx_succeeded,
_ingested_at
@ -83,8 +83,65 @@ combo AS (
moment_data
LEFT JOIN currency_data USING (tx_id)
LEFT JOIN nft_data USING (tx_id)
),
step_data AS (
SELECT
tx_id,
event_index,
event_type,
event_data
FROM
{{ ref('silver__events_final') }}
WHERE
tx_id IN (
SELECT
tx_id
FROM
combo
)
AND event_type IN (
'TokensWithdrawn',
'TokensDeposited',
'ForwardedDeposit'
)
),
counterparty_data AS (
SELECT
tx_id,
ARRAY_AGG(OBJECT_CONSTRUCT(event_type, event_data)) within GROUP (
ORDER BY
event_index
) AS tokenflow,
ARRAY_AGG(COALESCE(event_data :to, event_data :from) :: STRING) within GROUP (
ORDER BY
event_index
) AS counterparties
FROM
step_data
GROUP BY
1
),
FINAL AS (
SELECT
C.tx_id,
block_height,
block_timestamp,
marketplace,
nft_collection,
nft_id,
buyer,
seller,
price,
currency,
tx_succeeded,
_ingested_at,
cd.tokenflow,
cd.counterparties
FROM
combo C
LEFT JOIN counterparty_data cd USING (tx_id)
)
SELECT
*
FROM
combo
FINAL

23
models/silver/silver__nft_topshot_sales.yml
View File

@ -94,6 +94,7 @@ models:
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: currency
description: "{{ doc('currency') }}"
@ -118,4 +119,24 @@ models:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_NTZ
- TIMESTAMP_NTZ
- name: tokenflow
description: "{{ doc('tokenflow') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- ARRAY
- OBJECT
- VARIANT
- name: counterparties
description: "{{ doc('counterparties') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- ARRAY
- OBJECT
- VARIANT

192
models/silver/silver__nft_transactions_secondary_market.sql Normal file
View File

@ -0,0 +1,192 @@
{{ config(
materialized = 'incremental',
incremental_strategy = 'delete+insert',
cluster_by = ['_ingested_at::DATE, block_timestamp::DATE'],
unique_key = 'tx_id'
) }}
WITH silver_events AS (
SELECT
*
FROM
{{ ref('silver__events_final') }}
{% if is_incremental() %}
WHERE
_ingested_at :: DATE >= CURRENT_DATE - 2
{% endif %}
),
listing_data AS (
SELECT
tx_id,
block_timestamp,
block_height,
tx_succeeded,
event_index AS event_index_listing,
event_contract AS event_contract_listing,
event_data AS event_data_listing,
event_data :nftID :: STRING AS nft_id_listing,
event_data :nftType :: STRING AS nft_collection_listing,
event_data :purchased :: BOOLEAN AS purchased_listing,
_ingested_at
FROM
silver_events
WHERE
event_type = 'ListingCompleted'
AND event_contract = 'A.4eb8a10cb9f87357.NFTStorefront' -- general storefront
AND purchased_listing = TRUE
),
excl_multi_buys AS (
SELECT
tx_id,
COUNT(1) AS record_count
FROM
listing_data
GROUP BY
1
HAVING
record_count = 1
),
purchase_data AS (
SELECT
tx_id,
event_contract AS currency,
event_data :amount :: DOUBLE AS amount,
event_data :from :: STRING AS buyer_purchase
FROM
silver_events
WHERE
tx_id IN (
SELECT
tx_id
FROM
excl_multi_buys
)
AND event_index = 0
),
seller_data AS (
SELECT
tx_id,
event_index AS event_index_seller,
event_contract AS nft_collection_seller,
event_data :from :: STRING AS seller,
event_data :id :: STRING AS nft_id_seller
FROM
silver_events
WHERE
tx_id IN (
SELECT
tx_id
FROM
excl_multi_buys
)
AND event_type = 'Withdraw'
),
deposit_data AS (
SELECT
tx_id,
event_contract AS nft_collection_deposit,
event_data :id :: STRING AS nft_id_deposit,
event_data :to :: STRING AS buyer_deposit
FROM
silver_events
WHERE
tx_id IN (
SELECT
tx_id
FROM
excl_multi_buys
)
AND event_type = 'Deposit'
),
nft_sales AS (
SELECT
*
FROM
listing_data
LEFT JOIN purchase_data USING (tx_id)
LEFT JOIN seller_data USING (tx_id)
LEFT JOIN deposit_data USING (tx_id)
WHERE
tx_id IN (
SELECT
tx_id
FROM
excl_multi_buys
)
),
step_data AS (
SELECT
tx_id,
event_index,
event_type,
event_data
FROM
{{ ref('silver__events_final') }}
WHERE
tx_id IN (
SELECT
tx_id
FROM
nft_sales
)
AND event_type IN (
'TokensWithdrawn',
'TokensDeposited',
'ForwardedDeposit'
)
),
counterparty_data AS (
SELECT
tx_id,
ARRAY_AGG(OBJECT_CONSTRUCT(event_type, event_data)) within GROUP (
ORDER BY
event_index
) AS tokenflow,
ARRAY_SIZE(tokenflow) AS steps,
ARRAY_AGG(event_type) within GROUP (
ORDER BY
event_index
) AS action,
ARRAY_AGG(event_data) within GROUP (
ORDER BY
event_index
) AS step_data,
ARRAY_AGG(COALESCE(event_data :to, event_data :from) :: STRING) within GROUP (
ORDER BY
event_index
) AS counterparties
FROM
step_data
GROUP BY
1
),
FINAL AS (
SELECT
ns.tx_id,
block_timestamp,
block_height,
event_contract_listing AS marketplace,
event_data_listing,
nft_collection_seller AS nft_collection,
nft_id_listing AS nft_id,
currency,
amount AS price,
seller,
buyer_deposit AS buyer,
cd.tokenflow,
cd.steps AS num_steps,
cd.action AS step_action,
cd.step_data,
cd.counterparties,
tx_succeeded,
_ingested_at
FROM
nft_sales ns
LEFT JOIN counterparty_data cd USING (tx_id)
)
SELECT
*
FROM
FINAL

180
models/silver/silver__nft_transactions_secondary_market.yml Normal file
View File

@ -0,0 +1,180 @@
version: 2
models:
- name: silver__nft_transactions_secondary_market
description: |-
This table filters all NFT sales that interact with the general purpose contract `A.4eb8a10cb9f87357.NFTStorefront`. Transactions that purchase multiple NFTs in a single transaction are presently excluded from this table!
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- tx_id
columns:
- name: tx_id
description: "{{ doc('tx_id') }}"
tests:
- not_null
- unique
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- STRING
- VARCHAR
- name: block_height
description: "{{ doc('block_height') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: block_timestamp
description: "{{ doc('block_timestamp') }}"
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: day
interval: 1
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_NTZ
- name: marketplace
description: "{{ doc('marketplace') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- STRING
- VARCHAR
- name: event_data_listing
description: "{{ doc('event_data_listing') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- ARRAY
- VARIANT
- OBJECT
- name: nft_collection
description: "{{ doc('nft_collection') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- STRING
- VARCHAR
- name: nft_id
description: "{{ doc('nft_id') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- STRING
- VARCHAR
- NUMBER
- name: currency
description: "{{ doc('currency') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- STRING
- VARCHAR
- name: price
description: "{{ doc('price') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: seller
description: "{{ doc('seller') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- STRING
- VARCHAR
- name: buyer
description: "{{ doc('buyer') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- STRING
- VARCHAR
- name: tokenflow
description: "{{ doc('tokenflow') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- ARRAY
- OBJECT
- VARIANT
- name: num_steps
description: "{{ doc('num_steps') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- name: step_action
description: "{{ doc('step_action') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- ARRAY
- OBJECT
- VARIANT
- name: step_data
description: "{{ doc('step_data') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- ARRAY
- OBJECT
- VARIANT
- name: counterparties
description: "{{ doc('counterparties') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- ARRAY
- OBJECT
- VARIANT
- name: tx_succeeded
description: "{{ doc('tx_succeeded') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- BOOLEAN
- name: _ingested_at
description: "{{ doc('_ingested_at') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_NTZ

31
models/silver/silver__transactions_blocto.sql Normal file
View File

@ -0,0 +1,31 @@
{{ config(
materialized = 'incremental',
cluster_by = ['_ingested_at::DATE', 'block_timestamp::DATE'],
unique_key = 'tx_id',
incremental_strategy = 'delete+insert'
) }}
WITH silver_txs AS (
SELECT
*
FROM
{{ ref('silver__transactions') }}
{% if is_incremental() %}
WHERE
_ingested_at :: DATE >= CURRENT_DATE - 2
{% endif %}
),
blocto_txs AS (
SELECT
*
FROM
silver_txs
WHERE
LOWER(payer) = LOWER('0x55AD22F01EF568A1') -- Blocto network fee paying address
)
SELECT
*
FROM
blocto_txs

131
models/silver/silver__transactions_blocto.yml Normal file
View File

@ -0,0 +1,131 @@
version: 2
models:
- name: silver__transactions_blocto
description: |-
This table records all the transactions of the FLOW blockchain that are sent from a Blocto Wallet, as determined by the payer address. Blocto subsidizes network fees, so we can infer activity from the payer column.
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- tx_id
- block_height
columns:
- name: tx_id
description: "{{ doc('tx_id') }}"
tests:
- not_null
- unique
- name: block_timestamp
description: "{{ doc('block_timestamp') }}"
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: day
interval: 1
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_NTZ
- name: block_height
description: "{{ doc('block_height') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: chain_id
description: "{{ doc('chain_id') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- STRING
- VARCHAR
- name: tx_index
description: tbd
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- name: proposer
description: "{{ doc('proposer') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- STRING
- VARCHAR
- name: payer
description: "{{ doc('payer') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- STRING
- VARCHAR
- name: authorizers
description: "{{ doc('authorizers') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- ARRAY
- name: count_authorizers
description: "{{ doc('count_authorizers') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- name: gas_limit
description: "{{ doc('gas_limit') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- name: transaction_result
description: "{{ doc('transaction_result') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- VARIANT
- name: tx_succeeded
description: "{{ doc('tx_succeeded') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- BOOLEAN
- name: error_msg
description: tbd
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- STRING
- VARCHAR
- name: _ingested_at
description: "{{ doc('_ingested_at') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_NTZ