diff --git a/models/core/core__fact_nft_sales.sql b/models/core/core__fact_nft_sales.sql index 7b440ef..6ad08c3 100644 --- a/models/core/core__fact_nft_sales.sql +++ b/models/core/core__fact_nft_sales.sql @@ -23,9 +23,13 @@ gold_nfts AS ( seller, price, currency, - tx_succeeded + tx_succeeded, + tokenflow, + counterparties FROM silver_nfts + WHERE + tx_id != '8620792f30d607a35eb5a7ffe6ea2a088d448f1b706e8585ca8ae8697655e6fa' ) SELECT * diff --git a/models/core/core__fact_nft_sales.yml b/models/core/core__fact_nft_sales.yml index 0c08c39..e5a49b8 100644 --- a/models/core/core__fact_nft_sales.yml +++ b/models/core/core__fact_nft_sales.yml @@ -94,6 +94,7 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER + - FLOAT - name: currency description: "{{ doc('currency') }}" @@ -111,3 +112,23 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - BOOLEAN + + - name: tokenflow + description: "{{ doc('tokenflow') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - ARRAY + - OBJECT + - VARIANT + + - name: counterparties + description: "{{ doc('counterparties') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - ARRAY + - OBJECT + - VARIANT \ No newline at end of file diff --git a/models/descriptions/counterparties.md b/models/descriptions/counterparties.md new file mode 100644 index 0000000..1ff0c81 --- /dev/null +++ b/models/descriptions/counterparties.md @@ -0,0 +1,5 @@ +{% docs counterparties %} + +An array containing the addresses interacted with during the tokenflow of the transaction. This contains all payers and recipients of tokens within the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/event_data_listing.md b/models/descriptions/event_data_listing.md new file mode 100644 index 0000000..f316e8e --- /dev/null +++ b/models/descriptions/event_data_listing.md @@ -0,0 +1,5 @@ +{% docs event_data_listing %} + +The raw event data from the ListingCompleted event. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/marketplace.md b/models/descriptions/marketplace.md index 8047aae..410fda2 100644 --- a/models/descriptions/marketplace.md +++ b/models/descriptions/marketplace.md @@ -1,5 +1,5 @@ {% docs marketplace %} -Contract address for the marketplace where the transaction occurred. +Contract address for the marketplace where the transaction occurred. Flow uses a general-purpose contract at `A.4eb8a10cb9f87357.NFTStorefront` for a significant number of sales. This column does not necessarily indicate the website or specific platform on which the sale occurred. {% enddocs %} \ No newline at end of file diff --git a/models/descriptions/num_steps.md b/models/descriptions/num_steps.md new file mode 100644 index 0000000..b7a1fe8 --- /dev/null +++ b/models/descriptions/num_steps.md @@ -0,0 +1,5 @@ +{% docs num_steps %} + +The number of steps (events) taken by tokens within the transaction. This includes only tokenflow steps, and is not the count of total events within the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/step_action.md b/models/descriptions/step_action.md new file mode 100644 index 0000000..3fc257e --- /dev/null +++ b/models/descriptions/step_action.md @@ -0,0 +1,5 @@ +{% docs step_action %} + +An array containing the actions taken at index `n` within the tokenflow. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/step_data.md b/models/descriptions/step_data.md new file mode 100644 index 0000000..80d88bc --- /dev/null +++ b/models/descriptions/step_data.md @@ -0,0 +1,5 @@ +{% docs step_data %} + +An array containing the data passed at index `n` within the tokenflow. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tokenflow.md b/models/descriptions/tokenflow.md new file mode 100644 index 0000000..6c2c737 --- /dev/null +++ b/models/descriptions/tokenflow.md @@ -0,0 +1,5 @@ +{% docs tokenflow %} + +An array of events tracking the flow of tokens, in event_index order, for a transaction. This is constructed from the following events within a transaction: `TokensDeposited`, `TokensWithdrawn`, `ForwardedDeposit`. + +{% enddocs %} \ No newline at end of file diff --git a/models/silver/silver__nft_sales.sql b/models/silver/silver__nft_sales.sql index e72f720..a8b3522 100644 --- a/models/silver/silver__nft_sales.sql +++ b/models/silver/silver__nft_sales.sql @@ -16,8 +16,56 @@ WITH topshot AS ( WHERE _ingested_at :: DATE >= CURRENT_DATE - 2 {% endif %} +), +secondary AS ( + SELECT + * + FROM + {{ ref('silver__nft_transactions_secondary_market') }} + +{% if is_incremental() %} +WHERE + _ingested_at :: DATE >= CURRENT_DATE - 2 +{% endif %} +), +combo AS ( + SELECT + tx_id, + block_height, + block_timestamp, + marketplace, + nft_collection, + nft_id, + buyer, + seller, + price, + currency, + tx_succeeded, + _ingested_at, + tokenflow, + counterparties + FROM + topshot + UNION + SELECT + tx_id, + block_height, + block_timestamp, + marketplace, + nft_collection, + nft_id, + buyer, + seller, + price, + currency, + tx_succeeded, + _ingested_at, + tokenflow, + counterparties + FROM + secondary ) SELECT * FROM - topshot + combo diff --git a/models/silver/silver__nft_sales.yml b/models/silver/silver__nft_sales.yml index 2989b5b..8c24fd1 100644 --- a/models/silver/silver__nft_sales.yml +++ b/models/silver/silver__nft_sales.yml @@ -4,7 +4,7 @@ version: 2 models: - name: silver__nft_sales description: |- - NFT market sales on the Flow blockchain. + NFT market sales on the Flow blockchain. This table will only contain successful transactions, as failed transactions will not have the requisite events to determine it was an attempted NFT purchase. tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: @@ -118,4 +118,24 @@ models: - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - - TIMESTAMP_NTZ \ No newline at end of file + - TIMESTAMP_NTZ + + - name: tokenflow + description: "{{ doc('tokenflow') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - ARRAY + - OBJECT + - VARIANT + + - name: counterparties + description: "{{ doc('counterparties') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - ARRAY + - OBJECT + - VARIANT \ No newline at end of file diff --git a/models/silver/silver__nft_topshot_sales.sql b/models/silver/silver__nft_topshot_sales.sql index 0f9966e..ec34a6c 100644 --- a/models/silver/silver__nft_topshot_sales.sql +++ b/models/silver/silver__nft_topshot_sales.sql @@ -24,7 +24,7 @@ moment_data AS ( tx_id, event_contract :: STRING AS marketplace, event_data :id :: STRING AS nft_id, - event_data :price :: NUMBER AS price, + event_data :price :: DOUBLE AS price, event_data :seller :: STRING AS seller, tx_succeeded, _ingested_at @@ -83,8 +83,65 @@ combo AS ( moment_data LEFT JOIN currency_data USING (tx_id) LEFT JOIN nft_data USING (tx_id) +), +step_data AS ( + SELECT + tx_id, + event_index, + event_type, + event_data + FROM + {{ ref('silver__events_final') }} + WHERE + tx_id IN ( + SELECT + tx_id + FROM + combo + ) + AND event_type IN ( + 'TokensWithdrawn', + 'TokensDeposited', + 'ForwardedDeposit' + ) +), +counterparty_data AS ( + SELECT + tx_id, + ARRAY_AGG(OBJECT_CONSTRUCT(event_type, event_data)) within GROUP ( + ORDER BY + event_index + ) AS tokenflow, + ARRAY_AGG(COALESCE(event_data :to, event_data :from) :: STRING) within GROUP ( + ORDER BY + event_index + ) AS counterparties + FROM + step_data + GROUP BY + 1 +), +FINAL AS ( + SELECT + C.tx_id, + block_height, + block_timestamp, + marketplace, + nft_collection, + nft_id, + buyer, + seller, + price, + currency, + tx_succeeded, + _ingested_at, + cd.tokenflow, + cd.counterparties + FROM + combo C + LEFT JOIN counterparty_data cd USING (tx_id) ) SELECT * FROM - combo + FINAL diff --git a/models/silver/silver__nft_topshot_sales.yml b/models/silver/silver__nft_topshot_sales.yml index e53ce70..6a9c4cd 100644 --- a/models/silver/silver__nft_topshot_sales.yml +++ b/models/silver/silver__nft_topshot_sales.yml @@ -94,6 +94,7 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER + - FLOAT - name: currency description: "{{ doc('currency') }}" @@ -118,4 +119,24 @@ models: - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - - TIMESTAMP_NTZ \ No newline at end of file + - TIMESTAMP_NTZ + + - name: tokenflow + description: "{{ doc('tokenflow') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - ARRAY + - OBJECT + - VARIANT + + - name: counterparties + description: "{{ doc('counterparties') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - ARRAY + - OBJECT + - VARIANT \ No newline at end of file diff --git a/models/silver/silver__nft_transactions_secondary_market.sql b/models/silver/silver__nft_transactions_secondary_market.sql new file mode 100644 index 0000000..6cfefb8 --- /dev/null +++ b/models/silver/silver__nft_transactions_secondary_market.sql @@ -0,0 +1,192 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + cluster_by = ['_ingested_at::DATE, block_timestamp::DATE'], + unique_key = 'tx_id' +) }} + +WITH silver_events AS ( + + SELECT + * + FROM + {{ ref('silver__events_final') }} + +{% if is_incremental() %} +WHERE + _ingested_at :: DATE >= CURRENT_DATE - 2 +{% endif %} +), +listing_data AS ( + SELECT + tx_id, + block_timestamp, + block_height, + tx_succeeded, + event_index AS event_index_listing, + event_contract AS event_contract_listing, + event_data AS event_data_listing, + event_data :nftID :: STRING AS nft_id_listing, + event_data :nftType :: STRING AS nft_collection_listing, + event_data :purchased :: BOOLEAN AS purchased_listing, + _ingested_at + FROM + silver_events + WHERE + event_type = 'ListingCompleted' + AND event_contract = 'A.4eb8a10cb9f87357.NFTStorefront' -- general storefront + AND purchased_listing = TRUE +), +excl_multi_buys AS ( + SELECT + tx_id, + COUNT(1) AS record_count + FROM + listing_data + GROUP BY + 1 + HAVING + record_count = 1 +), +purchase_data AS ( + SELECT + tx_id, + event_contract AS currency, + event_data :amount :: DOUBLE AS amount, + event_data :from :: STRING AS buyer_purchase + FROM + silver_events + WHERE + tx_id IN ( + SELECT + tx_id + FROM + excl_multi_buys + ) + AND event_index = 0 +), +seller_data AS ( + SELECT + tx_id, + event_index AS event_index_seller, + event_contract AS nft_collection_seller, + event_data :from :: STRING AS seller, + event_data :id :: STRING AS nft_id_seller + FROM + silver_events + WHERE + tx_id IN ( + SELECT + tx_id + FROM + excl_multi_buys + ) + AND event_type = 'Withdraw' +), +deposit_data AS ( + SELECT + tx_id, + event_contract AS nft_collection_deposit, + event_data :id :: STRING AS nft_id_deposit, + event_data :to :: STRING AS buyer_deposit + FROM + silver_events + WHERE + tx_id IN ( + SELECT + tx_id + FROM + excl_multi_buys + ) + AND event_type = 'Deposit' +), +nft_sales AS ( + SELECT + * + FROM + listing_data + LEFT JOIN purchase_data USING (tx_id) + LEFT JOIN seller_data USING (tx_id) + LEFT JOIN deposit_data USING (tx_id) + WHERE + tx_id IN ( + SELECT + tx_id + FROM + excl_multi_buys + ) +), +step_data AS ( + SELECT + tx_id, + event_index, + event_type, + event_data + FROM + {{ ref('silver__events_final') }} + WHERE + tx_id IN ( + SELECT + tx_id + FROM + nft_sales + ) + AND event_type IN ( + 'TokensWithdrawn', + 'TokensDeposited', + 'ForwardedDeposit' + ) +), +counterparty_data AS ( + SELECT + tx_id, + ARRAY_AGG(OBJECT_CONSTRUCT(event_type, event_data)) within GROUP ( + ORDER BY + event_index + ) AS tokenflow, + ARRAY_SIZE(tokenflow) AS steps, + ARRAY_AGG(event_type) within GROUP ( + ORDER BY + event_index + ) AS action, + ARRAY_AGG(event_data) within GROUP ( + ORDER BY + event_index + ) AS step_data, + ARRAY_AGG(COALESCE(event_data :to, event_data :from) :: STRING) within GROUP ( + ORDER BY + event_index + ) AS counterparties + FROM + step_data + GROUP BY + 1 +), +FINAL AS ( + SELECT + ns.tx_id, + block_timestamp, + block_height, + event_contract_listing AS marketplace, + event_data_listing, + nft_collection_seller AS nft_collection, + nft_id_listing AS nft_id, + currency, + amount AS price, + seller, + buyer_deposit AS buyer, + cd.tokenflow, + cd.steps AS num_steps, + cd.action AS step_action, + cd.step_data, + cd.counterparties, + tx_succeeded, + _ingested_at + FROM + nft_sales ns + LEFT JOIN counterparty_data cd USING (tx_id) +) +SELECT + * +FROM + FINAL diff --git a/models/silver/silver__nft_transactions_secondary_market.yml b/models/silver/silver__nft_transactions_secondary_market.yml new file mode 100644 index 0000000..1499f47 --- /dev/null +++ b/models/silver/silver__nft_transactions_secondary_market.yml @@ -0,0 +1,180 @@ + +version: 2 + +models: + - name: silver__nft_transactions_secondary_market + description: |- + This table filters all NFT sales that interact with the general purpose contract `A.4eb8a10cb9f87357.NFTStorefront`. Transactions that purchase multiple NFTs in a single transaction are presently excluded from this table! + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_id + + columns: + - name: tx_id + description: "{{ doc('tx_id') }}" + tests: + - not_null + - unique + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: block_height + description: "{{ doc('block_height') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: block_timestamp + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + + - name: marketplace + description: "{{ doc('marketplace') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: event_data_listing + description: "{{ doc('event_data_listing') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - ARRAY + - VARIANT + - OBJECT + + - name: nft_collection + description: "{{ doc('nft_collection') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: nft_id + description: "{{ doc('nft_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - NUMBER + + - name: currency + description: "{{ doc('currency') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: price + description: "{{ doc('price') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: seller + description: "{{ doc('seller') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: buyer + description: "{{ doc('buyer') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: tokenflow + description: "{{ doc('tokenflow') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - ARRAY + - OBJECT + - VARIANT + + - name: num_steps + description: "{{ doc('num_steps') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + + - name: step_action + description: "{{ doc('step_action') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - ARRAY + - OBJECT + - VARIANT + + - name: step_data + description: "{{ doc('step_data') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - ARRAY + - OBJECT + - VARIANT + + - name: counterparties + description: "{{ doc('counterparties') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - ARRAY + - OBJECT + - VARIANT + + - name: tx_succeeded + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + + - name: _ingested_at + description: "{{ doc('_ingested_at') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/silver/silver__transactions_blocto.sql b/models/silver/silver__transactions_blocto.sql new file mode 100644 index 0000000..3a945c5 --- /dev/null +++ b/models/silver/silver__transactions_blocto.sql @@ -0,0 +1,31 @@ +{{ config( + materialized = 'incremental', + cluster_by = ['_ingested_at::DATE', 'block_timestamp::DATE'], + unique_key = 'tx_id', + incremental_strategy = 'delete+insert' +) }} + +WITH silver_txs AS ( + + SELECT + * + FROM + {{ ref('silver__transactions') }} + +{% if is_incremental() %} +WHERE + _ingested_at :: DATE >= CURRENT_DATE - 2 +{% endif %} +), +blocto_txs AS ( + SELECT + * + FROM + silver_txs + WHERE + LOWER(payer) = LOWER('0x55AD22F01EF568A1') -- Blocto network fee paying address +) +SELECT + * +FROM + blocto_txs diff --git a/models/silver/silver__transactions_blocto.yml b/models/silver/silver__transactions_blocto.yml new file mode 100644 index 0000000..a4ae612 --- /dev/null +++ b/models/silver/silver__transactions_blocto.yml @@ -0,0 +1,131 @@ + +version: 2 + +models: + - name: silver__transactions_blocto + description: |- + This table records all the transactions of the FLOW blockchain that are sent from a Blocto Wallet, as determined by the payer address. Blocto subsidizes network fees, so we can infer activity from the payer column. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_id + - block_height + + columns: + - name: tx_id + description: "{{ doc('tx_id') }}" + tests: + - not_null + - unique + + - name: block_timestamp + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + + - name: block_height + description: "{{ doc('block_height') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: chain_id + description: "{{ doc('chain_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: tx_index + description: tbd + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + + - name: proposer + description: "{{ doc('proposer') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: payer + description: "{{ doc('payer') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: authorizers + description: "{{ doc('authorizers') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - ARRAY + + - name: count_authorizers + description: "{{ doc('count_authorizers') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + + - name: gas_limit + description: "{{ doc('gas_limit') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + + - name: transaction_result + description: "{{ doc('transaction_result') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + + - name: tx_succeeded + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + + - name: error_msg + description: tbd + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: _ingested_at + description: "{{ doc('_ingested_at') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ \ No newline at end of file