From 77d51bd5dfacf5a3dde533a4fc395ce71fa84495 Mon Sep 17 00:00:00 2001 From: tarikceric <46071768+tarikceric@users.noreply.github.com> Date: Wed, 12 Mar 2025 11:44:10 -0700 Subject: [PATCH] An 5755/gold ez nft sales (#811) * docs update * silver and gold tables * reformat * clean up * filter to suceeded sales * coalesce for pk * update desc * join to metadata and use set sol var --- models/descriptions/__overview__.md | 1 + models/gold/nft/nft__ez_nft_sales.sql | 240 +++++++++++++ models/gold/nft/nft__ez_nft_sales.yml | 167 +++++++++ ...silver__nft_sales_legacy_combined_view.sql | 338 ++++++++++++++++++ 4 files changed, 746 insertions(+) create mode 100644 models/gold/nft/nft__ez_nft_sales.sql create mode 100644 models/gold/nft/nft__ez_nft_sales.yml create mode 100644 models/silver/nfts/silver__nft_sales_legacy_combined_view.sql diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index 9f9e0889..ad2f4864 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -69,6 +69,7 @@ There is more information on how to use dbt docs in the last section of this doc - [dim_nft_metadata](#!/model/model.solana_models.nft__dim_nft_metadata) - [fact_nft_mints](#!/model/model.solana_models.nft__fact_nft_mints) - [fact_nft_sales](#!/model/model.solana_models.nft__fact_nft_sales) +- [ez_nft_sales](#!/model/model.solana_models.nft__ez_nft_sales) - [fact_nft_burn_actions](#!/model/model.solana_models.nft__fact_nft_burn_actions) - [fact_nft_mint_actions](#!/model/model.solana_models.nft__fact_nft_mint_actions) diff --git a/models/gold/nft/nft__ez_nft_sales.sql b/models/gold/nft/nft__ez_nft_sales.sql new file mode 100644 index 00000000..9e0f1081 --- /dev/null +++ b/models/gold/nft/nft__ez_nft_sales.sql @@ -0,0 +1,240 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'NFT' }}}, + unique_key = ['ez_nft_sales_id'], + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], + cluster_by = ['block_timestamp::DATE','is_compressed','program_id'], + merge_exclude_columns = ["inserted_timestamp"], + post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(tx_id, buyer_address, seller_address, mint, marketplace)'), + tags = ['scheduled_non_core'] +) }} + +{% if execute %} + + {% set SOL_MINT = 'So11111111111111111111111111111111111111111' %} + {% set magic_eden_switchover_block_timestamp = '2024-03-16' %} + + {% if is_incremental() %} + {% set query %} + SELECT MAX(modified_timestamp) AS max_modified_timestamp + FROM {{ this }} + {% endset %} + + {% set max_modified_timestamp = run_query(query).columns[0].values()[0] %} + {% endif %} +{% endif %} + +{% set standard_platforms = [ + {'name': 'solanart', 'marketplace': 'solanart','marketplace_version': 'v1'}, + {'name': 'hadeswap_decoded', 'marketplace': 'hadeswap','marketplace_version': 'v1'}, + {'name': 'exchange_art', 'marketplace': 'exchange art', 'marketplace_version': 'v1'}, + {'name': 'amm_sell_decoded', 'marketplace': 'magic eden v2', 'marketplace_version': 'v2'}, + {'name': 'tensorswap', 'marketplace': 'tensorswap', 'marketplace_version': 'v1'}, + {'name': 'solsniper', 'marketplace': 'solsniper', 'marketplace_version': 'v1'}, + {'name': 'tensor_bid', 'marketplace': 'tensorswap', 'marketplace_version': 'v1'}, +] %} + +{% set cnft_platforms = [ + {'name': 'solsniper_cnft', 'marketplace': 'solsniper','marketplace_version': 'v1'}, + {'name': 'tensorswap_cnft', 'marketplace': 'tensorswap', 'marketplace_version': 'v1'}, + {'name': 'magic_eden_cnft', 'marketplace': 'magic eden v3', 'marketplace_version': 'v3'}, +] %} + +{% set multi_token_platforms = [ + {'name': 'magic_eden_v2_decoded', 'marketplace': 'magic eden v2', 'marketplace_version': 'v2'} +] %} + +With base_nft_sales as ( +-- Only select from the deprecated model during the initial FR +{% if not is_incremental() %} +SELECT + marketplace, + marketplace_version, + block_timestamp, + block_id, + tx_id, + succeeded, + index, + inner_index, + program_id, + purchaser, + seller, + mint, + sales_amount, + currency_address, + NULL as tree_authority, + NULL as merkle_tree, + NULL as leaf_index, + FALSE as is_compressed, + nft_sales_legacy_combined_id as ez_nft_sales_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref( + 'silver__nft_sales_legacy_combined_view' + ) }} + WHERE + succeeded +UNION ALL +{% endif %} +-- Only select from active models during incremental + {% for platform in standard_platforms %} + SELECT + '{{ platform.marketplace }}' AS marketplace, + '{{ platform.marketplace_version }}' AS marketplace_version, + block_timestamp, + block_id, + tx_id, + succeeded, + index, + inner_index, + program_id, + purchaser, + seller, + mint, + sales_amount, + '{{ SOL_MINT }}' AS currency_address, + NULL as tree_authority, + NULL as merkle_tree, + NULL as leaf_index, + FALSE as is_compressed, + nft_sales_{{ platform.name }}_id AS ez_nft_sales_id, + COALESCE(inserted_timestamp, '2000-01-01') AS inserted_timestamp, + COALESCE(modified_timestamp, '2000-01-01') AS modified_timestamp + FROM + {{ ref('silver__nft_sales_' ~ platform.name) }} + WHERE + succeeded + {% if is_incremental() %} + AND modified_timestamp >= '{{ max_modified_timestamp }}' + {% endif %} + + union all + {% endfor %} + {% for platform in cnft_platforms %} + SELECT + '{{ platform.marketplace }}' AS marketplace, + '{{ platform.marketplace_version }}' AS marketplace_version, + block_timestamp, + block_id, + tx_id, + succeeded, + index, + inner_index, + program_id, + purchaser, + seller, + mint, + sales_amount, + '{{ SOL_MINT }}' AS currency_address, + tree_authority, + merkle_tree, + leaf_index, + TRUE as is_compressed, + nft_sales_{{ platform.name }}_id AS ez_nft_sales_id, + inserted_timestamp, + modified_timestamp + FROM + {{ ref('silver__nft_sales_' ~ platform.name) }} + WHERE + succeeded + {% if is_incremental() %} + AND modified_timestamp >= '{{ max_modified_timestamp }}' + {% endif %} + union all + {% endfor %} + + {% for platform in multi_token_platforms %} + SELECT + '{{ platform.marketplace }}' AS marketplace, + '{{ platform.marketplace_version }}' AS marketplace_version, + block_timestamp, + block_id, + tx_id, + succeeded, + index, + inner_index, + program_id, + purchaser, + seller, + mint, + sales_amount, + currency_address, + NULL as tree_authority, + NULL as merkle_tree, + NULL as leaf_index, + FALSE as is_compressed, + nft_sales_{{ platform.name }}_id AS ez_nft_sales_id, + inserted_timestamp, + modified_timestamp + FROM + {{ ref('silver__nft_sales_' ~ platform.name) }} + WHERE + succeeded + {% if is_incremental() %} + AND + modified_timestamp >= '{{ max_modified_timestamp }}' + {% else %} + AND + block_timestamp::date >= '{{magic_eden_switchover_block_timestamp}}' + {% endif %} + {% if not loop.last %} + UNION ALL + {% endif %} + {% endfor %} +) + +SELECT + a.marketplace, + a.marketplace_version, + a.block_timestamp, + a.block_id, + a.tx_id, + a.succeeded, + a.index, + a.inner_index, + a.program_id, + a.purchaser AS buyer_address, + a.seller AS seller_address, + a.mint, + b.nft_name, + a.sales_amount AS price, + a.currency_address, + d.symbol AS currency_symbol, + (c.price * a.sales_amount) AS price_usd, + a.tree_authority, + a.merkle_tree, + a.leaf_index, + a.is_compressed, + b.nft_collection_name, + b.collection_id, + b.creators, + b.authority, + b.metadata, + b.image_url, + b.metadata_uri, + a.ez_nft_sales_id, + a.inserted_timestamp, + a.modified_timestamp +FROM + base_nft_sales a +LEFT JOIN + {{ ref('nft__dim_nft_metadata') }} b + ON a.mint = b.mint +LEFT JOIN + {{ ref('price__ez_prices_hourly') }} c + ON ( + CASE + WHEN a.currency_address = '{{ SOL_MINT }}' THEN 'So11111111111111111111111111111111111111112' + ELSE a.currency_address + END + ) = c.token_address + AND DATE_TRUNC('hour', a.block_timestamp) = c.hour +LEFT JOIN + {{ ref('price__ez_asset_metadata') }} d + ON ( + CASE + WHEN a.currency_address = '{{ SOL_MINT }}' THEN 'So11111111111111111111111111111111111111112' + ELSE a.currency_address + END + ) = d.token_address \ No newline at end of file diff --git a/models/gold/nft/nft__ez_nft_sales.yml b/models/gold/nft/nft__ez_nft_sales.yml new file mode 100644 index 00000000..d7efd774 --- /dev/null +++ b/models/gold/nft/nft__ez_nft_sales.yml @@ -0,0 +1,167 @@ +version: 2 +models: + - name: nft__ez_nft_sales + description: A convenience table containing NFT sales across multiple marketplaces, included information on metadata, USD prices and marketplace. Note that USD prices are not available prior to 2021-12-16. + recent_date_filter: &recent_date_filter + config: + where: modified_timestamp >= current_date - 7 + tests: + - reference_tx_missing: + reference_tables: + - 'silver__nft_sales_magic_eden_v2_decoded' + - 'silver__nft_sales_solanart' + - 'silver__nft_sales_hadeswap_decoded' + - 'silver__nft_sales_exchange_art' + - 'silver__nft_sales_amm_sell_decoded' + - 'silver__nft_sales_tensorswap' + - 'silver__nft_sales_solsniper' + - 'silver__nft_sales_tensorswap_cnft' + - 'silver__nft_sales_magic_eden_cnft' + - 'silver__nft_sales_solsniper_cnft' + id_column: 'tx_id' + columns: + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: INDEX + description: Location of the event within the instructions of a transaction + tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: INNER_INDEX + description: Location of the event within the inner instructions of a transaction + - name: PROGRAM_ID + description: "{{ doc('program_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: BUYER_ADDRESS + description: "{{ doc('purchaser') }}" + tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: SELLER_ADDRESS + description: "{{ doc('seller') }}" + tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: MINT + description: "{{ doc('mint') }}" + tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: NFT_NAME + description: "The name of the NFT" + tests: + - dbt_expectations.expect_column_to_exist + - name: PRICE + description: "{{ doc('sales_amount') }}" + tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: CURRENCY_ADDRESS + description: "Address of token used to pay for the NFT" + tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: CURRENCY_SYMBOL + description: "Symbol of token used to pay for the NFT" + tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: PRICE_USD + description: "Amount paid for NFT in USD" + tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: MARKETPLACE + description: "{{ doc('marketplace') }}" + tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: MARKETPLACE_VERSION + description: "The version of the NFT marketplace used for the transaction." + tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: TREE_AUTHORITY + description: "{{ doc('tree_authority') }}" + tests: + - dbt_expectations.expect_column_to_exist + - not_null: + where: (modified_timestamp >= current_date - 7) and IS_COMPRESSED + - name: MERKLE_TREE + description: "{{ doc('merkle_tree') }}" + tests: + - dbt_expectations.expect_column_to_exist + - not_null: + where: (modified_timestamp >= current_date - 7) and IS_COMPRESSED + - name: LEAF_INDEX + description: "{{ doc('leaf_index') }}" + tests: + - dbt_expectations.expect_column_to_exist + - not_null: + where: (modified_timestamp >= current_date - 7) and IS_COMPRESSED + - name: IS_COMPRESSED + description: "{{ doc('is_compressed') }}" + tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: NFT_COLLECTION_NAME + description: "The name of the collection provided by Solscan" + tests: + - dbt_expectations.expect_column_to_exist + - name: COLLECTION_ID + description: "The address of the NFT collection" + tests: + - dbt_expectations.expect_column_to_exist + - name: METADATA + description: "{{ doc('token_metadata') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: METADATA_URI + description: "{{ doc('token_metadata_uri') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: CREATORS + description: "Creators of the NFT and what percentage of royalties they receive" + tests: + - dbt_expectations.expect_column_to_exist + - name: AUTHORITY + description: "Authority address for the mint. When editions are minted, the authority remains the one from the master NFT" + tests: + - dbt_expectations.expect_column_to_exist + - name: IMAGE_URL + description: "{{ doc('image_url') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: EZ_NFT_SALES_ID + description: '{{ doc("pk") }}' + tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - unique: *recent_date_filter + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + tests: + - dbt_expectations.expect_column_to_exist + - not_null: *recent_date_filter + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/silver/nfts/silver__nft_sales_legacy_combined_view.sql b/models/silver/nfts/silver__nft_sales_legacy_combined_view.sql new file mode 100644 index 00000000..cd4c906f --- /dev/null +++ b/models/silver/nfts/silver__nft_sales_legacy_combined_view.sql @@ -0,0 +1,338 @@ +{{ config( + materialized = 'view' +) }} + +{% if execute %} + + {% set SOL_MINT = 'So11111111111111111111111111111111111111111' %} + +{% endif %} + +SELECT + 'magic eden v1' AS marketplace, + 'v1' AS marketplace_version, + block_timestamp, + block_id, + tx_id, + succeeded, + index, + inner_index, + program_id, + purchaser, + seller, + mint, + sales_amount, + '{{ SOL_MINT }}' as currency_address, + NULL as tree_authority, + NULL as merkle_tree, + NULL as leaf_index, + FALSE as is_compressed, + {{ dbt_utils.generate_surrogate_key( + ['tx_id', 'mint'] + ) }} AS nft_sales_legacy_combined_id, + '2000-01-01' as inserted_timestamp, + '2000-01-01' AS modified_timestamp +FROM + {{ ref( + 'silver__nft_sales_magic_eden_v1_view' + ) }} +UNION ALL +SELECT + 'solana monkey business marketplace', + 'v1' AS marketplace_version, + block_timestamp, + block_id, + tx_id, + succeeded, + index, + inner_index, + program_id, + purchaser, + seller, + mint, + sales_amount, + '{{ SOL_MINT }}' as currency_address, + NULL as tree_authority, + NULL as merkle_tree, + NULL as leaf_index, + FALSE as is_compressed, + {{ dbt_utils.generate_surrogate_key( + ['tx_id'] + ) }} AS nft_sales_legacy_combined_id, + '2000-01-01' as inserted_timestamp, + '2000-01-01' AS modified_timestamp +FROM + {{ ref('silver__nft_sales_smb_view') }} +UNION ALL +SELECT + 'solport', + 'v1' AS marketplace_version, + block_timestamp, + block_id, + tx_id, + succeeded, + index, + inner_index, + program_id, + purchaser, + seller, + mint, + sales_amount, + '{{ SOL_MINT }}' as currency_address, + NULL as tree_authority, + NULL as merkle_tree, + NULL as leaf_index, + FALSE as is_compressed, + {{ dbt_utils.generate_surrogate_key( + ['tx_id', 'mint'] + ) }} AS nft_sales_legacy_combined_id, + '2000-01-01' as inserted_timestamp, + '2000-01-01' AS modified_timestamp +FROM + {{ ref( + 'silver__nft_sales_solport_view' + ) }} +UNION ALL +SELECT + 'opensea', + 'v1' AS marketplace_version, + block_timestamp, + block_id, + tx_id, + succeeded, + index, + inner_index, + program_id, + purchaser, + seller, + mint, + sales_amount, + '{{ SOL_MINT }}' as currency_address, + NULL as tree_authority, + NULL as merkle_tree, + NULL as leaf_index, + FALSE as is_compressed, + {{ dbt_utils.generate_surrogate_key( + ['tx_id', 'mint'] + ) }} AS nft_sales_legacy_combined_id, + '2000-01-01' as inserted_timestamp, + '2000-01-01' AS modified_timestamp +FROM + {{ ref('silver__nft_sales_opensea_view') }} +UNION ALL +SELECT + 'yawww', + 'v1' AS marketplace_version, + block_timestamp, + block_id, + tx_id, + succeeded, + index, + inner_index, + program_id, + purchaser, + seller, + mint, + sales_amount, + '{{ SOL_MINT }}' as currency_address, + NULL as tree_authority, + NULL as merkle_tree, + NULL as leaf_index, + FALSE as is_compressed, + {{ dbt_utils.generate_surrogate_key( + ['tx_id'] + ) }} AS nft_sales_legacy_combined_id, + '2000-01-01' as inserted_timestamp, + '2000-01-01' AS modified_timestamp +FROM + {{ ref('silver__nft_sales_yawww_view') }} +UNION ALL +SELECT + 'coral cube', + 'v1' AS marketplace_version, + block_timestamp, + block_id, + tx_id, + succeeded, + index, + inner_index, + program_id, + purchaser, + seller, + mint, + sales_amount, + '{{ SOL_MINT }}' as currency_address, + NULL as tree_authority, + NULL as merkle_tree, + NULL as leaf_index, + FALSE as is_compressed, + {{ dbt_utils.generate_surrogate_key( + ['tx_id','mint'] + ) }} AS nft_sales_legacy_combined_id, + '2000-01-01' as inserted_timestamp, + '2000-01-01' AS modified_timestamp +FROM + {{ ref('silver__nft_sales_coral_cube_view') }} +UNION ALL +SELECT + marketplace, + case when marketplace = 'Magic Eden' then 'v2 AMM' else 'v1 AMM' end as marketplace_version, + block_timestamp, + block_id, + tx_id, + succeeded, + index, + inner_index, + program_id, + purchaser, + seller, + mint, + sales_amount, + '{{ SOL_MINT }}' as currency_address, + NULL as tree_authority, + NULL as merkle_tree, + NULL as leaf_index, + FALSE as is_compressed, + COALESCE ( + nft_sales_amm_sell_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_id','mint'] + ) }} + ) AS nft_sales_legacy_combined_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp +FROM + {{ ref('silver__nft_sales_amm_sell_view') }} +WHERE + block_timestamp::date < '2022-10-30' -- use new model after this date +UNION ALL +SELECT + 'solsniper', + 'v1' AS marketplace_version, + block_timestamp, + block_id, + tx_id, + succeeded, + index, + inner_index, + program_id, + purchaser, + seller, + mint, + sales_amount, + '{{ SOL_MINT }}' as currency_address, + NULL as tree_authority, + NULL as merkle_tree, + NULL as leaf_index, + FALSE as is_compressed, + nft_sales_solsniper_id AS nft_sales_legacy_combined_id, + inserted_timestamp, + modified_timestamp, +FROM + {{ ref('silver__nft_sales_solsniper_v1_events_view') }} +UNION ALL +SELECT + 'hyperspace', + 'v1' AS marketplace_version, + block_timestamp, + block_id, + tx_id, + succeeded, + index, + inner_index, + program_id, + purchaser, + seller, + mint, + sales_amount, + '{{ SOL_MINT }}' as currency_address, + NULL as tree_authority, + NULL as merkle_tree, + NULL as leaf_index, + FALSE as is_compressed, + COALESCE ( + nft_sales_hyperspace_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_id', 'mint'] + ) }} + ) AS nft_sales_legacy_combined_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__nft_sales_hyperspace_view') }} +UNION ALL +SELECT + 'hadeswap', + 'v1' AS marketplace_version, + block_timestamp, + block_id, + tx_id, + succeeded, + index, + inner_index, + program_id, + purchaser, + seller, + mint, + sales_amount, + '{{ SOL_MINT }}' as currency_address, + NULL as tree_authority, + NULL as merkle_tree, + NULL as leaf_index, + FALSE as is_compressed, + COALESCE ( + nft_sales_hadeswap_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_id','mint'] + ) }} + ) AS nft_sales_legacy_combined_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp +FROM + {{ ref('silver__nft_sales_hadeswap_view') }} +WHERE + block_timestamp::date <= '2023-02-08' -- use new model after this date +UNION ALL +SELECT + 'magic eden v2', + 'v2' AS marketplace_version, + block_timestamp, + block_id, + tx_id, + succeeded, + index, + inner_index, + program_id, + purchaser, + seller, + mint, + sales_amount, + '{{ SOL_MINT }}' as currency_address, + NULL as tree_authority, + NULL as merkle_tree, + NULL as leaf_index, + FALSE as is_compressed, + COALESCE ( + nft_sales_magic_eden_v2_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_id', 'mint'] + ) }} + ) AS nft_sales_legacy_combined_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__nft_sales_magic_eden_v2_view') }} +WHERE + block_timestamp::date < '2024-03-16' -- use new model after this date