diff --git a/.github/workflows/dbt_run_nft_compressed_mints_sales_realtime.yml b/.github/workflows/dbt_run_nft_compressed_mints_sales_realtime.yml index 6cb088a5..f73f782a 100644 --- a/.github/workflows/dbt_run_nft_compressed_mints_sales_realtime.yml +++ b/.github/workflows/dbt_run_nft_compressed_mints_sales_realtime.yml @@ -41,5 +41,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_sales_magic_eden.sql - dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_sales_solsniper.sql \ No newline at end of file + dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_sales_magic_eden.sql \ No newline at end of file diff --git a/models/bronze/bronze_api/bronze_api__parse_compressed_nft_sales_solsniper.sql b/models/bronze/bronze_api/bronze_api__parse_compressed_nft_sales_solsniper.sql index fff2818d..7d6f337b 100644 --- a/models/bronze/bronze_api/bronze_api__parse_compressed_nft_sales_solsniper.sql +++ b/models/bronze/bronze_api/bronze_api__parse_compressed_nft_sales_solsniper.sql @@ -1,8 +1,9 @@ -- depends_on: {{ ref('silver__nft_sales_solsniper_cnft_onchain') }} {{ config( materialized = 'incremental', - tags = ['bronze_api'], - cluster_by = ['end_inserted_timestamp::date'] + cluster_by = ['end_inserted_timestamp::date'], + full_refresh = false, + enabled = false, ) }} {% if execute %} @@ -14,7 +15,7 @@ tx_id, _inserted_timestamp FROM - {{ ref('silver__nft_sales_solsniper_cnft_onchain') }} + {{ ref('silver__nft_sales_solsniper_cnft_onchain_view') }} {% if is_incremental() %} WHERE _inserted_timestamp >= ( diff --git a/models/gold/defi/defi__fact_liquidity_pool_actions.sql b/models/gold/defi/defi__fact_liquidity_pool_actions.sql deleted file mode 100644 index 7cc68f3c..00000000 --- a/models/gold/defi/defi__fact_liquidity_pool_actions.sql +++ /dev/null @@ -1,178 +0,0 @@ -{{ config( - materialized = 'incremental', - meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STAKING' }}}, - unique_key = ['fact_liquidity_pool_actions_id'], - incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], - cluster_by = ['block_timestamp::DATE','action','program_id'], - merge_exclude_columns = ["inserted_timestamp"], - post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(tx_id, liquidity_provider, liquidity_pool_address, mint)'), - tags = ['scheduled_non_core'] -) }} - -SELECT - block_id, - block_timestamp, - tx_id, - succeeded, - program_id, - action, - liquidity_provider, - liquidity_pool_address, - amount, - mint, - _inserted_timestamp, - liquidity_pool_actions_raydium_id AS fact_liquidity_pool_actions_id, - inserted_timestamp, - modified_timestamp -FROM - {{ ref('silver__liquidity_pool_actions_raydium') }} -{% if is_incremental() %} -WHERE modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} -) -{% endif %} -UNION -SELECT - block_id, - block_timestamp, - tx_id, - succeeded, - program_id, - action, - liquidity_provider, - liquidity_pool_address, - amount, - mint, - _inserted_timestamp, - liquidity_pool_actions_orca_id as fact_liquidity_pool_actions_id, - inserted_timestamp, - modified_timestamp -FROM - {{ ref('silver__liquidity_pool_actions_orca') }} -{% if is_incremental() %} -WHERE modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} -) -{% endif %} -UNION -SELECT - block_id, - block_timestamp, - tx_id, - succeeded, - program_id, - action, - liquidity_provider, - liquidity_pool_address, - amount, - mint, - _inserted_timestamp, - COALESCE ( - liquidity_pool_actions_saber_id, - {{ dbt_utils.generate_surrogate_key( - ['BLOCK_ID','TX_ID','ACTION_INDEX'] - ) }} - ) AS fact_liquidity_pool_actions_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp -FROM - {{ ref('silver__liquidity_pool_actions_saber') }} -{% if is_incremental() %} -WHERE modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} -) -{% endif %} -UNION -SELECT - block_id, - block_timestamp, - tx_id, - succeeded, - program_id, - action, - liquidity_provider, - liquidity_pool_address, - amount, - mint, - _inserted_timestamp, - liquidity_pool_actions_meteora_id AS fact_liquidity_pool_actions_id, - inserted_timestamp, - modified_timestamp -FROM - {{ ref('silver__liquidity_pool_actions_meteora') }} -{% if is_incremental() %} -WHERE modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} -) -{% endif %} -UNION -SELECT - block_id, - block_timestamp, - tx_id, - succeeded, - program_id, - action, - liquidity_provider, - liquidity_pool_address, - amount, - mint, - _inserted_timestamp, - liquidity_pool_actions_meteora_dlmm_id AS fact_liquidity_pool_actions_id, - inserted_timestamp, - modified_timestamp -FROM - {{ ref('silver__liquidity_pool_actions_meteora_dlmm') }} -{% if is_incremental() %} -WHERE modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} -) -{% endif %} -UNION -SELECT - block_id, - block_timestamp, - tx_id, - succeeded, - program_id, - action, - liquidity_provider, - liquidity_pool_address, - amount, - mint, - _inserted_timestamp, - liquidity_pool_actions_meteora_multi_id AS fact_liquidity_pool_actions_id, - inserted_timestamp, - modified_timestamp -FROM - {{ ref('silver__liquidity_pool_actions_meteora_multi') }} -{% if is_incremental() %} -WHERE modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} -) -{% endif %} - diff --git a/models/gold/defi/defi__fact_liquidity_pool_actions.yml b/models/gold/defi/defi__fact_liquidity_pool_actions.yml deleted file mode 100644 index 2220df45..00000000 --- a/models/gold/defi/defi__fact_liquidity_pool_actions.yml +++ /dev/null @@ -1,84 +0,0 @@ -version: 2 -models: - - name: defi__fact_liquidity_pool_actions - description: Table containing liquidity pools actions on Orca, Raydium, Saber and Meteora. NOTE - actions for Orca pools created after 11/22/2024 are not tracked in this table, and we are working on updating our Orca data pipeline to collect these. - recent_date_filter: &recent_date_filter - config: - where: modified_timestamp >= current_date - 7 - tests: - - reference_tx_missing: - reference_tables: - - 'silver__liquidity_pool_actions_raydium' - - 'silver__liquidity_pool_actions_orca' - - 'silver__liquidity_pool_actions_saber' - - 'silver__liquidity_pool_actions_meteora' - - 'silver__liquidity_pool_actions_meteora_dlmm' - - 'silver__liquidity_pool_actions_meteora_multi' - id_column: 'tx_id' - columns: - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: BLOCK_ID - description: "{{ doc('block_id') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: TX_ID - description: "{{ doc('tx_id') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: SUCCEEDED - description: "{{ doc('tx_succeeded') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: PROGRAM_ID - description: "{{ doc('program_id') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: ACTION - description: Type of interaction performed with the liquidity pool - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: LIQUIDITY_PROVIDER - description: "{{ doc('liquidity_provider') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: LIQUIDITY_POOL_ADDRESS - description: "{{ doc('liquidity_pool_address') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: AMOUNT - description: "{{ doc('amount') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: MINT - description: "{{ doc('mint') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: FACT_LIQUIDITY_POOL_ACTIONS_ID - description: '{{ doc("pk") }}' - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - unique: *recent_date_filter - - name: INSERTED_TIMESTAMP - description: '{{ doc("inserted_timestamp") }}' - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter \ No newline at end of file diff --git a/models/gold/nft/nft__ez_nft_sales.sql b/models/gold/nft/nft__ez_nft_sales.sql index 9e0f1081..8c17a21f 100644 --- a/models/gold/nft/nft__ez_nft_sales.sql +++ b/models/gold/nft/nft__ez_nft_sales.sql @@ -25,17 +25,14 @@ {% endif %} {% set standard_platforms = [ - {'name': 'solanart', 'marketplace': 'solanart','marketplace_version': 'v1'}, {'name': 'hadeswap_decoded', 'marketplace': 'hadeswap','marketplace_version': 'v1'}, {'name': 'exchange_art', 'marketplace': 'exchange art', 'marketplace_version': 'v1'}, {'name': 'amm_sell_decoded', 'marketplace': 'magic eden v2', 'marketplace_version': 'v2'}, {'name': 'tensorswap', 'marketplace': 'tensorswap', 'marketplace_version': 'v1'}, - {'name': 'solsniper', 'marketplace': 'solsniper', 'marketplace_version': 'v1'}, {'name': 'tensor_bid', 'marketplace': 'tensorswap', 'marketplace_version': 'v1'}, ] %} {% set cnft_platforms = [ - {'name': 'solsniper_cnft', 'marketplace': 'solsniper','marketplace_version': 'v1'}, {'name': 'tensorswap_cnft', 'marketplace': 'tensorswap', 'marketplace_version': 'v1'}, {'name': 'magic_eden_cnft', 'marketplace': 'magic eden v3', 'marketplace_version': 'v3'}, ] %} diff --git a/models/gold/nft/nft__ez_nft_sales.yml b/models/gold/nft/nft__ez_nft_sales.yml index d7efd774..c678c842 100644 --- a/models/gold/nft/nft__ez_nft_sales.yml +++ b/models/gold/nft/nft__ez_nft_sales.yml @@ -9,15 +9,12 @@ models: - reference_tx_missing: reference_tables: - 'silver__nft_sales_magic_eden_v2_decoded' - - 'silver__nft_sales_solanart' - 'silver__nft_sales_hadeswap_decoded' - 'silver__nft_sales_exchange_art' - 'silver__nft_sales_amm_sell_decoded' - 'silver__nft_sales_tensorswap' - - 'silver__nft_sales_solsniper' - 'silver__nft_sales_tensorswap_cnft' - 'silver__nft_sales_magic_eden_cnft' - - 'silver__nft_sales_solsniper_cnft' id_column: 'tx_id' columns: - name: BLOCK_TIMESTAMP diff --git a/models/gold/nft/nft__fact_nft_sales.sql b/models/gold/nft/nft__fact_nft_sales.sql deleted file mode 100644 index 0d0eb1d5..00000000 --- a/models/gold/nft/nft__fact_nft_sales.sql +++ /dev/null @@ -1,639 +0,0 @@ -{{ config( - materialized = 'incremental', - meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'NFT' }}}, - unique_key = ['fact_nft_sales_id'], - incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], - cluster_by = ['block_timestamp::DATE','is_compressed','program_id'], - merge_exclude_columns = ["inserted_timestamp"], - post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(tx_id, purchaser, seller, mint, marketplace)'), - tags = ['scheduled_non_core'] -) }} - -{% if execute %} - - {% set SOL_MINT = 'So11111111111111111111111111111111111111111' %} - {% set magic_eden_switchover_block_timestamp = '2024-03-16' %} - - {% if is_incremental() %} - {% set query %} - SELECT MAX(modified_timestamp) AS max_modified_timestamp - FROM {{ this }} - {% endset %} - - {% set max_modified_timestamp = run_query(query).columns[0].values()[0] %} - {% endif %} -{% endif %} - - - --- Select from the deprecated _view models only during the initial FR -{% if not is_incremental() %} -SELECT - 'magic eden v1' AS marketplace, - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - '{{ SOL_MINT }}' as currency_address, - NULL as tree_authority, - NULL as merkle_tree, - NULL as leaf_index, - FALSE as is_compressed, - {{ dbt_utils.generate_surrogate_key( - ['tx_id', 'mint'] - ) }} AS fact_nft_sales_id, - '2000-01-01' as inserted_timestamp, - '2000-01-01' AS modified_timestamp -FROM - {{ ref( - 'silver__nft_sales_magic_eden_v1_view' - ) }} -UNION ALL -SELECT - 'solana monkey business marketplace', - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - '{{ SOL_MINT }}' as currency_address, - NULL as tree_authority, - NULL as merkle_tree, - NULL as leaf_index, - FALSE as is_compressed, - {{ dbt_utils.generate_surrogate_key( - ['tx_id'] - ) }} AS fact_nft_sales_id, - '2000-01-01' as inserted_timestamp, - '2000-01-01' AS modified_timestamp -FROM - {{ ref('silver__nft_sales_smb_view') }} -UNION ALL -SELECT - 'solport', - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - '{{ SOL_MINT }}' as currency_address, - NULL as tree_authority, - NULL as merkle_tree, - NULL as leaf_index, - FALSE as is_compressed, - {{ dbt_utils.generate_surrogate_key( - ['tx_id', 'mint'] - ) }} AS fact_nft_sales_id, - '2000-01-01' as inserted_timestamp, - '2000-01-01' AS modified_timestamp -FROM - {{ ref( - 'silver__nft_sales_solport_view' - ) }} -UNION ALL -SELECT - 'opensea', - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - '{{ SOL_MINT }}' as currency_address, - NULL as tree_authority, - NULL as merkle_tree, - NULL as leaf_index, - FALSE as is_compressed, - {{ dbt_utils.generate_surrogate_key( - ['tx_id', 'mint'] - ) }} AS fact_nft_sales_id, - '2000-01-01' as inserted_timestamp, - '2000-01-01' AS modified_timestamp -FROM - {{ ref('silver__nft_sales_opensea_view') }} -UNION ALL -SELECT - 'yawww', - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - '{{ SOL_MINT }}' as currency_address, - NULL as tree_authority, - NULL as merkle_tree, - NULL as leaf_index, - FALSE as is_compressed, - {{ dbt_utils.generate_surrogate_key( - ['tx_id'] - ) }} AS fact_nft_sales_id, - '2000-01-01' as inserted_timestamp, - '2000-01-01' AS modified_timestamp -FROM - {{ ref('silver__nft_sales_yawww_view') }} -UNION ALL -SELECT - 'coral cube', - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - '{{ SOL_MINT }}' as currency_address, - NULL as tree_authority, - NULL as merkle_tree, - NULL as leaf_index, - FALSE as is_compressed, - {{ dbt_utils.generate_surrogate_key( - ['tx_id','mint'] - ) }} AS fact_nft_sales_id, - '2000-01-01' as inserted_timestamp, - '2000-01-01' AS modified_timestamp -FROM - {{ ref('silver__nft_sales_coral_cube_view') }} -UNION ALL -SELECT - marketplace, - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - '{{ SOL_MINT }}' as currency_address, - NULL as tree_authority, - NULL as merkle_tree, - NULL as leaf_index, - FALSE as is_compressed, - COALESCE ( - nft_sales_amm_sell_id, - {{ dbt_utils.generate_surrogate_key( - ['tx_id','mint'] - ) }} - ) AS fact_nft_sales_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp -FROM - {{ ref('silver__nft_sales_amm_sell_view') }} -WHERE - block_timestamp::date < '2022-10-30' -UNION ALL -SELECT - 'solsniper', - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - '{{ SOL_MINT }}' as currency_address, - NULL as tree_authority, - NULL as merkle_tree, - NULL as leaf_index, - FALSE as is_compressed, - nft_sales_solsniper_id AS fact_nft_sales_id, - inserted_timestamp, - modified_timestamp, -FROM - {{ ref('silver__nft_sales_solsniper_v1_events_view') }} -UNION ALL -SELECT - 'hyperspace', - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - '{{ SOL_MINT }}' as currency_address, - NULL as tree_authority, - NULL as merkle_tree, - NULL as leaf_index, - FALSE as is_compressed, - nft_sales_hyperspace_id AS fact_nft_sales_id, - inserted_timestamp, - modified_timestamp -FROM - {{ ref('silver__nft_sales_hyperspace_view') }} -UNION ALL -SELECT - 'hadeswap', - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - '{{ SOL_MINT }}' as currency_address, - NULL as tree_authority, - NULL as merkle_tree, - NULL as leaf_index, - FALSE as is_compressed, - COALESCE ( - nft_sales_hadeswap_id, - {{ dbt_utils.generate_surrogate_key( - ['tx_id','mint'] - ) }} - ) AS fact_nft_sales_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp -FROM - {{ ref('silver__nft_sales_hadeswap_view') }} -WHERE - block_timestamp::date <= '2023-02-08' -UNION ALL -SELECT - 'magic eden v2', - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - currency_address, - NULL as tree_authority, - NULL as merkle_tree, - NULL as leaf_index, - FALSE as is_compressed, - nft_sales_magic_eden_v2_id as fact_nft_sales_id, - inserted_timestamp, - modified_timestamp -FROM - {{ ref('silver__nft_sales_magic_eden_v2_view') }} -WHERE - block_timestamp::date < '{{ magic_eden_switchover_block_timestamp }}' -UNION ALL -{% endif %} --- Only select from active models during incremental -SELECT - 'solanart' as marketplace, - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - '{{ SOL_MINT }}' as currency_address, - NULL as tree_authority, - NULL as merkle_tree, - NULL as leaf_index, - FALSE as is_compressed, - COALESCE ( - nft_sales_solanart_id, - {{ dbt_utils.generate_surrogate_key( - ['tx_id','mint'] - ) }} - ) AS fact_nft_sales_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp -FROM - {{ ref('silver__nft_sales_solanart') }} -{% if is_incremental() %} -WHERE - modified_timestamp >= '{{ max_modified_timestamp }}' -{% endif %} -UNION ALL -SELECT - 'hadeswap', - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - '{{ SOL_MINT }}' as currency_address, - NULL as tree_authority, - NULL as merkle_tree, - NULL as leaf_index, - FALSE as is_compressed, - nft_sales_hadeswap_decoded_id AS fact_nft_sales_id, - inserted_timestamp, - modified_timestamp -FROM - {{ ref('silver__nft_sales_hadeswap_decoded') }} -{% if is_incremental() %} -WHERE - modified_timestamp >= '{{ max_modified_timestamp }}' -{% endif %} -UNION ALL -SELECT - 'exchange art', - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - '{{ SOL_MINT }}' as currency_address, - NULL as tree_authority, - NULL as merkle_tree, - NULL as leaf_index, - FALSE as is_compressed, - COALESCE ( - nft_sales_exchange_art_id, - {{ dbt_utils.generate_surrogate_key( - ['tx_id','mint'] - ) }} - ) AS fact_nft_sales_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp -FROM - {{ ref('silver__nft_sales_exchange_art') }} -{% if is_incremental() %} -WHERE - modified_timestamp >= '{{ max_modified_timestamp }}' -{% endif %} -UNION ALL -SELECT - marketplace, - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - '{{ SOL_MINT }}' as currency_address, - NULL as tree_authority, - NULL as merkle_tree, - NULL as leaf_index, - FALSE as is_compressed, - nft_sales_amm_sell_decoded_id as fact_nft_sales_id, - inserted_timestamp, - modified_timestamp -FROM - {{ ref('silver__nft_sales_amm_sell_decoded') }} -{% if is_incremental() %} -WHERE - modified_timestamp >= '{{ max_modified_timestamp }}' -{% endif %} -UNION ALL -SELECT - 'tensorswap', - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - '{{ SOL_MINT }}' as currency_address, - NULL as tree_authority, - NULL as merkle_tree, - NULL as leaf_index, - FALSE as is_compressed, - COALESCE ( - nft_sales_tensorswap_id, - {{ dbt_utils.generate_surrogate_key( - ['tx_id','index','inner_index','mint'] - ) }} - ) AS fact_nft_sales_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp -FROM - {{ ref('silver__nft_sales_tensorswap') }} -{% if is_incremental() %} -WHERE - modified_timestamp >= '{{ max_modified_timestamp }}' -{% endif %} -UNION ALL -SELECT - 'solsniper', - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - '{{ SOL_MINT }}' as currency_address, - NULL as tree_authority, - NULL as merkle_tree, - NULL as leaf_index, - FALSE as is_compressed, - nft_sales_solsniper_id AS fact_nft_sales_id, - inserted_timestamp, - modified_timestamp -FROM - {{ ref('silver__nft_sales_solsniper') }} -{% if is_incremental() %} -WHERE - modified_timestamp >= '{{ max_modified_timestamp }}' -{% endif %} -UNION ALL -SELECT - 'tensorswap', - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - '{{ SOL_MINT }}' as currency_address, - tree_authority, - merkle_tree, - leaf_index, - TRUE as is_compressed, - nft_sales_tensorswap_cnft_id AS fact_nft_sales_id, - inserted_timestamp, - modified_timestamp, -FROM - {{ ref('silver__nft_sales_tensorswap_cnft') }} -{% if is_incremental() %} -WHERE - modified_timestamp >= '{{ max_modified_timestamp }}' -{% endif %} -UNION ALL -SELECT - 'magic eden v3' AS marketplace, - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - '{{ SOL_MINT }}' as currency_address, - tree_authority, - merkle_tree, - leaf_index, - TRUE as is_compressed, - nft_sales_magic_eden_cnft_id AS fact_nft_sales_id, - inserted_timestamp, - modified_timestamp, -FROM - {{ ref('silver__nft_sales_magic_eden_cnft') }} -{% if is_incremental() %} -WHERE - modified_timestamp >= '{{ max_modified_timestamp }}' -{% endif %} -UNION ALL -SELECT - 'solsniper' AS marketplace, - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - '{{ SOL_MINT }}' as currency_address, - tree_authority, - merkle_tree, - leaf_index, - TRUE as is_compressed, - nft_sales_solsniper_cnft_id AS fact_nft_sales_id, - inserted_timestamp, - modified_timestamp, -FROM - {{ ref('silver__nft_sales_solsniper_cnft') }} -{% if is_incremental() %} -WHERE - modified_timestamp >= '{{ max_modified_timestamp }}' -{% endif %} -UNION ALL -SELECT - 'tensor' AS marketplace, - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - '{{ SOL_MINT }}' as currency_address, - NULL as tree_authority, - NULL as merkle_tree, - NULL as leaf_index, - FALSE as is_compressed, - nft_sales_tensor_bid_id AS fact_nft_sales_id, - inserted_timestamp, - modified_timestamp, -FROM - {{ ref('silver__nft_sales_tensor_bid') }} -{% if is_incremental() %} -WHERE - modified_timestamp >= '{{ max_modified_timestamp }}' -{% endif %} -UNION ALL -SELECT - 'magic eden v2' AS marketplace, - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount, - currency_address, - NULL as tree_authority, - NULL as merkle_tree, - NULL as leaf_index, - FALSE as is_compressed, - nft_sales_magic_eden_v2_decoded_id AS fact_nft_sales_id, - inserted_timestamp, - modified_timestamp, -FROM - {{ ref('silver__nft_sales_magic_eden_v2_decoded') }} -{% if is_incremental() %} -WHERE - modified_timestamp >= '{{ max_modified_timestamp }}' -{% else %} -WHERE - block_timestamp::date >= '{{magic_eden_switchover_block_timestamp}}' -{% endif %} diff --git a/models/gold/nft/nft__fact_nft_sales.yml b/models/gold/nft/nft__fact_nft_sales.yml deleted file mode 100644 index bf07e014..00000000 --- a/models/gold/nft/nft__fact_nft_sales.yml +++ /dev/null @@ -1,114 +0,0 @@ -version: 2 -models: - - name: nft__fact_nft_sales - description: NFT sales on Solana that occur on Magic Eden, Yawww, Opensea, the SMB marketplace, Solanart, Solport, Coral Cube, Hyperspace, Hadeswap, Exchange Art, Tensorswap and Solsniper. - recent_date_filter: &recent_date_filter - config: - where: modified_timestamp >= current_date - 7 - tests: - - reference_tx_missing: - reference_tables: - - 'silver__nft_sales_magic_eden_v2_decoded' - - 'silver__nft_sales_solanart' - - 'silver__nft_sales_hadeswap_decoded' - - 'silver__nft_sales_hyperspace' - - 'silver__nft_sales_exchange_art' - - 'silver__nft_sales_amm_sell_decoded' - - 'silver__nft_sales_tensorswap' - - 'silver__nft_sales_solsniper' - - 'silver__nft_sales_tensorswap_cnft' - - 'silver__nft_sales_magic_eden_cnft' - - 'silver__nft_sales_solsniper_cnft' - id_column: 'tx_id' - columns: - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: BLOCK_ID - description: "{{ doc('block_id') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: TX_ID - description: "{{ doc('tx_id') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: SUCCEEDED - description: "{{ doc('tx_succeeded') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: PROGRAM_ID - description: "{{ doc('program_id') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: PURCHASER - description: "{{ doc('purchaser') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: SELLER - description: "{{ doc('seller') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: MINT - description: "{{ doc('mint') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: SALES_AMOUNT - description: "{{ doc('sales_amount') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: CURRENCY_ADDRESS - description: "Address of token used to pay for the NFT" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: MARKETPLACE - description: "{{ doc('marketplace') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: TREE_AUTHORITY - description: "{{ doc('tree_authority') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: - where: (modified_timestamp >= current_date - 7) and IS_COMPRESSED - - name: MERKLE_TREE - description: "{{ doc('merkle_tree') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: - where: (modified_timestamp >= current_date - 7) and IS_COMPRESSED - - name: LEAF_INDEX - description: "{{ doc('leaf_index') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: - where: (modified_timestamp >= current_date - 7) and IS_COMPRESSED - - name: IS_COMPRESSED - description: "{{ doc('is_compressed') }}" - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: FACT_NFT_SALES_ID - description: '{{ doc("pk") }}' - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - unique: *recent_date_filter - - name: INSERTED_TIMESTAMP - description: '{{ doc("inserted_timestamp") }}' - tests: - - dbt_expectations.expect_column_to_exist - - not_null: *recent_date_filter - - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/silver/nfts/silver__nft_sales_legacy_combined_view.sql b/models/silver/nfts/silver__nft_sales_legacy_combined_view.sql index cd4c906f..1941df80 100644 --- a/models/silver/nfts/silver__nft_sales_legacy_combined_view.sql +++ b/models/silver/nfts/silver__nft_sales_legacy_combined_view.sql @@ -306,8 +306,8 @@ WHERE block_timestamp::date <= '2023-02-08' -- use new model after this date UNION ALL SELECT - 'magic eden v2', - 'v2' AS marketplace_version, + 'solanart', + 'v1' AS marketplace_version, block_timestamp, block_id, tx_id, @@ -324,15 +324,59 @@ SELECT NULL as merkle_tree, NULL as leaf_index, FALSE as is_compressed, - COALESCE ( - nft_sales_magic_eden_v2_id, - {{ dbt_utils.generate_surrogate_key( - ['tx_id', 'mint'] - ) }} - ) AS nft_sales_legacy_combined_id, + nft_sales_solanart_id AS nft_sales_legacy_combined_id, inserted_timestamp, modified_timestamp FROM - {{ ref('silver__nft_sales_magic_eden_v2_view') }} -WHERE - block_timestamp::date < '2024-03-16' -- use new model after this date + {{ ref('silver__nft_sales_solanart_view') }} +UNION ALL +SELECT + 'solsniper', + 'v1' AS marketplace_version, + block_timestamp, + block_id, + tx_id, + succeeded, + index, + inner_index, + program_id, + purchaser, + seller, + mint, + sales_amount, + '{{ SOL_MINT }}' as currency_address, + NULL as tree_authority, + NULL as merkle_tree, + NULL as leaf_index, + FALSE as is_compressed, + nft_sales_solsniper_id AS nft_sales_legacy_combined_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__nft_sales_solsniper_view') }} +UNION ALL +SELECT + 'solsniper', + 'v1' AS marketplace_version, + block_timestamp, + block_id, + tx_id, + succeeded, + index, + inner_index, + program_id, + purchaser, + seller, + mint, + sales_amount, + '{{ SOL_MINT }}' AS currency_address, + tree_authority, + merkle_tree, + leaf_index, + TRUE as is_compressed, + nft_sales_solsniper_cnft_id AS nft_sales_legacy_combined_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__nft_sales_solsniper_cnft_view') }} + diff --git a/models/silver/nfts/silver__nft_sales_solanart.sql b/models/silver/nfts/silver__nft_sales_solanart.sql index c4c79b72..dba0ebdb 100644 --- a/models/silver/nfts/silver__nft_sales_solanart.sql +++ b/models/silver/nfts/silver__nft_sales_solanart.sql @@ -3,7 +3,8 @@ unique_key = "CONCAT_WS('-', tx_id, mint)", incremental_strategy = 'delete+insert', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], - tags = ['scheduled_non_core'] + full_refresh = false, + enabled = false, ) }} WITH sales_inner_instructions AS ( diff --git a/models/silver/nfts/silver__nft_sales_solanart_view.sql b/models/silver/nfts/silver__nft_sales_solanart_view.sql new file mode 100644 index 00000000..8cc149b7 --- /dev/null +++ b/models/silver/nfts/silver__nft_sales_solanart_view.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + block_timestamp, + block_id, + tx_id, + succeeded, + program_id, + index, + inner_index, + mint, + purchaser, + seller, + sales_amount, + _inserted_timestamp, + nft_sales_solanart_id, + inserted_timestamp, + modified_timestamp, + _invocation_id +FROM + {{ source( + 'solana_silver', + 'nft_sales_solanart' + ) }} diff --git a/models/silver/nfts/silver__nft_sales_solsniper.sql b/models/silver/nfts/silver__nft_sales_solsniper.sql index f5604416..8d36d784 100644 --- a/models/silver/nfts/silver__nft_sales_solsniper.sql +++ b/models/silver/nfts/silver__nft_sales_solsniper.sql @@ -3,8 +3,8 @@ incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], unique_key = ['nft_sales_solsniper_id'], cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], - merge_exclude_columns = ["inserted_timestamp"], - tags = ['scheduled_non_core'] + full_refresh = false, + enabled = false, ) }} -- depends_on: {{ ref('silver__decoded_instructions_combined') }} /* run incremental timestamp value first then use it as a static value */ diff --git a/models/silver/nfts/silver__nft_sales_solsniper_cnft.sql b/models/silver/nfts/silver__nft_sales_solsniper_cnft.sql index 7f7025fd..951fab81 100644 --- a/models/silver/nfts/silver__nft_sales_solsniper_cnft.sql +++ b/models/silver/nfts/silver__nft_sales_solsniper_cnft.sql @@ -4,7 +4,8 @@ incremental_strategy = 'delete+insert', merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], - tags = ['scheduled_non_core'] + full_refresh = false, + enabled = false, ) }} WITH mint_addresses AS ( diff --git a/models/silver/nfts/silver__nft_sales_solsniper_cnft_onchain.sql b/models/silver/nfts/silver__nft_sales_solsniper_cnft_onchain.sql index acf5fae0..20788541 100644 --- a/models/silver/nfts/silver__nft_sales_solsniper_cnft_onchain.sql +++ b/models/silver/nfts/silver__nft_sales_solsniper_cnft_onchain.sql @@ -4,7 +4,8 @@ unique_key = ['nft_sales_solsniper_cnft_onchain_id'], cluster_by = ['block_timestamp::DATE'], merge_exclude_columns = ["inserted_timestamp"], - tags = ['scheduled_non_core'] + full_refresh = false, + enabled = false, ) }} -- depends_on: {{ ref('silver__decoded_instructions_combined') }} diff --git a/models/silver/nfts/silver__nft_sales_solsniper_cnft_view.sql b/models/silver/nfts/silver__nft_sales_solsniper_cnft_view.sql new file mode 100644 index 00000000..bd6267bc --- /dev/null +++ b/models/silver/nfts/silver__nft_sales_solsniper_cnft_view.sql @@ -0,0 +1,29 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + block_id, + block_timestamp, + program_id, + tx_id, + succeeded, + index, + inner_index, + purchaser, + seller, + tree_authority, + merkle_tree, + leaf_index, + mint, + mint_inserted_timestamp, + sales_amount, + _inserted_timestamp, + nft_sales_solsniper_cnft_id, + inserted_timestamp, + modified_timestamp +FROM + {{ source( + 'solana_silver', + 'nft_sales_solsniper_cnft' + ) }} \ No newline at end of file diff --git a/models/silver/nfts/silver__nft_sales_solsniper_view.sql b/models/silver/nfts/silver__nft_sales_solsniper_view.sql new file mode 100644 index 00000000..d1b1e59b --- /dev/null +++ b/models/silver/nfts/silver__nft_sales_solsniper_view.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + block_timestamp, + block_id, + tx_id, + succeeded, + index, + inner_index, + program_id, + purchaser, + seller, + mint, + sales_amount, + _inserted_timestamp, + nft_sales_solsniper_id, + inserted_timestamp, + modified_timestamp, + invocation_id +FROM + {{ source( + 'solana_silver', + 'nft_sales_solsniper' + ) }} diff --git a/models/silver/parser/silver__decoded_instructions_combined.yml b/models/silver/parser/silver__decoded_instructions_combined.yml index 82397247..047c5291 100644 --- a/models/silver/parser/silver__decoded_instructions_combined.yml +++ b/models/silver/parser/silver__decoded_instructions_combined.yml @@ -17,15 +17,6 @@ models: description: "{{ doc('tx_id') }}" tests: - not_null - - dbt_utils.relationships_where: - name: dbt_utils_relationships_where_silver__decoded_instructions_combined_nft_sales_solsniper_tx_id - to: ref('silver__nft_sales_solsniper') - field: tx_id - from_condition: > - program_id = 'SNPRohhBurQwrpwAptw1QYtpFdfEKitr4WSJ125cN1g' - and event_type = 'executeSolNftOrder' - and _inserted_timestamp between current_date - 7 and current_timestamp() - INTERVAL '4 HOUR' - to_condition: "_inserted_timestamp >= current_date - 7" - dbt_utils.relationships_where: name: dbt_utils_relationships_where_silver__decoded_instructions_combined_nft_sales_tensorswap_cnft_tx_id to: ref('silver__nft_sales_tensorswap_cnft') @@ -35,15 +26,6 @@ models: and event_type = 'buy' and _inserted_timestamp between current_date - 7 and current_timestamp() - INTERVAL '4 HOUR' to_condition: "_inserted_timestamp >= current_date - 7" - - dbt_utils.relationships_where: - name: dbt_utils_relationships_where_silver__decoded_instructions_combined_nft_sales_solsniper_cnft_onchain_tx_id - to: ref('silver__nft_sales_solsniper_cnft_onchain') - field: tx_id - from_condition: > - program_id = 'SNPRohhBurQwrpwAptw1QYtpFdfEKitr4WSJ125cN1g' - and event_type = 'executeSolCnftOrder' - and _inserted_timestamp between current_date - 7 and current_timestamp() - INTERVAL '4 HOUR' - to_condition: "_inserted_timestamp >= current_date - 7" - dbt_utils.relationships_where: name: dbt_utils_relationships_where_silver__decoded_instructions_combined_nft_sales_magic_eden_cnft_onchain_tx_id to: ref('silver__nft_sales_magic_eden_cnft_onchain') diff --git a/models/solana_share/solana_share__fact_nft_sales.sql b/models/solana_share/solana_share__fact_nft_sales.sql deleted file mode 100644 index 5fa6b122..00000000 --- a/models/solana_share/solana_share__fact_nft_sales.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = "tx_id", - cluster_by = ['block_timestamp::date'], - tags = ['share'] -) }} - - SELECT - marketplace, - block_timestamp, - block_id, - tx_id, - succeeded, - program_id, - purchaser, - seller, - mint, - sales_amount -FROM {{ref('nft__fact_nft_sales')}} -where block_timestamp::date between '2021-12-01' and '2021-12-31' \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index 862228da..9842ab95 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -115,6 +115,9 @@ sources: - name: nft_sales_magic_eden_v2 - name: snapshot_block_production - name: bridge_mayan_transfers + - name: nft_sales_solsniper_cnft + - name: nft_sales_solsniper + - name: nft_sales_solanart - name: solana_streamline database: solana schema: streamline