diff --git a/models/gold/defi/defi__ez_dex_swaps.sql b/models/gold/defi/defi__ez_dex_swaps.sql index 9a6b4db2..404921de 100644 --- a/models/gold/defi/defi__ez_dex_swaps.sql +++ b/models/gold/defi/defi__ez_dex_swaps.sql @@ -1,9 +1,27 @@ {{ config( - materialized = 'view', - meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'SWAPS' }} }, + materialized = 'incremental', + unique_key = ['ez_swaps_id'], + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], + cluster_by = ['block_timestamp::DATE','ROUND(block_id, -3)', 'program_id'], + merge_exclude_columns = ["inserted_timestamp"], + post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(tx_id, swapper, swap_from_mint, swap_to_mint, program_id, ez_swaps_id)'), + + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'SWAPS' }}}, tags = ['scheduled_non_core','scheduled_non_core_hourly','exclude_change_tracking'] ) }} +{% if execute %} + {% if is_incremental() %} + {% set max_modified_query %} + SELECT + MAX(modified_timestamp) AS modified_timestamp + FROM + {{ this }} + {% endset %} + {% set max_modified_timestamp = run_query(max_modified_query)[0][0] %} + {% endif %} +{% endif %} + WITH swaps AS ( SELECT @@ -25,6 +43,10 @@ WITH swaps AS ( {{ ref('defi__fact_swaps') }} WHERE succeeded +{% if is_incremental() %} +AND + modified_timestamp >= '{{ max_modified_timestamp }}' +{% endif %} ), prices AS ( SELECT @@ -34,6 +56,14 @@ prices AS ( price FROM {{ ref('price__ez_prices_hourly') }} + WHERE + hour >= ( + SELECT + MIN(DATE_TRUNC('hour', block_timestamp)) + FROM + swaps + ) + ) SELECT d.swap_program, @@ -57,8 +87,8 @@ SELECT 2 ) AS swap_to_amount_usd, d._log_id, - d.inserted_timestamp, - d.modified_timestamp, + sysdate() AS inserted_timestamp, + sysdate() AS modified_timestamp, d.ez_swaps_id, FROM swaps d diff --git a/models/gold/defi/defi__fact_token_burn_actions.sql b/models/gold/defi/defi__fact_token_burn_actions.sql index 88d8cb61..20e35f08 100644 --- a/models/gold/defi/defi__fact_token_burn_actions.sql +++ b/models/gold/defi/defi__fact_token_burn_actions.sql @@ -1,9 +1,27 @@ {{ config( - materialized = 'view', + materialized = 'incremental', + unique_key = ['fact_token_burn_actions_id'], + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], + cluster_by = ['block_timestamp::DATE','ROUND(block_id, -3)', 'event_type'], + merge_exclude_columns = ["inserted_timestamp"], + post_hook = enable_search_optimization('{{this.schema}}', '{{this.identifier}}', 'ON EQUALITY(tx_id, mint, burn_authority, token_account)'), + full_refresh = false, meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'Token' }}}, tags = ['scheduled_non_core'] ) }} +{% if execute %} + {% if is_incremental() %} + {% set max_modified_query %} + SELECT + MAX(modified_timestamp) AS modified_timestamp + FROM + {{ this }} + {% endset %} + {% set max_modified_timestamp = run_query(max_modified_query)[0][0] %} + {% endif %} +{% endif %} + SELECT block_id, block_timestamp, @@ -25,14 +43,12 @@ SELECT ['tx_id', 'index', 'inner_index', 'mint'] ) }} ) AS fact_token_burn_actions_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp + sysdate() AS inserted_timestamp, + sysdate() AS modified_timestamp FROM {{ ref('silver__token_burn_actions') }} +{% if is_incremental() %} +WHERE + modified_timestamp >= '{{ max_modified_timestamp }}' +{% endif %} diff --git a/models/gold/defi/defi__fact_token_mint_actions.sql b/models/gold/defi/defi__fact_token_mint_actions.sql index a6a4afb7..9a6dd691 100644 --- a/models/gold/defi/defi__fact_token_mint_actions.sql +++ b/models/gold/defi/defi__fact_token_mint_actions.sql @@ -1,9 +1,27 @@ {{ config( - materialized = 'view', + materialized = 'incremental', + unique_key = ['fact_token_mint_actions_id'], + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], + cluster_by = ['block_timestamp::DATE','ROUND(block_id, -3)', 'event_type'], + merge_exclude_columns = ["inserted_timestamp"], + post_hook = enable_search_optimization('{{this.schema}}', '{{this.identifier}}', 'ON EQUALITY(tx_id, mint, mint_authority, token_account)'), + full_refresh = false, meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'Token' }}}, tags = ['scheduled_non_core'] ) }} +{% if execute %} + {% if is_incremental() %} + {% set max_modified_query %} + SELECT + MAX(modified_timestamp) AS modified_timestamp + FROM + {{ this }} + {% endset %} + {% set max_modified_timestamp = run_query(max_modified_query)[0][0] %} + {% endif %} +{% endif %} + SELECT block_id, block_timestamp, @@ -25,16 +43,14 @@ SELECT ['tx_id', 'index', 'inner_index', 'mint'] ) }} ) AS fact_token_mint_actions_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp + sysdate() AS inserted_timestamp, + sysdate() AS modified_timestamp FROM {{ ref('silver__token_mint_actions') }} +{% if is_incremental() %} +WHERE + modified_timestamp >= '{{ max_modified_timestamp }}' +{% endif %} diff --git a/models/gold/nft/nft__dim_nft_metadata.sql b/models/gold/nft/nft__dim_nft_metadata.sql index af9d990a..04f66b16 100644 --- a/models/gold/nft/nft__dim_nft_metadata.sql +++ b/models/gold/nft/nft__dim_nft_metadata.sql @@ -1,9 +1,26 @@ {{ config( - materialized = 'view', - meta = { 'database_tags':{ 'table':{ 'PURPOSE': 'NFT' }} }, - tags = ['scheduled_non_core', 'exclude_change_tracking'] + materialized = 'incremental', + unique_key = ['dim_nft_metadata_id'], + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + post_hook = enable_search_optimization('{{this.schema}}', '{{this.identifier}}', 'ON EQUALITY(mint, nft_name)'), + full_refresh = false, + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'NFT' }}}, + tags = ['scheduled_non_core', 'exclude_change_tracking'] ) }} +{% if execute %} + {% if is_incremental() %} + {% set max_modified_query %} + SELECT + MAX(modified_timestamp) AS modified_timestamp + FROM + {{ this }} + {% endset %} + {% set max_modified_timestamp = run_query(max_modified_query)[0][0] %} + {% endif %} +{% endif %} + SELECT A.mint, b.nft_collection_name, @@ -15,13 +32,17 @@ SELECT A.metadata_uri, A.nft_name, A.helius_nft_metadata_id AS dim_nft_metadata_id, - A.inserted_timestamp, - A.modified_timestamp + sysdate() AS inserted_timestamp, + sysdate() AS modified_timestamp FROM {{ ref('silver__helius_nft_metadata') }} A LEFT JOIN {{ ref('silver__nft_collection_view') }} b ON A.nft_collection_id = b.nft_collection_id +{% if is_incremental() %} +WHERE + a.modified_timestamp >= '{{ max_modified_timestamp }}' +{% endif %} UNION ALL SELECT mint, @@ -34,7 +55,11 @@ SELECT metadata_uri, nft_name, helius_cnft_metadata_id AS dim_nft_metadata_id, - inserted_timestamp, - modified_timestamp + sysdate() AS inserted_timestamp, + sysdate() AS modified_timestamp FROM - {{ ref('silver__helius_cnft_metadata') }} \ No newline at end of file + {{ ref('silver__helius_cnft_metadata') }} +{% if is_incremental() %} +WHERE + modified_timestamp >= '{{ max_modified_timestamp }}' +{% endif %} \ No newline at end of file diff --git a/models/gold/nft/nft__fact_nft_burn_actions.sql b/models/gold/nft/nft__fact_nft_burn_actions.sql index 32407f54..84d45e5c 100644 --- a/models/gold/nft/nft__fact_nft_burn_actions.sql +++ b/models/gold/nft/nft__fact_nft_burn_actions.sql @@ -1,9 +1,27 @@ {{ config( - materialized = 'view', + materialized = 'incremental', + unique_key = ['fact_nft_burn_actions_id'], + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], + cluster_by = ['block_timestamp::DATE','ROUND(block_id, -3)', 'event_type'], + merge_exclude_columns = ["inserted_timestamp"], + post_hook = enable_search_optimization('{{this.schema}}', '{{this.identifier}}', 'ON EQUALITY(tx_id, mint, burn_authority)'), + full_refresh = false, meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'NFT' }}}, tags = ['scheduled_non_core'] ) }} +{% if execute %} + {% if is_incremental() %} + {% set max_modified_query %} + SELECT + MAX(modified_timestamp) AS modified_timestamp + FROM + {{ this }} + {% endset %} + {% set max_modified_timestamp = run_query(max_modified_query)[0][0] %} + {% endif %} +{% endif %} + SELECT block_id, block_timestamp, @@ -24,13 +42,11 @@ SELECT ['tx_id', 'index', 'inner_index', 'mint'] ) }} ) AS fact_nft_burn_actions_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp + sysdate() AS inserted_timestamp, + sysdate() AS modified_timestamp FROM {{ ref('silver__nft_burn_actions') }} +{% if is_incremental() %} +WHERE + modified_timestamp >= '{{ max_modified_timestamp }}' +{% endif %} diff --git a/models/gold/nft/nft__fact_nft_mint_actions.sql b/models/gold/nft/nft__fact_nft_mint_actions.sql index 06b27196..0cd0d4e0 100644 --- a/models/gold/nft/nft__fact_nft_mint_actions.sql +++ b/models/gold/nft/nft__fact_nft_mint_actions.sql @@ -1,9 +1,27 @@ {{ config( - materialized = 'view', + materialized = 'incremental', + unique_key = ['fact_nft_mint_actions_id'], + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], + cluster_by = ['block_timestamp::DATE','ROUND(block_id, -3)', 'event_type'], + merge_exclude_columns = ["inserted_timestamp"], + post_hook = enable_search_optimization('{{this.schema}}', '{{this.identifier}}', 'ON EQUALITY(tx_id, mint, mint_authority)'), + full_refresh = false, meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'NFT' }}}, tags = ['scheduled_non_core'] ) }} +{% if execute %} + {% if is_incremental() %} + {% set max_modified_query %} + SELECT + MAX(modified_timestamp) AS modified_timestamp + FROM + {{ this }} + {% endset %} + {% set max_modified_timestamp = run_query(max_modified_query)[0][0] %} + {% endif %} +{% endif %} + SELECT block_id, block_timestamp, @@ -24,16 +42,14 @@ SELECT ['tx_id', 'index', 'inner_index', 'mint'] ) }} ) AS fact_nft_mint_actions_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp + sysdate() AS inserted_timestamp, + sysdate() AS modified_timestamp FROM {{ ref('silver__nft_mint_actions') }} +{% if is_incremental() %} +WHERE + modified_timestamp >= '{{ max_modified_timestamp }}' +{% endif %} diff --git a/models/silver/metadata/helius/silver__helius_cnft_metadata.sql b/models/silver/metadata/helius/silver__helius_cnft_metadata.sql index 715d9432..8c515503 100644 --- a/models/silver/metadata/helius/silver__helius_cnft_metadata.sql +++ b/models/silver/metadata/helius/silver__helius_cnft_metadata.sql @@ -4,7 +4,6 @@ materialized = 'incremental', unique_key = "mint", cluster_by = ['_inserted_timestamp::DATE'], - post_hook = enable_search_optimization('{{this.schema}}', '{{this.identifier}}', 'ON EQUALITY(mint,nft_name,nft_collection_id)'), tags = ['scheduled_non_core'] ) }} diff --git a/models/silver/metadata/helius/silver__helius_nft_metadata.sql b/models/silver/metadata/helius/silver__helius_nft_metadata.sql index a3690987..73319471 100644 --- a/models/silver/metadata/helius/silver__helius_nft_metadata.sql +++ b/models/silver/metadata/helius/silver__helius_nft_metadata.sql @@ -3,7 +3,6 @@ unique_key = "mint", incremental_strategy = 'delete+insert', cluster_by = ['_inserted_timestamp::DATE'], - post_hook = enable_search_optimization('{{this.schema}}', '{{this.identifier}}', 'ON EQUALITY(mint,nft_name,nft_collection_id)'), tags = ['nft_api'] ) }} diff --git a/models/silver/nfts/silver__nft_burn_actions.sql b/models/silver/nfts/silver__nft_burn_actions.sql index 9549d858..7d3979ec 100644 --- a/models/silver/nfts/silver__nft_burn_actions.sql +++ b/models/silver/nfts/silver__nft_burn_actions.sql @@ -3,7 +3,6 @@ unique_key = "CONCAT_WS('-', tx_id, index, inner_index, mint)", incremental_strategy = 'delete+insert', cluster_by = ['block_timestamp::DATE'], - post_hook = enable_search_optimization('{{this.schema}}', '{{this.identifier}}', 'ON EQUALITY(tx_id,mint,burn_authority)'), tags = ['scheduled_non_core'] ) }} diff --git a/models/silver/nfts/silver__nft_mint_actions.sql b/models/silver/nfts/silver__nft_mint_actions.sql index 18ea7077..0f4335d1 100644 --- a/models/silver/nfts/silver__nft_mint_actions.sql +++ b/models/silver/nfts/silver__nft_mint_actions.sql @@ -4,7 +4,6 @@ incremental_strategy = 'delete+insert', incremental_predicates = ['block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'], cluster_by = ['block_timestamp::DATE'], - post_hook = enable_search_optimization('{{this.schema}}', '{{this.identifier}}', 'ON EQUALITY(tx_id,mint,mint_authority)'), tags = ['scheduled_non_core'] ) }} diff --git a/models/silver/tokens/silver__token_burn_actions.sql b/models/silver/tokens/silver__token_burn_actions.sql index 1fc818ab..2efc2aa8 100644 --- a/models/silver/tokens/silver__token_burn_actions.sql +++ b/models/silver/tokens/silver__token_burn_actions.sql @@ -3,7 +3,6 @@ unique_key = "CONCAT_WS('-', tx_id, index, inner_index, mint)", incremental_strategy = 'delete+insert', cluster_by = ['block_timestamp::DATE'], - post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(tx_id,mint,burn_authority,token_account)'), tags = ['scheduled_non_core'] ) }} diff --git a/models/silver/tokens/silver__token_mint_actions.sql b/models/silver/tokens/silver__token_mint_actions.sql index 5989d810..b3931308 100644 --- a/models/silver/tokens/silver__token_mint_actions.sql +++ b/models/silver/tokens/silver__token_mint_actions.sql @@ -4,7 +4,6 @@ incremental_strategy = 'delete+insert', incremental_predicates = ['block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'], cluster_by = ['block_timestamp::DATE'], - post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(tx_id,mint,mint_authority,token_account)'), tags = ['scheduled_non_core'] ) }}