From 45a68debed36be9e41a4885fbf318c727dbb4b75 Mon Sep 17 00:00:00 2001
From: Jack Forgash <58153492+forgxyz@users.noreply.github.com>
Date: Wed, 6 Aug 2025 12:40:17 -0400
Subject: [PATCH] AN-6231/modernize incremental predicates (#442)

* upd incr predicate on silver core models

* upd rest of the models

* rm predicate from d+i model
---
 macros/dbt/get_merge.sql                                    | 4 ++++
 models/evm/silver/core/silver_evm__created_contracts.sql    | 1 +
 .../core/complete/streamline__complete_get_evm_blocks.sql   | 1 +
 .../core/complete/streamline__complete_get_evm_receipts.sql | 1 +
 .../core/complete/streamline__complete_get_evm_traces.sql   | 1 +
 models/gold/core/core__dim_address_mapping.sql              | 1 +
 models/gold/core/core__ez_token_transfers.sql               | 2 +-
 models/gold/core/core__ez_transaction_actors.sql            | 2 +-
 models/silver/core/silver__streamline_blocks.sql            | 1 +
 models/silver/core/silver__streamline_collections.sql       | 1 +
 models/silver/core/silver__streamline_events.sql            | 1 +
 .../silver/core/silver__streamline_transaction_results.sql  | 2 +-
 models/silver/core/silver__streamline_transactions.sql      | 1 +
 .../silver/core/silver__streamline_transactions_final.sql   | 1 +
 models/silver/curated/silver__flow_evm_address_map.sql      | 1 +
 models/silver/defi/silver__swaps_aggregator.sql             | 1 +
 models/silver/transfers/silver__bridge_complete.sql         | 1 +
 models/silver/transfers/silver__token_transfers.sql         | 2 +-
 package-lock.yml                                            | 6 +++---
 packages.yml                                                | 2 +-
 20 files changed, 25 insertions(+), 8 deletions(-)
 create mode 100644 macros/dbt/get_merge.sql

diff --git a/macros/dbt/get_merge.sql b/macros/dbt/get_merge.sql
new file mode 100644
index 0000000..e3c75ef
--- /dev/null
+++ b/macros/dbt/get_merge.sql
@@ -0,0 +1,4 @@
+{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}
+    {% set merge_sql = fsc_utils.get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %}
+    {{ return(merge_sql) }}
+{% endmacro %}
diff --git a/models/evm/silver/core/silver_evm__created_contracts.sql b/models/evm/silver/core/silver_evm__created_contracts.sql
index 5a31cd3..5e4f038 100644
--- a/models/evm/silver/core/silver_evm__created_contracts.sql
+++ b/models/evm/silver/core/silver_evm__created_contracts.sql
@@ -1,5 +1,6 @@
 {{ config (
     materialized = "incremental",
+    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"],
     unique_key = "created_contract_address",
     merge_exclude_columns = ["inserted_timestamp"],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(block_timestamp, tx_hash, created_contract_address, creator_address), SUBSTRING(created_contract_address, creator_address)",
diff --git a/models/evm/streamline/core/complete/streamline__complete_get_evm_blocks.sql b/models/evm/streamline/core/complete/streamline__complete_get_evm_blocks.sql
index 2d4ea66..05813ed 100644
--- a/models/evm/streamline/core/complete/streamline__complete_get_evm_blocks.sql
+++ b/models/evm/streamline/core/complete/streamline__complete_get_evm_blocks.sql
@@ -2,6 +2,7 @@
 -- depends_on: {{ ref('bronze_evm__FR_blocks') }}
 {{ config (
     materialized = "incremental",
+    incremental_predicates = ["dynamic_range_predicate", "partition_key"],
     unique_key = "block_number",
     cluster_by = "ROUND(block_number, -3)",
     merge_exclude_columns = ["inserted_timestamp"],
diff --git a/models/evm/streamline/core/complete/streamline__complete_get_evm_receipts.sql b/models/evm/streamline/core/complete/streamline__complete_get_evm_receipts.sql
index 07d101e..ffc7c97 100644
--- a/models/evm/streamline/core/complete/streamline__complete_get_evm_receipts.sql
+++ b/models/evm/streamline/core/complete/streamline__complete_get_evm_receipts.sql
@@ -2,6 +2,7 @@
 -- depends_on: {{ ref('bronze_evm__FR_receipts') }}
 {{ config (
     materialized = "incremental",
+    incremental_predicates = ["dynamic_range_predicate", "partition_key"],
     unique_key = "block_number",
     cluster_by = "ROUND(block_number, -3)",
     merge_exclude_columns = ["inserted_timestamp"],
diff --git a/models/evm/streamline/core/complete/streamline__complete_get_evm_traces.sql b/models/evm/streamline/core/complete/streamline__complete_get_evm_traces.sql
index 3b0d012..35ae4f4 100644
--- a/models/evm/streamline/core/complete/streamline__complete_get_evm_traces.sql
+++ b/models/evm/streamline/core/complete/streamline__complete_get_evm_traces.sql
@@ -2,6 +2,7 @@
 -- depends_on: {{ ref('bronze_evm__FR_traces') }}
 {{ config (
     materialized = "incremental",
+    incremental_predicates = ["dynamic_range_predicate", "partition_key"],
     unique_key = "block_number",
     cluster_by = "ROUND(block_number, -3)",
     merge_exclude_columns = ["inserted_timestamp"],
diff --git a/models/gold/core/core__dim_address_mapping.sql b/models/gold/core/core__dim_address_mapping.sql
index 9de9bc5..5e45c24 100644
--- a/models/gold/core/core__dim_address_mapping.sql
+++ b/models/gold/core/core__dim_address_mapping.sql
@@ -1,5 +1,6 @@
 {{ config (
     materialized = 'incremental',
+    incremental_predicates = ["dynamic_range_predicate", "block_timestamp_associated::DATE"],
     incremental_strategy = 'merge',
     merge_exclude_columns = ['inserted_timestamp'],
     unique_key = 'dim_address_mapping_id',
diff --git a/models/gold/core/core__ez_token_transfers.sql b/models/gold/core/core__ez_token_transfers.sql
index b6ffe73..e14172e 100644
--- a/models/gold/core/core__ez_token_transfers.sql
+++ b/models/gold/core/core__ez_token_transfers.sql
@@ -2,7 +2,7 @@
     materialized = 'incremental',
     incremental_strategy = 'merge',
     merge_exclude_columns = ['inserted_timestamp'],
-    incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"],
+    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"],
     cluster_by = ['block_timestamp::date', 'modified_timestamp::date'],
     unique_key = "ez_token_transfers_id",
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_id,sender,recipient,token_contract);",
diff --git a/models/gold/core/core__ez_transaction_actors.sql b/models/gold/core/core__ez_transaction_actors.sql
index b7693bc..b95ec49 100644
--- a/models/gold/core/core__ez_transaction_actors.sql
+++ b/models/gold/core/core__ez_transaction_actors.sql
@@ -3,7 +3,7 @@
     unique_key = 'ez_transaction_actors_id',
     incremental_strategy = 'merge',
     merge_exclude_columns = ['inserted_timestamp'],
-    incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"],
+    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"],
     cluster_by = 'block_timestamp::date',
     post_hook = 'ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_id,actors);',
     tags = ['scheduled_non_core']
diff --git a/models/silver/core/silver__streamline_blocks.sql b/models/silver/core/silver__streamline_blocks.sql
index 74b7d51..a82182f 100644
--- a/models/silver/core/silver__streamline_blocks.sql
+++ b/models/silver/core/silver__streamline_blocks.sql
@@ -5,6 +5,7 @@
     materialized = 'incremental',
     unique_key = "block_number",
     incremental_strategy = 'merge',
+    incremental_predicates = ["dynamic_range_predicate", "_partition_by_block_id"],
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = "block_timestamp::date",
     tags = ['streamline_load', 'core', 'scheduled_core']
diff --git a/models/silver/core/silver__streamline_collections.sql b/models/silver/core/silver__streamline_collections.sql
index d7d1bc4..077b528 100644
--- a/models/silver/core/silver__streamline_collections.sql
+++ b/models/silver/core/silver__streamline_collections.sql
@@ -3,6 +3,7 @@
     materialized = 'incremental',
     unique_key = "collection_id",
     incremental_strategy = 'merge',
+    incremental_predicates = ["dynamic_range_predicate", "block_number"],
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = ['_inserted_timestamp :: DATE', 'block_number'],
     tags = ['streamline_load', 'core', 'scheduled_core']
diff --git a/models/silver/core/silver__streamline_events.sql b/models/silver/core/silver__streamline_events.sql
index 341f15b..5c72df1 100644
--- a/models/silver/core/silver__streamline_events.sql
+++ b/models/silver/core/silver__streamline_events.sql
@@ -2,6 +2,7 @@
     materialized = 'incremental',
     unique_key = 'event_id',
     incremental_strategy = 'merge',
+    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"],
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = "block_timestamp::date",
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_id,event_id,event_contract,event_type);",
diff --git a/models/silver/core/silver__streamline_transaction_results.sql b/models/silver/core/silver__streamline_transaction_results.sql
index 2c17f89..013cd29 100644
--- a/models/silver/core/silver__streamline_transaction_results.sql
+++ b/models/silver/core/silver__streamline_transaction_results.sql
@@ -1,7 +1,7 @@
 -- depends_on: {{ ref('bronze__streamline_transaction_results') }}
 {{ config(
     materialized = 'incremental',
-    incremental_predicates = ['DBT_INTERNAL_DEST.block_number >= (select min(block_number) from ' ~ generate_tmp_view_name(this) ~ ')'],
+    incremental_predicates = ["dynamic_range_predicate", "_partition_by_block_id"],
     unique_key = "tx_id",
     incremental_strategy = 'merge',
     merge_exclude_columns = ["inserted_timestamp"],
diff --git a/models/silver/core/silver__streamline_transactions.sql b/models/silver/core/silver__streamline_transactions.sql
index cf7b680..dcb497c 100644
--- a/models/silver/core/silver__streamline_transactions.sql
+++ b/models/silver/core/silver__streamline_transactions.sql
@@ -3,6 +3,7 @@
     materialized = 'incremental',
     unique_key = "tx_id",
     incremental_strategy = 'merge',
+    incremental_predicates = ["dynamic_range_predicate", "_partition_by_block_id"],
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = "_inserted_timestamp::date",
     tags = ['streamline_load', 'core', 'scheduled_core']
diff --git a/models/silver/core/silver__streamline_transactions_final.sql b/models/silver/core/silver__streamline_transactions_final.sql
index 7321de9..404c2ba 100644
--- a/models/silver/core/silver__streamline_transactions_final.sql
+++ b/models/silver/core/silver__streamline_transactions_final.sql
@@ -3,6 +3,7 @@
     materialized = 'incremental',
     unique_key = "tx_id",
     incremental_strategy = 'merge',
+    incremental_predicates = ["dynamic_range_predicate", "_partition_by_block_id"],
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = "block_timestamp::date",
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_id,proposer,payer,authorizers);",
diff --git a/models/silver/curated/silver__flow_evm_address_map.sql b/models/silver/curated/silver__flow_evm_address_map.sql
index 6f0a698..2458d74 100644
--- a/models/silver/curated/silver__flow_evm_address_map.sql
+++ b/models/silver/curated/silver__flow_evm_address_map.sql
@@ -1,5 +1,6 @@
 {{ config(
     materialized = 'incremental',
+    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"],
     unique_key = 'flow_evm_address_map_id',
     incremental_strategy = 'merge',
     merge_exclude_columns = ['inserted_timestamp'],
diff --git a/models/silver/defi/silver__swaps_aggregator.sql b/models/silver/defi/silver__swaps_aggregator.sql
index 835e826..d9a29f0 100644
--- a/models/silver/defi/silver__swaps_aggregator.sql
+++ b/models/silver/defi/silver__swaps_aggregator.sql
@@ -1,5 +1,6 @@
 {{ config(
     materialized = 'incremental',
+    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"],
     incremental_strategy = 'merge',
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = ['inserted_timestamp::DATE'],
diff --git a/models/silver/transfers/silver__bridge_complete.sql b/models/silver/transfers/silver__bridge_complete.sql
index 1c66e4b..5c76630 100644
--- a/models/silver/transfers/silver__bridge_complete.sql
+++ b/models/silver/transfers/silver__bridge_complete.sql
@@ -1,5 +1,6 @@
 {{ config(
     materialized = 'incremental',
+    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"],
     incremental_strategy = 'merge',
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = ['block_timestamp::date'],
diff --git a/models/silver/transfers/silver__token_transfers.sql b/models/silver/transfers/silver__token_transfers.sql
index f0dd543..ca20a22 100644
--- a/models/silver/transfers/silver__token_transfers.sql
+++ b/models/silver/transfers/silver__token_transfers.sql
@@ -2,7 +2,7 @@
     materialized = 'incremental',
     incremental_strategy = 'merge',
     merge_exclude_columns = ['inserted_timestamp'],
-    incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"],
+    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"],
     cluster_by = ['block_timestamp::date', 'modified_timestamp::date'],
     unique_key = "token_transfers_id",
     tags = ['scheduled_non_core']
diff --git a/package-lock.yml b/package-lock.yml
index acb1731..68f92db 100644
--- a/package-lock.yml
+++ b/package-lock.yml
@@ -6,11 +6,11 @@ packages:
   - package: dbt-labs/dbt_utils
     version: 1.0.0
   - git: https://github.com/FlipsideCrypto/fsc-utils.git
-    revision: d3cf679e079f0cf06142de9386f215e55fe26b3b
+    revision: 87e00eb90acddcc7a34aa8e67e3b3bac86b262e6
   - package: get-select/dbt_snowflake_query_tags
     version: 2.5.0
   - package: calogica/dbt_date
     version: 0.7.2
   - git: https://github.com/FlipsideCrypto/livequery-models.git
-    revision: b024188be4e9c6bc00ed77797ebdc92d351d620e
-sha1_hash: 3fb8d6ca492a03f5aef6f281508aaa0b34c989d4
+    revision: 2651a45b7e123f7bd421bcc0e7e2a7bcbaf7652f
+sha1_hash: a1cc3545d7ef13fcf5b3908a9e888b4421018792
diff --git a/packages.yml b/packages.yml
index c5ea06d..1b272c8 100644
--- a/packages.yml
+++ b/packages.yml
@@ -6,6 +6,6 @@ packages:
   - package: dbt-labs/dbt_utils
     version: 1.0.0
   - git: https://github.com/FlipsideCrypto/fsc-utils.git
-    revision: v1.32.0
+    revision: v1.35.1
   - package: get-select/dbt_snowflake_query_tags
     version: [">=2.0.0", "<3.0.0"]