diff --git a/macros/dbt/get_merge.sql b/macros/dbt/get_merge.sql new file mode 100644 index 0000000..e3c75ef --- /dev/null +++ b/macros/dbt/get_merge.sql @@ -0,0 +1,4 @@ +{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%} + {% set merge_sql = fsc_utils.get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %} + {{ return(merge_sql) }} +{% endmacro %} diff --git a/models/silver/core/silver__blocks_v2.sql b/models/silver/core/silver__blocks_v2.sql index 687cd22..f3fa684 100644 --- a/models/silver/core/silver__blocks_v2.sql +++ b/models/silver/core/silver__blocks_v2.sql @@ -3,7 +3,7 @@ {{ config ( materialized = "incremental", incremental_strategy = 'merge', - incremental_predicates = ["DBT_INTERNAL_DEST.block_timestamp::DATE >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], + incremental_predicates = ["dynamic_range_predicate","block_timestamp::date"], unique_key = "block_hash", cluster_by = ['modified_timestamp::DATE','partition_key'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_hash)", diff --git a/models/silver/core/silver__chunks_v2.sql b/models/silver/core/silver__chunks_v2.sql index 25f1bd0..bdfb51c 100644 --- a/models/silver/core/silver__chunks_v2.sql +++ b/models/silver/core/silver__chunks_v2.sql @@ -3,7 +3,7 @@ {{ config ( materialized = "incremental", incremental_strategy = 'merge', - incremental_predicates = ["DBT_INTERNAL_DEST.block_timestamp::DATE >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], + incremental_predicates = ["dynamic_range_predicate","block_timestamp::date"], unique_key = "chunk_hash", cluster_by = ['modified_timestamp::DATE','partition_key'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(chunk_hash)", diff --git a/models/silver/core/silver__transactions_v2.sql b/models/silver/core/silver__transactions_v2.sql index 7dbe259..e8e38c7 100644 --- a/models/silver/core/silver__transactions_v2.sql +++ b/models/silver/core/silver__transactions_v2.sql @@ -3,7 +3,7 @@ {{ config ( materialized = "incremental", incremental_strategy = 'merge', - incremental_predicates = ["DBT_INTERNAL_DEST.block_timestamp::DATE >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], + incremental_predicates = ["dynamic_range_predicate","block_timestamp::date"], unique_key = "tx_hash", cluster_by = ['modified_timestamp::DATE','partition_key'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(tx_hash)", diff --git a/models/streamline/core/realtime/streamline__blocks_realtime.sql b/models/streamline/core/realtime/streamline__blocks_realtime.sql index 20b60e6..be321d8 100644 --- a/models/streamline/core/realtime/streamline__blocks_realtime.sql +++ b/models/streamline/core/realtime/streamline__blocks_realtime.sql @@ -10,7 +10,7 @@ "external_table": "blocks_v2", "sql_limit": "10000", "producer_batch_size": "5000", - "worker_batch_size": "3500", + "worker_batch_size": "2500", "sql_source": "{{this.identifier}}", "order_by_column": "block_id DESC" }