An 2636/upgrade to dbt 1.4 (#224)

* update dbt to 1.4rc1, use native merge predicate

* upgrade to dbt 1.4

* use env vars for dbt version and profiles dir

.github/workflows/dbt_docs_update.yml

@@ -6,7 +6,7 @@ on:
       - "main"
 
 env:
-  DBT_PROFILES_DIR: ./
+  DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}"
 
   ACCOUNT: "${{ secrets.ACCOUNT }}"
   ROLE: "${{ secrets.ROLE }}"
@@ -36,7 +36,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip install dbt-snowflake==1.2.0
+          pip install dbt-snowflake==${{ secrets.DBT_VERSION }}
           dbt deps
 
       - name: checkout docs branch

.github/workflows/dbt_run_adhoc.yml

@@ -28,7 +28,7 @@ on:
         required: true
     
 env:
-  DBT_PROFILES_DIR: ./
+  DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}"
 
   ACCOUNT: "${{ secrets.ACCOUNT }}"
   ROLE: "${{ secrets.ROLE }}"
@@ -60,7 +60,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip3 install dbt-snowflake==1.2.0 cli_passthrough requests click
+          pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click
           dbt deps
       - name: Run DBT Jobs
         run: |

.github/workflows/dbt_run_daily.yml

@@ -7,7 +7,7 @@ on:
     - cron: '35 0 * * *'
     
 env:
-  DBT_PROFILES_DIR: ./
+  DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}"
 
   ACCOUNT: "${{ secrets.ACCOUNT }}"
   ROLE: "${{ secrets.ROLE }}"
@@ -36,7 +36,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip3 install dbt-snowflake==1.2.0 cli_passthrough requests click
+          pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click
           dbt deps
       - name: Run DBT Jobs
         run: |

.github/workflows/dbt_run_dev_refresh.yml

@@ -8,7 +8,7 @@ on:
     - cron: '0 6 * * *'
     
 env:
-  DBT_PROFILES_DIR: ./
+  DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}"
 
   ACCOUNT: "${{ secrets.ACCOUNT }}"
   ROLE: "${{ secrets.ROLE }}"
@@ -37,7 +37,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip3 install dbt-snowflake==1.2.0 cli_passthrough requests click
+          pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click
           dbt deps
       - name: Run DBT Jobs
         run: |

.github/workflows/dbt_run_incremental.yml

@@ -8,7 +8,7 @@ on:
     - cron: '0 */1 * * *'
     
 env:
-  DBT_PROFILES_DIR: ./
+  DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}"
 
   ACCOUNT: "${{ secrets.ACCOUNT }}"
   ROLE: "${{ secrets.ROLE }}"
@@ -37,7 +37,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip3 install dbt-snowflake==1.2.0 cli_passthrough requests click
+          pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click
           dbt deps
       - name: Run DBT Jobs
         run: |

.github/workflows/dbt_run_macro_get_token_data.yml

@@ -7,7 +7,7 @@ on:
     - cron: '*/2 * * * *'
     
 env:
-  DBT_PROFILES_DIR: ./
+  DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}"
 
   ACCOUNT: "${{ secrets.ACCOUNT }}"
   ROLE: "${{ secrets.ROLE }}"
@@ -36,7 +36,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip3 install dbt-snowflake==1.2.0 cli_passthrough requests click
+          pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click
           dbt deps
       - name: Run DBT Jobs
         run: |

.github/workflows/dbt_run_weekly.yml

@@ -7,7 +7,7 @@ on:
     - cron: '0 1 * * 0'
     
 env:
-  DBT_PROFILES_DIR: ./
+  DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}"
 
   ACCOUNT: "${{ secrets.ACCOUNT }}"
   ROLE: "${{ secrets.ROLE }}"
@@ -36,7 +36,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip3 install dbt-snowflake==1.2.0 cli_passthrough requests click
+          pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click
           dbt deps
       - name: Run DBT Jobs
         run: |

.github/workflows/dbt_test.yml

@@ -8,7 +8,7 @@ on:
     - cron: '0 9 * * *'
     
 env:
-  DBT_PROFILES_DIR: ./
+  DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}"
 
   ACCOUNT: "${{ secrets.ACCOUNT }}"
   ROLE: "${{ secrets.ROLE }}"
@@ -37,7 +37,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip3 install dbt-snowflake==1.2.0 cli_passthrough requests click
+          pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click
           dbt deps
       - name: Run DBT Jobs
         run: |

.github/workflows/dbt_test_hourly.yml

@@ -8,7 +8,7 @@ on:
     - cron: '20 */1 * * *'
     
 env:
-  DBT_PROFILES_DIR: ./
+  DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}"
 
   ACCOUNT: "${{ secrets.ACCOUNT }}"
   ROLE: "${{ secrets.ROLE }}"
@@ -37,7 +37,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip3 install dbt-snowflake==1.2.0 cli_passthrough requests click
+          pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click
           dbt deps
       - name: Run DBT Jobs
         run: |

dbt_project.yml

@@ -5,6 +5,8 @@ name: "solana_models"
 version: "1.0.0"
 config-version: 2
 
+require-dbt-version: ">=1.4.0"
+
 # This setting configures which "profile" dbt uses for this project.
 profile: "solana"
 

macros/custom_naming_macros.sql

@@ -15,3 +15,9 @@
     {% set split_name = node_name.split('__') %}
     {{ split_name [1] | trim }}
 {%- endmacro %}
+
+{% macro generate_tmp_view_name(model_name) -%}
+    {% set node_name = model_name.name %}
+    {% set split_name = node_name.split('__') %}
+    {{ target.database ~ '.' ~ split_name[0] ~ '.' ~ split_name [1] ~ '__dbt_tmp' | trim }}
+{%- endmacro %}

macros/dbt/get_merge_sql.sql

@@ -1,74 +0,0 @@
-{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}
-  {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}
-{% endmacro %}
-
-{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}
-  {%- set default_cols = dest_columns | map(attribute="quoted") | list -%}
-
-  {%- if merge_update_columns and merge_exclude_columns -%}
-    {{ exceptions.raise_compiler_error(
-        'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'
-    )}}
-  {%- elif merge_update_columns -%}
-    {%- set update_columns = merge_update_columns -%}
-  {%- elif merge_exclude_columns -%}
-    {%- set update_columns = [] -%}
-    {%- for column in dest_columns -%}
-      {% if column.column | lower not in merge_exclude_columns | map("lower") | list %}
-        {%- do update_columns.append(column.quoted) -%}
-      {% endif %}
-    {%- endfor -%}
-  {%- else -%}
-    {%- set update_columns = default_cols -%}
-  {%- endif -%}
-
-  {{ return(update_columns) }}
-
-{% endmacro %}
-
-{% macro snowflake__get_merge_sql(target, source, unique_key, dest_columns, predicates) -%}
-    {%- set predicates = [] if config.get('merge_predicates') is none else config.get('merge_predicates') -%}
-    {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%}
-    {%- set merge_update_columns = config.get('merge_update_columns') -%}
-    {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}
-    {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}
-    {%- set sql_header = config.get('sql_header', none) -%}
-
-    {% if unique_key %}
-        {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}
-            {% for key in unique_key %}
-                {% set this_key_match %}
-                    DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}
-                {% endset %}
-                {% do predicates.append(this_key_match) %}
-            {% endfor %}
-        {% else %}
-            {% set unique_key_match %}
-                DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}
-            {% endset %}
-            {% do predicates.append(unique_key_match) %}
-        {% endif %}
-    {% else %}
-        {% do predicates.append('FALSE') %}
-    {% endif %}
-
-    {{ sql_header if sql_header is not none }}    
-
-    merge into {{ target }} as DBT_INTERNAL_DEST
-        using {{ source }} as DBT_INTERNAL_SOURCE
-        on {{ render(predicates | join(' and ')) }}
-
-    {% if unique_key %}
-    when matched then update set
-        {% for column_name in update_columns -%}
-            {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}
-            {%- if not loop.last %}, {%- endif %}
-        {%- endfor %}
-    {% endif %}
-
-    when not matched then insert
-        ({{ dest_cols_csv }})
-    values
-        ({{ dest_cols_csv }})
-
-{% endmacro %}

macros/dbt/get_tmp_relation_type.sql

@@ -0,0 +1,4 @@
+{% macro dbt_snowflake_get_tmp_relation_type(strategy, unique_key, language) %}
+    -- always table
+    {{ return('table') }}
+{% endmacro %}

macros/tests/compare_model_subset.sql

@@ -11,12 +11,12 @@ b as (
 ),
 a_minus_b as (
     select * from a
-    {{ dbt_utils.except() }}
+    except
     select * from b
 ),
 b_minus_a as (
     select * from b
-    {{ dbt_utils.except() }}
+    except
     select * from a
 ),
 unioned as (

models/silver/silver___inner_instructions.sql

@@ -1,7 +1,7 @@
 {{ config(
     materialized = 'incremental',
     unique_key = ['block_id','tx_id','mapped_instruction_index'],
-    merge_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from {{ this }}__dbt_tmp))'],
+    incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'],
     cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
     full_refresh = false
 ) }}

models/silver/silver___instructions.sql

@@ -1,7 +1,7 @@
 {{ config(
     materialized = 'incremental',
     unique_key = ['block_id','tx_id','index'],
-    merge_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from {{ this }}__dbt_tmp))'],
+    incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'],
     cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
     full_refresh = false
 ) }}

models/silver/silver___post_token_balances.sql

@@ -1,7 +1,7 @@
 {{ config(
     materialized = 'incremental',
     unique_key = ['block_id','tx_id','index'],
-    merge_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from {{ this }}__dbt_tmp))'],
+    incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'],
     cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
     full_refresh = false
 ) }}

models/silver/silver___pre_token_balances.sql

@@ -1,7 +1,7 @@
 {{ config(
     materialized = 'incremental',
     unique_key = ['block_id','tx_id','index'],
-    merge_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from {{ this }}__dbt_tmp))'],
+    incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'],
     cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
     full_refresh = false
 ) }}

models/silver/silver__events.sql

@@ -1,7 +1,7 @@
 {{ config(
     materialized = 'incremental',
     unique_key = ['block_id','tx_id','index'],
-    merge_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from {{ this }}__dbt_tmp))'],
+    incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'],
     cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE','program_id'],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", 
     full_refresh = false

models/silver/silver__transactions.sql

@@ -1,7 +1,7 @@
 {{ config(
     materialized = 'incremental',
     unique_key = "tx_id",
-    merge_predicates = ["DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from {{ this }}__dbt_tmp))"],
+    incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'],
     cluster_by = ['block_timestamp::DATE','block_id','_inserted_timestamp::DATE'],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", 
     full_refresh = false

models/silver/silver__transfers.sql

@@ -1,7 +1,7 @@
 {{ config(
     materialized = 'incremental',
     unique_key = ["block_id","tx_id","index"],
-    merge_predicates = ["DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from {{ this }}__dbt_tmp))"],
+    incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'],
     cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
 ) }}

models/silver/silver__votes.sql

@@ -1,7 +1,7 @@
 {{ config(
     materialized = 'incremental',
     unique_key = "tx_id",
-    merge_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from {{ this }}__dbt_tmp))'],
+    incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'],
     cluster_by = ['block_timestamp::DATE','block_id','_inserted_timestamp::DATE'],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
     full_refresh = false

models/silver/swaps/silver__swaps_intermediate_generic.sql

@@ -1,7 +1,7 @@
 {{ config(
     materialized = 'incremental',
     unique_key = ["block_id","tx_id","swap_index"],
-    merge_predicates = ["DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from {{ this }}__dbt_tmp))"],
+    incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'],
     cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
 ) }}

models/silver/swaps/silver__swaps_intermediate_jupiterv4.sql

@@ -1,7 +1,7 @@
 {{ config(
     materialized = 'incremental',
     unique_key = ["block_id","tx_id","swap_index"],
-    merge_predicates = ["DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from {{ this }}__dbt_tmp))"],
+    incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'],
     cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
 ) }}

models/silver/swaps/silver__swaps_intermediate_orca.sql

@@ -1,7 +1,7 @@
 {{ config(
     materialized = 'incremental',
     unique_key = ["block_id","tx_id","swap_index"],
-    merge_predicates = ["DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from {{ this }}__dbt_tmp))"],
+    incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'],
     cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
 ) }}

models/silver/swaps/silver__swaps_intermediate_raydium.sql

@@ -1,7 +1,7 @@
 {{ config(
     materialized = 'incremental',
     unique_key = ["block_id","tx_id","swap_index"],
-    merge_predicates = ["DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from {{ this }}__dbt_tmp))"],
+    incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'],
     cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
 ) }}
 

packages.yml

@@ -1,7 +1,7 @@
 packages:
   - package: calogica/dbt_expectations
-    version: 0.7.0
+    version: 0.8.2
   - package: dbt-labs/dbt_external_tables
-    version: 0.8.0
+    version: 0.8.2
   - package: dbt-labs/dbt_utils
-    version: 0.9.2
+    version: 1.0.0