Sources and disable old models (#344)

* sources and views for disabled models

* disable models and clean workflow

* reference views

* set full_refresh to false

.github/workflows/dbt_run_incremental.yml

@@ -42,6 +42,6 @@ jobs:
       - name: Run DBT Jobs
         run: |
           dbt run-operation run_sp_refresh_external_tables_full
-          dbt run -s ./models --exclude models/gold tag:share models/streamline models/silver/silver__daily_signers.sql models/silver/silver__signers.sql models/silver/accounts/silver__token_account_owners_intermediate.sql models/silver/accounts/silver__token_account_owners.sql models/silver/silver__signers_nfts_held.sql models/silver/validator models/silver/nfts/silver__nft_sales_opensea.sql models/silver/nfts/silver__nft_sales_yawww.sql models/silver/nfts/silver__nft_bids_yawww.sql models/silver/_observability models/silver/nfts/silver__nft_compressed_mints_onchain.sql models/silver/nfts/silver__nft_compressed_mints.sql
+          dbt run -s ./models --exclude models/gold tag:share models/streamline models/silver/silver__daily_signers.sql models/silver/silver__signers.sql models/silver/validator models/silver/_observability models/silver/nfts/silver__nft_compressed_mints_onchain.sql models/silver/nfts/silver__nft_compressed_mints.sql
           dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":True}' -s ./models/gold --exclude models/gold/core/core__ez_signers.sql
           

models/gold/nft/nft__fact_nft_sales.sql

@@ -89,7 +89,7 @@ SELECT
     mint,
     sales_amount
 FROM
-    {{ ref('silver__nft_sales_opensea') }}
+    {{ ref('silver__nft_sales_opensea_view') }}
 UNION
 SELECT
     'yawww',
@@ -103,7 +103,7 @@ SELECT
     mint,
     sales_amount
 FROM
-    {{ ref('silver__nft_sales_yawww') }}
+    {{ ref('silver__nft_sales_yawww_view') }}
 UNION
 SELECT
     'hadeswap',

models/silver/accounts/silver__stake_account_ownership_events.sql

@@ -2,7 +2,9 @@
     materialized = 'incremental',
     unique_key = ["block_id","tx_id","index","inner_index","authority_type"],
     incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'],
-    cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE']
+    cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
+    full_refresh = false,
+    enabled = false,
 ) }}
 
 with base_events as (

models/silver/accounts/silver__token_account_owners.sql

@@ -4,6 +4,8 @@
     unique_key = ["account_address"],
     cluster_by = ['_inserted_timestamp::DATE'],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
+    full_refresh = false,
+    enabled = false,
 ) }}
 
 /* need to rebucket and regroup the intermediate model due to possibility of change events coming in out of order */

models/silver/accounts/silver__token_account_owners_intermediate.sql

@@ -2,6 +2,8 @@
     materialized = 'incremental',
     unique_key = ["account_address","owner","start_block_id"],
     cluster_by = ['_inserted_timestamp::DATE'],
+    full_refresh = false,
+    enabled = false,
 ) }}
 
 /*

models/silver/accounts/silver__token_account_ownership_events.sql

@@ -3,7 +3,8 @@
     unique_key = ["block_id","tx_id","index","inner_index"],
     incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'],
     cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
-    full_refresh = false
+    full_refresh = false,
+    enabled = false,
 ) }}
 
 with base_events as (

models/silver/nfts/silver___nft_distinct_mints.sql

@@ -2,6 +2,8 @@
     materialized = 'incremental',
     unique_key = "mint",
     incremental_strategy = 'delete+insert',
+    full_refresh = false,
+    enabled = false,
 ) }}
 
 WITH base AS (
@@ -112,7 +114,10 @@ SELECT
     mint,
     _inserted_timestamp
 FROM
-    {{ ref('silver__nft_sales_opensea') }}
+    {{ source(
+        'solana_silver',
+        'nft_sales_opensea'
+    ) }}
 
 {% if is_incremental() %}
 WHERE
@@ -128,7 +133,10 @@ SELECT
     mint,
     _inserted_timestamp
 FROM
-    {{ ref('silver__nft_sales_yawww') }}
+    {{ source(
+        'solana_silver',
+        'nft_sales_yawww'
+    ) }}
 
 {% if is_incremental() %}
 WHERE

models/silver/nfts/silver__nft_bids_yawww.sql

@@ -3,6 +3,8 @@
     unique_key = "tx_id",
     incremental_strategy = 'delete+insert',
     cluster_by = ['block_timestamp::DATE'],
+    full_refresh = false,
+    enabled = false,
 ) }}
 
 WITH yawww_txs AS (

models/silver/nfts/silver__nft_sales_opensea.sql

@@ -3,6 +3,8 @@
     unique_key = "CONCAT_WS('-', tx_id, mint)",
     incremental_strategy = 'delete+insert',
     cluster_by = ['block_timestamp::DATE'],
+    full_refresh = false,
+    enabled = false,
 ) }}
 
 WITH sales_inner_instructions AS (

models/silver/nfts/silver__nft_sales_opensea_view.sql

@@ -0,0 +1,19 @@
+{{ config(
+  materialized = 'view'
+) }}
+
+SELECT
+  block_timestamp,
+  block_id,
+  tx_id,
+  succeeded,
+  program_id,
+  purchaser,
+  seller,
+  mint,
+  sales_amount
+FROM
+  {{ source(
+    'solana_silver',
+    'nft_sales_opensea'
+  ) }}

models/silver/nfts/silver__nft_sales_solport.sql

@@ -3,6 +3,7 @@
   unique_key = "CONCAT_WS('-', tx_id, mint)",
   incremental_strategy = 'delete+insert',
   cluster_by = ['block_timestamp::DATE'],
+  full_refresh = false,
   enabled = false,
 ) }}
 

models/silver/nfts/silver__nft_sales_yawww.sql

@@ -3,6 +3,8 @@
     unique_key = "tx_id",
     incremental_strategy = 'delete+insert',
     cluster_by = ['block_timestamp::DATE'],
+    full_refresh = false,
+    enabled = false,
 ) }}
 
 WITH base_table AS (
@@ -109,7 +111,10 @@ price_bids AS (
         acct_2,
         bid_amount
     FROM
-        {{ ref('silver__nft_bids_yawww') }}
+        {{ source(
+            'solana_silver',
+            'nft_bids_yawww'
+        ) }}
         qualify(ROW_NUMBER() over (PARTITION BY bidder, acct_2
     ORDER BY
         block_timestamp DESC)) = 1

models/silver/nfts/silver__nft_sales_yawww_view.sql

@@ -0,0 +1,19 @@
+{{ config(
+  materialized = 'view'
+) }}
+
+SELECT
+  block_timestamp,
+  block_id,
+  tx_id,
+  succeeded,
+  program_id,
+  purchaser,
+  seller,
+  mint,
+  sales_amount
+FROM
+  {{ source(
+    'solana_silver',
+    'nft_sales_yawww'
+  ) }}

models/silver/silver__signers_nfts_held.sql

@@ -3,7 +3,9 @@
     unique_key = "CONCAT_WS('-', signer, nft_held)",
     incremental_strategy = 'delete+insert',
     cluster_by = 'signer', 
-    full_refresh = false
+    full_refresh = false,
+    enabled = false,
+    
 ) }} 
 WITH dates_changed AS (
     SELECT

models/sources.yml

@@ -68,6 +68,9 @@ sources:
     tables:
       - name: nft_sales_solport
       - name: nft_sales_magic_eden_v1
+      - name: nft_sales_opensea
+      - name: nft_sales_yawww
+      - name: nft_bids_yawww
   - name: solana_streamline
     schema: streamline
     tables: