From f2f28ee9e4a4edfcaebded966df19177da1092cf Mon Sep 17 00:00:00 2001
From: desmond-hui <97470747+desmond-hui@users.noreply.github.com>
Date: Wed, 3 Jan 2024 12:39:56 -0800
Subject: [PATCH] An 4323/initial streamline 2.0 setup (#431)

* wip

* change udf name, fix surrogate key for inner instruct calls

* (wip) decoded instructions backfill helpers

* complete backfill helpers, clean up views with 0 requests

* rename

* revert

* temp change will revert

* rename

* use streamline 2.0 verify idl api endpoint

* this is placeholder model, will replace existing when ready

* reorg files

* add workflow for decode instructions real time

* use pip cache

* update with prod endpoints

* update sql limit to real value, should be union all

* use ref for model names

* remove placeholder, replace existing model logic

* add new decoder flow to gha

* temp disable incremental

* add inner index to core tables

* match existing behavior until we properly incorporate inner_instruction txs

* temp disable

* re-enable models

* remove unused comment

* properly handle inner_index null instances in unique key
---
 .github/workflows/dbt_run_decode_instructions.yml            | 3 ---
 models/gold/core/core__ez_events_decoded.sql                 | 1 +
 models/gold/core/core__fact_decoded_instructions.sql         | 3 ++-
 models/silver/parser/silver__decoded_instructions.sql        | 5 +++--
 models/silver/swaps/silver__swaps_intermediate_jupiterv6.sql | 1 +
 5 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/dbt_run_decode_instructions.yml b/.github/workflows/dbt_run_decode_instructions.yml
index b5c3bfeb..4134d8d8 100644
--- a/.github/workflows/dbt_run_decode_instructions.yml
+++ b/.github/workflows/dbt_run_decode_instructions.yml
@@ -5,9 +5,6 @@ on:
   workflow_dispatch:
     branches:
       - "main"
-#   schedule:
-#     # Runs every 30 mins (see https://crontab.guru)
-#     - cron: '*/30 * * * *'
     
 env:
   DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
diff --git a/models/gold/core/core__ez_events_decoded.sql b/models/gold/core/core__ez_events_decoded.sql
index 5589be12..9c5583af 100644
--- a/models/gold/core/core__ez_events_decoded.sql
+++ b/models/gold/core/core__ez_events_decoded.sql
@@ -1,5 +1,6 @@
 {{ config(
     materialized = 'view',
+    tags = ['scheduled_non_core'],
 ) }}
 
 SELECT
diff --git a/models/gold/core/core__fact_decoded_instructions.sql b/models/gold/core/core__fact_decoded_instructions.sql
index 5d91e2c3..b4df1742 100644
--- a/models/gold/core/core__fact_decoded_instructions.sql
+++ b/models/gold/core/core__fact_decoded_instructions.sql
@@ -1,5 +1,6 @@
 {{ config(
-    materialized = 'view'
+    materialized = 'view',
+    tags = ['scheduled_non_core'],
 ) }}
 
 SELECT
diff --git a/models/silver/parser/silver__decoded_instructions.sql b/models/silver/parser/silver__decoded_instructions.sql
index 5f8a922b..f4f6a837 100644
--- a/models/silver/parser/silver__decoded_instructions.sql
+++ b/models/silver/parser/silver__decoded_instructions.sql
@@ -2,10 +2,11 @@
 -- depends_on: {{ ref('bronze__streamline_FR_decoded_instructions_2') }}
 {{ config(
     materialized = 'incremental',
-    unique_key = ["tx_id", "index", "inner_index" ],
+    unique_key = "decoded_instructions_id",
     cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE','program_id'],
     post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}'),
     merge_exclude_columns = ["inserted_timestamp"],
+    tags = ['scheduled_non_core'],
 ) }}
 
 SELECT
@@ -54,6 +55,6 @@ WHERE
     )
 {% endif %}
 
-qualify(ROW_NUMBER() over (PARTITION BY tx_id, INDEX, inner_index
+qualify(ROW_NUMBER() over (PARTITION BY tx_id, INDEX, coalesce(inner_index,-1)
 ORDER BY
     A._inserted_timestamp DESC)) = 1
diff --git a/models/silver/swaps/silver__swaps_intermediate_jupiterv6.sql b/models/silver/swaps/silver__swaps_intermediate_jupiterv6.sql
index 7777c02e..8041712b 100644
--- a/models/silver/swaps/silver__swaps_intermediate_jupiterv6.sql
+++ b/models/silver/swaps/silver__swaps_intermediate_jupiterv6.sql
@@ -4,6 +4,7 @@
     incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'],
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
+    tags = ['scheduled_non_core'],
 ) }}
 
 WITH base AS (