An 4323/initial streamline 2.0 setup (#431)

* wip

* change udf name, fix surrogate key for inner instruct calls

* (wip) decoded instructions backfill helpers

* complete backfill helpers, clean up views with 0 requests

* rename

* revert

* temp change will revert

* rename

* use streamline 2.0 verify idl api endpoint

* this is placeholder model, will replace existing when ready

* reorg files

* add workflow for decode instructions real time

* use pip cache

* update with prod endpoints

* update sql limit to real value, should be union all

* use ref for model names

* remove placeholder, replace existing model logic

* add new decoder flow to gha

* temp disable incremental

* add inner index to core tables

* match existing behavior until we properly incorporate inner_instruction txs

* temp disable

* re-enable models

* remove unused comment

* properly handle inner_index null instances in unique key

.github/workflows/dbt_run_decode_instructions.yml

@@ -5,9 +5,6 @@ on:
   workflow_dispatch:
     branches:
       - "main"
-#   schedule:
-#     # Runs every 30 mins (see https://crontab.guru)
-#     - cron: '*/30 * * * *'
     
 env:
   DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"

models/gold/core/core__ez_events_decoded.sql

@@ -1,5 +1,6 @@
 {{ config(
     materialized = 'view',
+    tags = ['scheduled_non_core'],
 ) }}
 
 SELECT

models/gold/core/core__fact_decoded_instructions.sql

@@ -1,5 +1,6 @@
 {{ config(
-    materialized = 'view'
+    materialized = 'view',
+    tags = ['scheduled_non_core'],
 ) }}
 
 SELECT

models/silver/parser/silver__decoded_instructions.sql

@@ -2,10 +2,11 @@
 -- depends_on: {{ ref('bronze__streamline_FR_decoded_instructions_2') }}
 {{ config(
     materialized = 'incremental',
-    unique_key = ["tx_id", "index", "inner_index" ],
+    unique_key = "decoded_instructions_id",
     cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE','program_id'],
     post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}'),
     merge_exclude_columns = ["inserted_timestamp"],
+    tags = ['scheduled_non_core'],
 ) }}
 
 SELECT
@@ -54,6 +55,6 @@ WHERE
     )
 {% endif %}
 
-qualify(ROW_NUMBER() over (PARTITION BY tx_id, INDEX, inner_index
+qualify(ROW_NUMBER() over (PARTITION BY tx_id, INDEX, coalesce(inner_index,-1)
 ORDER BY
     A._inserted_timestamp DESC)) = 1

models/silver/swaps/silver__swaps_intermediate_jupiterv6.sql

@@ -4,6 +4,7 @@
     incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::date >= LEAST(current_date-7,(select min(block_timestamp)::date from ' ~ generate_tmp_view_name(this) ~ '))'],
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
+    tags = ['scheduled_non_core'],
 ) }}
 
 WITH base AS (