blocks silver

models/bronze/bronze__FR_blocks.sql

@@ -1,3 +1,4 @@
+-- TODO - v2 naming?
 {{ config (
     materialized = 'view',
     tags = ['streamline_helper']

models/silver/actions/silver__actions_events_addkey_s3.sql

@@ -1,3 +1,5 @@
+-- TODO slated for deprecation and drop
+-- Note - a model in Social does use this 
 {{ config(
   materialized = 'incremental',
   incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"],

models/silver/actions/silver__actions_events_function_call_s3.sql

@@ -1,3 +1,6 @@
+-- TODO slated for deprecation and drop
+-- Note - must migrate curated to new ez_actions first
+
 {{ config(
   materialized = 'incremental',
   incremental_strategy = 'merge',

models/silver/actions/silver__actions_events_s3.sql

@@ -1,3 +1,5 @@
+-- TODO slated for deprecation and drop
+-- Note - must migrate curated to new ez_actions first
 {{ config(
     materialized = 'incremental',
     incremental_strategy = 'merge',

models/silver/atlas/silver__atlas_accounts_created.sql

@@ -1,3 +1,5 @@
+-- TODO slated for deprecation and drop
+
 {{ config(
     materialized = 'incremental',
     unique_key = 'atlas_account_created_id',

models/silver/atlas/silver__atlas_address_first_action.sql

@@ -1,3 +1,5 @@
+-- TODO slated for deprecation and drop
+
 {{ config(
     materialized = 'incremental',
     incremental_stratege = 'merge',

models/silver/atlas/silver__atlas_maa.sql

@@ -1,3 +1,5 @@
+-- TODO slated for deprecation and drop
+
 {{ config(
     materialized = 'incremental',
     incremental_stratege = 'merge',

models/silver/atlas/silver__atlas_nft_30_trailing.sql

@@ -1,3 +1,5 @@
+-- TODO slated for deprecation and drop
+
 {{ config(
     materialized = 'incremental',
     unique_key = 'atlas_nft_30_trailing_id',

models/silver/atlas/silver__atlas_nft_detailed.sql

@@ -1,3 +1,5 @@
+-- TODO slated for deprecation and drop
+
 {{ config(
     materialized = 'table',
     unique_key = 'atlas_nft_detailed_id',

models/silver/atlas/silver__atlas_nft_table.sql

@@ -1,3 +1,5 @@
+-- TODO slated for deprecation and drop
+
 {{ config(
   materialized = 'table',
   unique_key = 'atlas_nft_table_id',

models/silver/atlas/silver__atlas_nft_transactions.sql

@@ -1,3 +1,5 @@
+-- TODO slated for deprecation and drop
+
 {{ config(
     materialized = "incremental",
     cluster_by = ["day"],

models/silver/core/lake/silver__streamline_blocks.sql

@@ -1,3 +1,4 @@
+-- Deprecated Process
 {{ config(
     materialized = 'incremental',
     incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"],

models/silver/core/silver__blocks_v2.sql

@@ -0,0 +1,46 @@
+-- depends_on: {{ ref('bronze__blocks') }}
+-- depends_on: {{ ref('bronze__FR_blocks') }}
+
+{{ config (
+    materialized = "incremental",
+    incremental_strategy = 'merge',
+    unique_key = "block_hash",
+    cluster_by = ['modified_timestamp::DATE','partition_key'],
+    post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)",
+    tags = ['scheduled_core']
+) }}
+
+WITH bronze_blocks AS (
+    SELECT 
+        VALUE :BLOCK_NUMBER :: INT AS block_number,
+        DATA :header :hash :: STRING AS block_hash,
+        partition_key,
+        DATA :: VARIANT AS block_json,
+        _inserted_timestamp
+    FROM 
+    {% if is_incremental() %}
+    {{ ref('bronze__blocks') }}
+    WHERE _inserted_timestamp >= (
+        SELECT 
+            COALESCE(MAX(_inserted_timestamp), '1900-01-01'::TIMESTAMP) AS _inserted_timestamp
+        FROM {{ this }}
+    ) AND DATA IS NOT NULL
+    {% else %}
+    {{ ref('bronze__FR_blocks') }}
+    WHERE DATA IS NOT NULL
+    {% endif %}
+)
+
+SELECT 
+    block_number,
+    block_hash,
+    partition_key,
+    block_json,
+    _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(['block_hash']) }} AS blocks_v2_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM bronze_blocks
+
+QUALIFY ROW_NUMBER() OVER (PARTITION BY block_hash ORDER BY _inserted_timestamp DESC) = 1

models/streamline/core/complete/streamline__blocks_complete.sql

@@ -10,7 +10,7 @@
 ) }}
 
 SELECT
-    VALUE :block_number :: INT AS block_number,
+    VALUE :BLOCK_NUMBER :: INT AS block_number,
     DATA :header :hash :: STRING AS block_hash,
     partition_key,
     _inserted_timestamp,

models/streamline/core/realtime/streamline__blocks_realtime.sql

@@ -5,10 +5,11 @@
         target = "{{this.schema}}.{{this.identifier}}",
         params = {
             "external_table": "blocks_v2",
-            "sql_limit": "100",
-            "producer_batch_size": "100",
-            "worker_batch_size": "100",
-            "sql_source": "{{this.identifier}}"
+            "sql_limit": "3000",
+            "producer_batch_size": "3000",
+            "worker_batch_size": "3000",
+            "sql_source": "{{this.identifier}}",
+            "order_by_column": "block_number DESC"
         }
     ),
     tags = ['streamline_realtime']
@@ -53,7 +54,7 @@ tbl AS (
             -4,
             SYSDATE()
         )
-        -- AND {} IS NOT NULL -- TODO, determine identifier for bad response
+        AND block_hash IS NOT NULL
 )
 SELECT
     block_number,
@@ -75,5 +76,3 @@ SELECT
         'Vault/prod/near/quicknode/mainnet'
     ) AS request
 FROM tbl
-ORDER BY 
-    block_number DESC