1 time gap heal logic

dbt_project.yml

@@ -90,6 +90,13 @@ vars:
         ROLES:
           - AWS_LAMBDA_NEAR_API
           - INTERNAL_DEV
+    prod:
+        API_INTEGRATION: AWS_NEAR_API_PROD
+        EXTERNAL_FUNCTION_URI: todo
+        API_AWS_ROLE_ARN: todo
+        ROLES:
+          - AWS_LAMBDA_NEAR_API
+          - DBT_CLOUD_NEAR
 
 dispatch:
   - macro_namespace: dbt

models/streamline/core/realtime/streamline__blocks_realtime.sql

@@ -21,10 +21,11 @@
 
 WITH 
 {% if var('STREAMLINE_PARTIAL_BACKFILL', false) %}
-last_3_days AS (
+tbl AS (
     SELECT
-        120960000 as block_id
-),
+        distinct block_id
+    FROM near.tests_full.final_gaps_tx
+)
 {% else %}
 last_3_days AS (
 
@@ -33,7 +34,6 @@ last_3_days AS (
     FROM
         {{ ref("_block_lookback") }}
 ),
-{% endif %}
 tbl AS (
     SELECT
         block_id
@@ -70,6 +70,7 @@ tbl AS (
         )
         AND block_hash IS NOT NULL
 )
+{% endif %}
 SELECT
     block_id,
     DATE_PART('EPOCH', SYSDATE()) :: INTEGER AS partition_key,

models/streamline/core/realtime/streamline__chunks_realtime.sql

@@ -11,7 +11,7 @@
             "producer_batch_size": "42000",
             "worker_batch_size": "21000",
             "sql_source": "{{this.identifier}}",
-            "order_by_column": "block_id DESC"
+            "order_by_column": "block_id ASC"
         }
     ),
     tags = ['streamline_realtime']
@@ -22,7 +22,7 @@ WITH
 {% if var('STREAMLINE_PARTIAL_BACKFILL', false) %}
 last_3_days AS (
     SELECT
-        120960000 as block_id
+        120939872 as block_id
 ),
 {% else %}
 last_3_days AS (

models/streamline/core/realtime/streamline__transactions_realtime.sql

@@ -11,7 +11,7 @@
             "producer_batch_size": "100000",
             "worker_batch_size": "25000",
             "sql_source": "{{this.identifier}}",
-            "order_by_column": "block_id DESC"
+            "order_by_column": "block_id ASC"
         }
     ),
     tags = ['streamline_realtime']
@@ -27,7 +27,7 @@ WITH
 {% if var('STREAMLINE_PARTIAL_BACKFILL', false) %}
 last_3_days AS (
     SELECT
-        120960000 as block_id
+        120939872 as block_id
 ),
 {% else %}
 last_3_days AS (

models/streamline/core/streamline__chunks.sql

@@ -1,3 +1,4 @@
+-- depends_on: {{ ref('streamline__blocks_complete') }}
 {{ config (
     materialized = "incremental",
     unique_key = "chunk_hash",

models/streamline/core/streamline__transactions.sql

@@ -1,3 +1,4 @@
+-- depends_on: {{ ref('streamline__chunks_complete') }}
 {{ config (
     materialized = "incremental",
     unique_key = "tx_hash",

tests/gaps/downstream/tests__final_gaps_tx.sql

@@ -9,7 +9,8 @@ r_transactions AS (
     SELECT
         DISTINCT tx_hash,
         block_id,
-        _signer_id AS signer_id
+        _signer_id AS signer_id,
+        shard_id
     FROM
         {{ ref('silver__streamline_transactions') }}
 
@@ -42,7 +43,8 @@ f_transactions AS (
 SELECT
     r_transactions.tx_hash AS tx_hash,
     r_transactions.block_id,
-    r_transactions.signer_id
+    r_transactions.signer_id,
+    r_transactions.shard_id
 FROM
    r_transactions
 LEFT JOIN