upgrades (#27)

* upgrades

* adds unconfirmed blocks test

models/evm/silver/core/tests/confirmed_blocks/test_silver_evm__confirmed_blocks_full.sql

@@ -0,0 +1,9 @@
+{{ config (
+    materialized = 'view',
+    tags = ['full_evm_test']
+) }}
+
+SELECT
+    *
+FROM
+    {{ ref('silver_evm__confirmed_blocks') }}

models/evm/silver/core/tests/confirmed_blocks/test_silver_evm__confirmed_blocks_full.yml

@@ -0,0 +1,34 @@
+version: 2
+models:
+  - name: test_silver_evm__confirmed_blocks_full
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - TX_HASH
+    columns:
+      - name: BLOCK_NUMBER
+        tests:
+          - not_null  
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER    
+      - name: BLOCK_HASH
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: TX_HASH
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 1
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+                - TIMESTAMP_LTZ

models/evm/silver/core/tests/confirmed_blocks/test_silver_evm__confirmed_blocks_recent.sql

@@ -0,0 +1,23 @@
+{{ config (
+    materialized = 'view',
+    tags = ['recent_evm_test']
+) }}
+
+WITH last_3_days AS (
+
+    SELECT
+        block_number
+    FROM
+        {{ ref("_evm_block_lookback") }}
+)
+SELECT
+    *
+FROM
+    {{ ref('silver_evm__confirmed_blocks') }}
+WHERE
+    block_number >= (
+        SELECT
+            block_number
+        FROM
+            last_3_days
+    )

models/evm/silver/core/tests/confirmed_blocks/test_silver_evm__confirmed_blocks_recent.yml

@@ -0,0 +1,34 @@
+version: 2
+models:
+  - name: test_silver_evm__confirmed_blocks_recent
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - TX_HASH
+    columns:
+      - name: BLOCK_NUMBER
+        tests:
+          - not_null  
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER    
+      - name: BLOCK_HASH
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: TX_HASH
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: hour
+              interval: 3
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+                - TIMESTAMP_LTZ

models/evm/silver/core/tests/transactions/test_silver_evm__transactions_full.yml

@@ -5,11 +5,6 @@ models:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
             - TX_HASH
-      - fsc_utils.sequence_gaps:
-          partition_by:
-            - BLOCK_NUMBER
-          column_name: POSITION
-          where: BLOCK_TIMESTAMP < CURRENT_DATE - 1
     columns:
       - name: BLOCK_NUMBER
         tests:

models/evm/silver/core/tests/transactions/test_silver_evm__transactions_recent.yml

@@ -5,10 +5,6 @@ models:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
             - TX_HASH
-      - fsc_utils.sequence_gaps:
-          partition_by:
-            - BLOCK_NUMBER
-          column_name: POSITION
     columns:
       - name: BLOCK_NUMBER
         tests:

models/evm/streamline/silver/history/streamline__evm_blocks_history.sql

@@ -0,0 +1,83 @@
+{{ config (
+    materialized = "view",
+    post_hook = fsc_utils.if_data_call_function_v2(
+        func = 'streamline.udf_bulk_rest_api_v2',
+        target = "{{this.schema}}.{{this.identifier}}",
+        params ={ "external_table" :"evm_blocks",
+        "sql_limit" :"25000",
+        "producer_batch_size" :"100000",
+        "worker_batch_size" :"10000",
+        "sql_source" :"{{this.identifier}}" }
+    ),
+    tags = ['streamline_core_evm_history']
+) }}
+
+WITH last_3_days AS (
+
+    SELECT
+        block_number
+    FROM
+        {{ ref("_evm_block_lookback") }}
+),
+to_do AS (
+    SELECT
+        block_number
+    FROM
+        {{ ref("streamline__evm_blocks") }}
+    WHERE
+        (
+            block_number < (
+                SELECT
+                    block_number
+                FROM
+                    last_3_days
+            )
+        )
+        AND block_number IS NOT NULL
+    EXCEPT
+    SELECT
+        block_number
+    FROM
+        {{ ref("streamline__complete_evm_blocks") }}
+    WHERE
+        block_number < (
+            SELECT
+                block_number
+            FROM
+                last_3_days
+        )
+),
+ready_blocks AS (
+    SELECT
+        block_number
+    FROM
+        to_do
+)
+SELECT
+    block_number,
+    ROUND(
+        block_number,
+        -3
+    ) :: INT AS partition_key,
+    {{ target.database }}.live.udf_api(
+        'POST',
+        '{Service}/{Authentication}',
+        OBJECT_CONSTRUCT(
+            'Content-Type',
+            'application/json'
+        ),
+        OBJECT_CONSTRUCT(
+            'id',
+            block_number :: STRING,
+            'jsonrpc',
+            '2.0',
+            'method',
+            'eth_getBlockByNumber',
+            'params',
+            ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), FALSE)),
+            'Vault/prod/sei/quicknode/mainnet'
+        ) AS request
+        FROM
+            ready_blocks
+        ORDER BY
+            block_number DESC

models/evm/streamline/silver/history/streamline__evm_confirm_blocks_history.sql

@@ -0,0 +1,92 @@
+{{ config (
+    materialized = "view",
+    post_hook = fsc_utils.if_data_call_function_v2(
+        func = 'streamline.udf_bulk_rest_api_v2',
+        target = "{{this.schema}}.{{this.identifier}}",
+        params ={ "external_table" :"evm_confirm_blocks",
+        "sql_limit" :"25000",
+        "producer_batch_size" :"10000",
+        "worker_batch_size" :"5000",
+        "sql_source" :"{{this.identifier}}" }
+    ),
+    tags = ['streamline_core_evm_history']
+) }}
+
+WITH last_3_days AS (
+
+    SELECT
+        block_number
+    FROM
+        {{ ref("_evm_block_lookback") }}
+),
+look_back AS (
+    SELECT
+        block_number
+    FROM
+        {{ ref("_max_evm_block_by_hour") }}
+        qualify ROW_NUMBER() over (
+            ORDER BY
+                block_number DESC
+        ) = 6
+),
+to_do AS (
+    SELECT
+        block_number
+    FROM
+        {{ ref("streamline__evm_blocks") }}
+    WHERE
+        block_number IS NOT NULL
+        AND block_number < (
+            SELECT
+                block_number
+            FROM
+                last_3_days
+        )
+    EXCEPT
+    SELECT
+        block_number
+    FROM
+        {{ ref("streamline__complete_evm_confirmed_blocks") }}
+    WHERE
+        block_number IS NOT NULL
+        AND block_number < (
+            SELECT
+                block_number
+            FROM
+                last_3_days
+        )
+),
+ready_blocks AS (
+    SELECT
+        block_number
+    FROM
+        to_do
+)
+SELECT
+    block_number,
+    ROUND(
+        block_number,
+        -3
+    ) :: INT AS partition_key,
+    {{ target.database }}.live.udf_api(
+        'POST',
+        '{Service}/{Authentication}',
+        OBJECT_CONSTRUCT(
+            'Content-Type',
+            'application/json'
+        ),
+        OBJECT_CONSTRUCT(
+            'id',
+            block_number :: STRING,
+            'jsonrpc',
+            '2.0',
+            'method',
+            'eth_getBlockByNumber',
+            'params',
+            ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), FALSE)),
+            'Vault/prod/sei/quicknode/mainnet'
+        ) AS request
+        FROM
+            ready_blocks
+        ORDER BY
+            block_number ASC

models/evm/streamline/silver/history/streamline__evm_receipts_history.sql

@@ -0,0 +1,84 @@
+{{ config (
+    materialized = "view",
+    post_hook = fsc_utils.if_data_call_function_v2(
+        func = 'streamline.udf_bulk_rest_api_v2',
+        target = "{{this.schema}}.{{this.identifier}}",
+        params ={ "external_table" :"evm_receipts",
+        "sql_limit" :"25000",
+        "producer_batch_size" :"100000",
+        "worker_batch_size" :"10000",
+        "sql_source" :"{{this.identifier}}",
+        "exploded_key": tojson(["result"]) }
+    ),
+    tags = ['streamline_core_evm_history']
+) }}
+
+WITH last_3_days AS (
+
+    SELECT
+        block_number
+    FROM
+        {{ ref("_evm_block_lookback") }}
+),
+to_do AS (
+    SELECT
+        block_number
+    FROM
+        {{ ref("streamline__evm_blocks") }}
+    WHERE
+        (
+            block_number < (
+                SELECT
+                    block_number
+                FROM
+                    last_3_days
+            )
+        )
+        AND block_number IS NOT NULL
+    EXCEPT
+    SELECT
+        block_number
+    FROM
+        {{ ref("streamline__complete_evm_receipts") }}
+    WHERE
+        block_number < (
+            SELECT
+                block_number
+            FROM
+                last_3_days
+        )
+),
+ready_blocks AS (
+    SELECT
+        block_number
+    FROM
+        to_do
+)
+SELECT
+    block_number,
+    ROUND(
+        block_number,
+        -3
+    ) :: INT AS partition_key,
+    {{ target.database }}.live.udf_api(
+        'POST',
+        '{Service}/{Authentication}',
+        OBJECT_CONSTRUCT(
+            'Content-Type',
+            'application/json'
+        ),
+        OBJECT_CONSTRUCT(
+            'id',
+            block_number :: STRING,
+            'jsonrpc',
+            '2.0',
+            'method',
+            'eth_getBlockReceipts',
+            'params',
+            ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number))),
+            'Vault/prod/sei/quicknode/mainnet'
+        ) AS request
+        FROM
+            ready_blocks
+        ORDER BY
+            block_number DESC

models/evm/streamline/silver/history/streamline__evm_traces_history.sql

@@ -0,0 +1,85 @@
+{{ config (
+    materialized = "view",
+    post_hook = fsc_utils.if_data_call_function_v2(
+        func = 'streamline.udf_bulk_rest_api_v2',
+        target = "{{this.schema}}.{{this.identifier}}",
+        params ={ "external_table" :"evm_traces",
+        "sql_limit" :"25000",
+        "producer_batch_size" :"1000",
+        "worker_batch_size" :"100",
+        "sql_source" :"{{this.identifier}}",
+        "exploded_key": tojson(["result"]) }
+    ),
+    tags = ['streamline_core_evm_history']
+) }}
+
+WITH last_3_days AS (
+
+    SELECT
+        block_number
+    FROM
+        {{ ref("_evm_block_lookback") }}
+),
+to_do AS (
+    SELECT
+        block_number
+    FROM
+        {{ ref("streamline__evm_blocks") }}
+    WHERE
+        (
+            block_number < (
+                SELECT
+                    block_number
+                FROM
+                    last_3_days
+            )
+        )
+        AND block_number IS NOT NULL
+    EXCEPT
+    SELECT
+        block_number
+    FROM
+        {{ ref("streamline__complete_evm_traces") }}
+    WHERE
+        block_number < (
+            SELECT
+                block_number
+            FROM
+                last_3_days
+        )
+),
+ready_blocks AS (
+    SELECT
+        block_number
+    FROM
+        to_do
+)
+SELECT
+    block_number,
+    ROUND(
+        block_number,
+        -3
+    ) :: INT AS partition_key,
+    {{ target.database }}.live.udf_api(
+        'POST',
+        '{Service}/{Authentication}',
+        OBJECT_CONSTRUCT(
+            'Content-Type',
+            'application/json'
+        ),
+        OBJECT_CONSTRUCT(
+            'id',
+            block_number :: STRING,
+            'jsonrpc',
+            '2.0',
+            'method',
+            'debug_traceBlockByNumber',
+            'params',
+            ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), OBJECT_CONSTRUCT('tracer', 'callTracer', 'timeout', '30s'))
+        ),
+        'Vault/prod/sei/quicknode/mainnet'
+    ) AS request
+FROM
+    ready_blocks
+ORDER BY
+    block_number DESC

models/evm/streamline/silver/history/streamline__evm_transactions_history.sql

@@ -0,0 +1,84 @@
+{{ config (
+    materialized = "view",
+    post_hook = fsc_utils.if_data_call_function_v2(
+        func = 'streamline.udf_bulk_rest_api_v2',
+        target = "{{this.schema}}.{{this.identifier}}",
+        params ={ "external_table" :"evm_transactions",
+        "sql_limit" :"25000",
+        "producer_batch_size" :"100000",
+        "worker_batch_size" :"10000",
+        "sql_source" :"{{this.identifier}}",
+        "exploded_key": tojson(["result.transactions"]) }
+    ),
+    tags = ['streamline_core_evm_history']
+) }}
+
+WITH last_3_days AS (
+
+    SELECT
+        block_number
+    FROM
+        {{ ref("_evm_block_lookback") }}
+),
+to_do AS (
+    SELECT
+        block_number
+    FROM
+        {{ ref("streamline__evm_blocks") }}
+    WHERE
+        (
+            block_number < (
+                SELECT
+                    block_number
+                FROM
+                    last_3_days
+            )
+        )
+        AND block_number IS NOT NULL
+    EXCEPT
+    SELECT
+        block_number
+    FROM
+        {{ ref("streamline__complete_evm_transactions") }}
+    WHERE
+        block_number < (
+            SELECT
+                block_number
+            FROM
+                last_3_days
+        )
+),
+ready_blocks AS (
+    SELECT
+        block_number
+    FROM
+        to_do
+)
+SELECT
+    block_number,
+    ROUND(
+        block_number,
+        -3
+    ) :: INT AS partition_key,
+    {{ target.database }}.live.udf_api(
+        'POST',
+        '{Service}/{Authentication}',
+        OBJECT_CONSTRUCT(
+            'Content-Type',
+            'application/json'
+        ),
+        OBJECT_CONSTRUCT(
+            'id',
+            block_number :: STRING,
+            'jsonrpc',
+            '2.0',
+            'method',
+            'eth_getBlockByNumber',
+            'params',
+            ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), TRUE)),
+            'Vault/prod/sei/quicknode/mainnet'
+        ) AS request
+        FROM
+            ready_blocks
+        ORDER BY
+            block_number DESC

models/evm/streamline/silver/realtime/streamline__evm_confirm_blocks_realtime.sql

@@ -13,15 +13,11 @@
 ) }}
 
 WITH last_3_days AS (
-    {#
 
     SELECT
         block_number
     FROM
         {{ ref("_evm_block_lookback") }}
-        #}
-    SELECT
-        79123881 AS block_number
 ),
 look_back AS (
     SELECT

models/evm/streamline/silver/realtime/streamline__evm_receipts_realtime.sql

@@ -63,6 +63,11 @@ ready_blocks AS (
         block_number
     FROM
         {{ ref("_missing_receipts") }}
+    UNION
+    SELECT
+        block_number
+    FROM
+        {{ ref("_unconfirmed_blocks") }}
 )
 SELECT
     block_number,

models/evm/streamline/silver/realtime/streamline__evm_traces_realtime.sql

@@ -63,6 +63,11 @@ ready_blocks AS (
         block_number
     FROM
         {{ ref("_missing_traces") }}
+    UNION
+    SELECT
+        block_number
+    FROM
+        {{ ref("_unconfirmed_blocks") }}
 )
 SELECT
     block_number,

models/evm/streamline/silver/realtime/streamline__evm_transactions_realtime.sql

@@ -63,6 +63,11 @@ ready_blocks AS (
         block_number
     FROM
         {{ ref("_missing_receipts") }}
+    UNION
+    SELECT
+        block_number
+    FROM
+        {{ ref("_unconfirmed_blocks") }}
 )
 SELECT
     block_number,

models/evm/streamline/silver/retry/_unconfirmed_blocks.sql

@@ -0,0 +1,34 @@
+{{ config (
+    materialized = "ephemeral"
+) }}
+
+WITH lookback AS (
+
+    SELECT
+        block_number
+    FROM
+        {{ ref("_evm_block_lookback") }}
+)
+SELECT
+    DISTINCT cb.block_number AS block_number
+FROM
+    {{ ref("silver_evm__confirmed_blocks") }}
+    cb
+    LEFT JOIN {{ ref("silver_evm__transactions") }}
+    txs USING (
+        block_number,
+        block_hash,
+        tx_hash
+    )
+WHERE
+    txs.tx_hash IS NULL
+    AND cb.block_number >= (
+        SELECT
+            block_number
+        FROM
+            lookback
+    )
+    AND cb._inserted_timestamp >= DATEADD('hour', -84, SYSDATE())
+    AND (
+        txs._inserted_timestamp >= DATEADD('hour', -84, SYSDATE())
+        OR txs._inserted_timestamp IS NULL)

tests/sei_evm/test_silver_evm__confirmed_blocks.sql

@@ -0,0 +1 @@
+{{ missing_confirmed_txs(ref("test_silver_evm__confirmed_blocks_full"), ref("test_silver_evm__transactions_full")) }}

tests/sei_evm/test_silver_evm__missing_receipts.sql

@@ -0,0 +1,2 @@
+-- depends_on: {{ ref('test_silver_evm__transactions_full') }}
+{{ missing_txs(ref("test_silver_evm__receipts_full")) }}

tests/sei_evm/test_silver_evm__missing_traces.sql

@@ -0,0 +1,2 @@
+-- depends_on: {{ ref('test_silver_evm__transactions_full') }}
+{{ missing_txs(ref("test_silver_evm__traces_full")) }}

tests/sei_evm/test_silver_evm__recent_confirmed_blocks.sql

@@ -0,0 +1 @@
+{{ missing_confirmed_txs(ref("test_silver_evm__confirmed_blocks_recent"), ref("test_silver_evm__transactions_recent")) }}