Merge branch 'main' into AN-6550-testnet

.github/workflows/dbt_run_streamline_evm_testnet_realtime.yml

@@ -4,8 +4,8 @@ run-name: dbt_run_streamline_evm_testnet_realtime
 on:
   workflow_dispatch:
   schedule:
-    # Every 4 hours
-    - cron: "0 */4 * * *"
+    # Every hour at minute 2
+    - cron: "2 * * * *"
 
 env:
   USE_VARS: "${{ vars.USE_VARS }}"

models/evm/streamline/testnet/history/streamline__get_evm_testnet_blocks_history.sql

@@ -0,0 +1,62 @@
+{{ config (
+    materialized = "view",
+    post_hook = fsc_utils.if_data_call_function_v2(
+        func = 'streamline.udf_bulk_rest_api_v2',
+        target = "{{this.schema}}.{{this.identifier}}",
+        params ={ "external_table" :"evm_testnet_blocks",
+        "sql_limit" :"1000000",
+        "producer_batch_size" :"10000",
+        "worker_batch_size" :"1000",
+        "async_concurrent_requests" :"10",
+        "sql_source" :"{{this.identifier}}" }
+    ),
+    tags = ['streamline_history_evm_testnet']
+) }}
+
+WITH tbl AS (
+    SELECT
+        block_number
+    FROM
+        {{ ref('streamline__evm_testnet_blocks') }}
+    WHERE block_number IS NOT NULL
+    EXCEPT
+    SELECT
+        block_number
+    FROM
+        {{ ref('streamline__complete_get_evm_testnet_blocks') }}
+    WHERE block_number IS NOT NULL
+)
+SELECT
+    block_number,
+    DATE_PART(epoch_second, SYSDATE()) :: STRING AS request_timestamp,
+    '{{ invocation_id }}' AS _invocation_id,
+    ROUND(
+        block_number,
+        -3
+    ) :: INT AS partition_key,
+    {{ target.database }}.live.udf_api(
+        'POST',
+        '{Service}/{Authentication}',
+        OBJECT_CONSTRUCT(
+            'Content-Type', 'application/json',
+            'fsc-quantum-state', 'streamline'
+        ),
+        OBJECT_CONSTRUCT(
+            'id',
+            block_number,
+            'jsonrpc',
+            '2.0',
+            'method',
+            'eth_getBlockByNumber',
+            'params',
+            ARRAY_CONSTRUCT(
+                utils.udf_int_to_hex(block_number),
+                TRUE -- Include transactions
+            )
+        ),
+        'Vault/prod/flow/quicknode/testnet'
+    ) AS request
+FROM
+    tbl
+ORDER BY
+    block_number ASC

models/evm/streamline/testnet/history/streamline__get_evm_testnet_receipts_history.sql

@@ -0,0 +1,68 @@
+{{ config (
+    materialized = "view",
+    post_hook = fsc_utils.if_data_call_function_v2(
+        func = 'streamline.udf_bulk_rest_api_v2',
+        target = "{{this.schema}}.{{this.identifier}}",
+        params ={ "external_table" :"evm_testnet_receipts",
+        "sql_limit" :"1000000",
+        "async_concurrent_requests" :"10",
+        "producer_batch_size" :"10000",
+        "worker_batch_size" :"1000",
+        "sql_source" :"{{this.identifier}}" }
+    ),
+    tags = ['streamline_history_evm_testnet']
+) }}
+
+WITH tbl AS (
+
+    SELECT
+        block_number
+    FROM
+        {{ ref('streamline__evm_testnet_blocks') }}
+    WHERE block_number IS NOT NULL
+    EXCEPT
+    SELECT
+        block_number
+    FROM
+        {{ ref('streamline__complete_get_evm_testnet_receipts') }}
+    WHERE block_number IS NOT NULL
+),
+ready_blocks AS (
+    SELECT
+        block_number
+    FROM
+        tbl
+)
+SELECT
+    block_number,
+    DATE_PART(epoch_second, SYSDATE())::STRING AS request_timestamp,
+    '{{ invocation_id }}' AS _invocation_id,
+    ROUND(
+        block_number,
+        -3
+    ) :: INT AS partition_key,
+    {{ target.database }}.live.udf_api(
+        'POST',
+        '{Service}/{Authentication}',
+        OBJECT_CONSTRUCT(
+            'Content-Type', 'application/json',
+            'fsc-quantum-state', 'streamline'
+        ),
+        OBJECT_CONSTRUCT(
+            'id',
+            block_number,
+            'jsonrpc',
+            '2.0',
+            'method',
+            'eth_getBlockReceipts',
+            'params',
+            ARRAY_CONSTRUCT(
+                utils.udf_int_to_hex(block_number)
+            )
+        ),
+        'Vault/prod/flow/quicknode/testnet'
+    ) AS request
+FROM
+    ready_blocks
+ORDER BY
+    block_number ASC

models/evm/streamline/testnet/history/streamline__get_evm_testnet_traces_history.sql

@@ -0,0 +1,76 @@
+{{ config (
+    materialized = "view",
+    post_hook = fsc_utils.if_data_call_function_v2(
+        func = 'streamline.udf_bulk_rest_api_v2',
+        target = "{{this.schema}}.{{this.identifier}}",
+        params ={ "external_table" :"evm_testnet_traces",
+        "sql_limit" :"1000000",
+        "async_concurrent_requests" :"10",
+        "producer_batch_size" :"10000",
+        "worker_batch_size" :"1000",
+        "sql_source" :"{{this.identifier}}",
+        "exploded_key": tojson(["result"])}
+    ),
+    tags = ['streamline_history_evm_testnet']
+) }}
+
+
+WITH tbl AS (
+
+    SELECT
+        block_number
+    FROM
+        {{ ref('streamline__evm_testnet_blocks') }}
+    WHERE
+        block_number IS NOT NULL
+    EXCEPT
+    SELECT
+        block_number
+    FROM
+        {{ ref('streamline__complete_get_evm_testnet_traces') }}
+    WHERE
+        block_number IS NOT NULL
+),
+ready_blocks AS (
+    SELECT
+        block_number
+    FROM
+        tbl
+)
+SELECT
+    block_number,
+    DATE_PART(epoch_second, SYSDATE())::STRING AS request_timestamp,
+    '{{ invocation_id }}' AS _invocation_id,
+    ROUND(
+        block_number,
+        -3
+    ) :: INT AS partition_key,
+    {{ target.database }}.live.udf_api(
+        'POST',
+        '{Service}/{Authentication}',
+        OBJECT_CONSTRUCT(
+            'Content-Type', 'application/json',
+            'fsc-quantum-state', 'streamline'
+        ),
+        OBJECT_CONSTRUCT(
+            'id',
+            block_number,
+            'jsonrpc',
+            '2.0',
+            'method',
+            'debug_traceBlockByNumber',
+            'params',
+            ARRAY_CONSTRUCT(
+                utils.udf_int_to_hex(block_number),
+                OBJECT_CONSTRUCT(
+                    'tracer', 'callTracer', 
+                    'timeout', '180s'
+                )
+            )
+        ),
+        'Vault/prod/flow/quicknode/testnet'
+    ) AS request
+FROM
+    ready_blocks
+ORDER BY
+    block_number ASC    

models/evm/streamline/testnet/realtime/streamline__get_evm_testnet_blocks_realtime.sql

@@ -65,8 +65,8 @@ SELECT
         'POST',
         '{Service}/{Authentication}',
         OBJECT_CONSTRUCT(
-            'Content-Type',
-            'application/json'
+            'Content-Type', 'application/json',
+            'fsc-quantum-state', 'streamline'
         ),
         OBJECT_CONSTRUCT(
             'id',

models/evm/streamline/testnet/realtime/streamline__get_evm_testnet_receipts_realtime.sql

@@ -76,8 +76,8 @@ SELECT
         'POST',
         '{Service}/{Authentication}',
         OBJECT_CONSTRUCT(
-            'Content-Type',
-            'application/json'
+            'Content-Type', 'application/json',
+            'fsc-quantum-state', 'streamline'
         ),
         OBJECT_CONSTRUCT(
             'id',

models/evm/streamline/testnet/realtime/streamline__get_evm_testnet_traces_realtime.sql

@@ -73,8 +73,8 @@ SELECT
         'POST',
         '{Service}/{Authentication}',
         OBJECT_CONSTRUCT(
-            'Content-Type',
-            'application/json'
+            'Content-Type', 'application/json',
+            'fsc-quantum-state', 'streamline'
         ),
         OBJECT_CONSTRUCT(
             'id',

models/evm/streamline/testnet/streamline__evm_testnet_blocks.sql

@@ -11,4 +11,5 @@ FROM
         'number_sequence'
     ) }}
 WHERE
-    _id <= (SELECT block_number FROM {{ ref('streamline__evm_testnet_chainhead') }})
+    _id <= (SELECT block_number FROM {{ ref('streamline__evm_testnet_chainhead') }})
+AND _id >= 67860000