batch tx_results success

models/silver/streamline/bronze/core/history/bronze__streamline_collections_history.sql

@@ -2,12 +2,21 @@
     materialized = 'view'
 ) }}
 
+{#
 {% set 
     table_names = 
     [
         'COLLECTIONS_CANDIDATE_07', 'COLLECTIONS_CANDIDATE_08', 'COLLECTIONS_CANDIDATE_09', 'COLLECTIONS_MAINNET_01', 'COLLECTIONS_MAINNET_02', 'COLLECTIONS_MAINNET_03', 'COLLECTIONS_MAINNET_04', 'COLLECTIONS_MAINNET_05', 'COLLECTIONS_MAINNET_06', 'COLLECTIONS_MAINNET_07', 'COLLECTIONS_MAINNET_08', 'COLLECTIONS_MAINNET_09', 'COLLECTIONS_MAINNET_10', 'COLLECTIONS_MAINNET_11', 'COLLECTIONS_MAINNET_12', 'COLLECTIONS_MAINNET_13', 'COLLECTIONS_MAINNET_14', 'COLLECTIONS_MAINNET_15', 'COLLECTIONS_MAINNET_16', 'COLLECTIONS_MAINNET_17', 'COLLECTIONS_MAINNET_18', 'COLLECTIONS_MAINNET_19', 'COLLECTIONS_MAINNET_20', 'COLLECTIONS_MAINNET_21', 'COLLECTIONS_MAINNET_22'
     ]
 %}
+#}
+
+{% set 
+    table_names = 
+    [
+        'COLLECTIONS_MAINNET_18'
+    ]
+%}
 
 {{ streamline_multiple_external_table_query(
     table_names,

models/silver/streamline/bronze/core/history/bronze__streamline_transaction_results_history.sql

@@ -23,7 +23,7 @@
 {% set 
     table_names = 
     [
-        'TRANSACTION_RESULTS_MAINNET_14', 'TRANSACTION_RESULTS_MAINNET_15', 'TRANSACTION_RESULTS_MAINNET_16', 'TRANSACTION_RESULTS_MAINNET_17', 'TRANSACTION_RESULTS_MAINNET_18', 'TRANSACTION_RESULTS_MAINNET_19', 'TRANSACTION_RESULTS_MAINNET_22'
+        'TRANSACTION_RESULTS_MAINNET_18'
     ]
 %}
 

models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_18.sql

@@ -0,0 +1,66 @@
+{{ config (
+    materialized = "view",
+    post_hook = fsc_utils.if_data_call_function_v2(
+        func = 'udf_bulk_grpc_us_east_2',
+        target = "{{this.schema}}.{{this.identifier}}",
+        params = {
+            "node_url":"access-001.mainnet18.nodes.onflow.org:9000",
+            "external_table": "transaction_results_batch_mainnet_18",
+            "sql_limit": "100",
+            "producer_batch_size": "100",
+            "worker_batch_size": "100",
+            "sql_source": "{{this.identifier}}"
+        }
+    )
+) }}
+
+WITH blocks AS (
+-- CTE to identify blocks that doesn't have tx_results ingested for mainnet 18
+    SELECT
+        block_height
+    FROM
+        {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 31735955
+        AND 35858810
+    EXCEPT
+    SELECT
+        distinct block_number AS block_height
+    FROM
+        {{ ref("streamline__complete_get_transaction_results_history") }}
+    WHERE
+        block_number BETWEEN 31735955 
+        AND 35858810
+),
+block_ids AS (
+-- CTE to generate the block_ids for the missing block transactions
+    SELECT
+        data:id::STRING as block_id,
+        block_number
+    FROM
+        {{ ref("streamline__complete_get_blocks_history")}}
+    WHERE
+        block_number BETWEEN 31735955
+        AND 35858810
+)
+-- Generate the requests based on the missing block transactions
+SELECT
+    OBJECT_CONSTRUCT(
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_results_by_block_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'block_id',
+            block_id
+        )
+    ) AS request
+FROM
+    blocks
+JOIN
+    block_ids on blocks.block_height = block_ids.block_number 
+ORDER BY
+    block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql

@@ -1,8 +1,16 @@
 {{ config (
     materialized = "view",
-    post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet18.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_18', 'sql_limit', '500000', 'producer_batch_size', '1250', 'worker_batch_size', '1', 'batch_call_limit', {{var('batch_call_limit','1')}}))",
-        target = "{{this.schema}}.{{this.identifier}}"
+    post_hook = fsc_utils.if_data_call_function_v2(
+        func = 'udf_bulk_grpc_us_east_2',
+        target = "{{this.schema}}.{{this.identifier}}",
+        params = {
+            "node_url":"access-001.mainnet18.nodes.onflow.org:9000",
+            "external_table": "transaction_results_mainnet_18",
+            "sql_limit": "25000",
+            "producer_batch_size": "1000",
+            "worker_batch_size": "200",
+            "sql_source": "{{this.identifier}}"
+        }
     )
 ) }}
 

package-lock.yml

@@ -1,16 +1,16 @@
 packages:
-  - package: calogica/dbt_expectations
-    version: 0.8.0
-  - package: dbt-labs/dbt_external_tables
-    version: 0.8.0
-  - package: dbt-labs/dbt_utils
-    version: 1.0.0
-  - git: https://github.com/FlipsideCrypto/fsc-utils.git
-    revision: 3a45311f3e94c8a695443f77aab433a05b894f90
-  - package: get-select/dbt_snowflake_query_tags
-    version: 2.3.2
-  - package: calogica/dbt_date
-    version: 0.7.2
-  - git: https://github.com/FlipsideCrypto/livequery-models.git
-    revision: bca494102fbd2d621d32746e9a7fe780678044f8
-sha1_hash: 82abbdb1baa6b6b88e7d0e40f3ee8aaf2850e052
+- package: calogica/dbt_expectations
+  version: 0.8.0
+- package: dbt-labs/dbt_external_tables
+  version: 0.8.0
+- package: dbt-labs/dbt_utils
+  version: 1.0.0
+- git: https://github.com/FlipsideCrypto/fsc-utils.git
+  revision: ec7703e90ad70dfd997abe57717a07400aae2f17
+- package: get-select/dbt_snowflake_query_tags
+  version: 2.3.3
+- package: calogica/dbt_date
+  version: 0.7.2
+- git: https://github.com/FlipsideCrypto/livequery-models.git
+  revision: bca494102fbd2d621d32746e9a7fe780678044f8
+sha1_hash: bb079c5efcbe9e6e5f17498d0c787df276e58f67

packages.yml

@@ -6,6 +6,6 @@ packages:
   - package: dbt-labs/dbt_utils
     version: 1.0.0
   - git: https://github.com/FlipsideCrypto/fsc-utils.git
-    revision: "v1.9.3"
+    revision: "v1.17.2"
   - package: get-select/dbt_snowflake_query_tags
     version: [">=2.0.0", "<3.0.0"]