From 3ea1836cc395351190b8e63e5d2cf5d248a0dacc Mon Sep 17 00:00:00 2001 From: WHYTEWYLL <49686519+WHYTEWYLL@users.noreply.github.com> Date: Thu, 14 Mar 2024 19:23:38 -0600 Subject: [PATCH] v2 to v1 --- macros/streamline/utils.sql | 38 ------------------- ...transaction_results_history_mainnet_17.sql | 18 +++------ ...transaction_results_history_mainnet_18.sql | 18 +++------ ...transaction_results_history_mainnet_19.sql | 19 +++------- ...transaction_results_history_mainnet_22.sql | 18 +++------ ...ine__get_batch_transactions_mainnet_18.sql | 18 +++------ ...ine__get_batch_transactions_mainnet_19.sql | 18 +++------ 7 files changed, 31 insertions(+), 116 deletions(-) diff --git a/macros/streamline/utils.sql b/macros/streamline/utils.sql index 1e6fc56..ac23d7b 100644 --- a/macros/streamline/utils.sql +++ b/macros/streamline/utils.sql @@ -46,42 +46,4 @@ SELECT _utils.UDF_REGISTER_SECRET(REQUEST_ID, _utils.UDF_WHOAMI(), KEY) -{% endmacro %} - - -{% macro if_data_call_function_v2( - func, - target, - params - ) %} - {% if var( - "STREAMLINE_INVOKE_STREAMS" - ) %} - {% if execute %} - {{ log( - "Running macro `if_data_call_function`: Calling udf " ~ func ~ " with params: \n" ~ params | tojson(indent=2) ~ "\n on " ~ target, - True - ) }} - {% endif %} - SELECT - {{ func }}( parse_json($${{ params | tojson }}$$) ) - WHERE - EXISTS( - SELECT - 1 - FROM - {{ target }} - LIMIT - 1 - ) - {% else %} - {% if execute %} - {{ log( - "Running macro `if_data_call_function`: NOOP", - False - ) }} - {% endif %} - SELECT - NULL - {% endif %} {% endmacro %} \ No newline at end of file diff --git a/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_17.sql b/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_17.sql index 90a4268..1cdd411 100644 --- a/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_17.sql +++ b/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_17.sql @@ -1,19 +1,11 @@ {{ config ( materialized = "view", - post_hook = if_data_call_function_v2( - func = 'udf_bulk_grpc', - target = "streamline.{{this.identifier}}", - params = { - "node_url":"access-001.mainnet17.nodes.onflow.org:9000", - "external_table": "transaction_results_mainnet_17", - "sql_limit": "188000", - "producer_batch_size": "14000", - "worker_batch_size": "100", - "sql_source": "{{this.identifier}}", - "concurrent_requests": "800" - } + post_hook = if_data_call_function( + func = "streamline.udf_bulk_grpc(object_construct('node_url','access-001.mainnet17.nodes.onflow.org:9000', 'external_table', 'transaction_results_mainnet_17', 'sql_limit', '188000', 'producer_batch_size', '14000', 'worker_batch_size', '100', 'sql_source', '{{this.identifier}}', 'concurrent_requests', '800'))", + target = "streamline.{{this.identifier}}" + ) ) -) }} +}} WITH blocks AS ( -- CTE to identify blocks that doesn't have tx_results ingested for mainnet 14 diff --git a/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_18.sql b/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_18.sql index d6e0af5..d259024 100644 --- a/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_18.sql +++ b/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_18.sql @@ -1,19 +1,11 @@ {{ config ( materialized = "view", - post_hook = if_data_call_function_v2( - func = 'udf_bulk_grpc', - target = "streamline.{{this.identifier}}", - params = { - "node_url":"access-001.mainnet18.nodes.onflow.org:9000", - "external_table": "transaction_results_mainnet_18", - "sql_limit": "188000", - "producer_batch_size": "14000", - "worker_batch_size": "100", - "sql_source": "{{this.identifier}}", - "concurrent_requests": "770" - } + post_hook = if_data_call_function( + func = "streamline.udf_bulk_grpc(object_construct('node_url','access-001.mainnet18.nodes.onflow.org:9000', 'external_table', 'transaction_results_mainnet_18', 'sql_limit', '188000', 'producer_batch_size', '14000', 'worker_batch_size', '100', 'sql_source', '{{this.identifier}}', 'concurrent_requests', '770'))", + target = "streamline.{{this.identifier}}" + ) ) -) }} +}} WITH blocks AS ( -- CTE to identify blocks that doesn't have tx_results ingested for mainnet 18 diff --git a/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_19.sql b/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_19.sql index 0f6ba66..c28b9b8 100644 --- a/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_19.sql +++ b/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_19.sql @@ -1,19 +1,12 @@ {{ config ( materialized = "view", - post_hook = if_data_call_function_v2( - func = 'udf_bulk_grpc', - target = "streamline.{{this.identifier}}", - params = { - "node_url":"access-001.mainnet19.nodes.onflow.org:9000", - "external_table": "transaction_results_mainnet_19", - "sql_limit": "188000", - "producer_batch_size": "14000", - "worker_batch_size": "100", - "sql_source": "{{this.identifier}}", - "concurrent_requests": "750" - } + post_hook = if_data_call_function( + func = "streamline.udf_bulk_grpc(object_construct('node_url','access-001.mainnet19.nodes.onflow.org:9000', 'external_table', 'transaction_results_mainnet_19', 'sql_limit', '188000', 'producer_batch_size', '14000', 'worker_batch_size', '100', 'sql_source', '{{this.identifier}}', 'concurrent_requests', '750'))", + target = "streamline.{{this.identifier}}" + ) ) -) }} +}} + WITH blocks AS ( -- CTE to identify blocks that doesn't have tx_results ingested for mainnet 18 diff --git a/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_22.sql b/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_22.sql index 1564b53..15696c1 100644 --- a/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_22.sql +++ b/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_22.sql @@ -1,19 +1,11 @@ {{ config ( materialized = "view", - post_hook = if_data_call_function_v2( - func = 'udf_bulk_grpc', - target = "streamline.{{this.identifier}}", - params = { - "node_url":"access-001.mainnet22.nodes.onflow.org:9000", - "external_table": "transaction_results_mainnet_22", - "sql_limit": "188000", - "producer_batch_size": "14000", - "worker_batch_size": "100", - "sql_source": "{{this.identifier}}", - "concurrent_requests": "800" - } + post_hook = if_data_call_function( + func = "streamline.udf_bulk_grpc(object_construct('node_url','access-001.mainnet22.nodes.onflow.org:9000', 'external_table', 'transaction_results_mainnet_22', 'sql_limit', '188000', 'producer_batch_size', '14000', 'worker_batch_size', '100', 'sql_source', '{{this.identifier}}', 'concurrent_requests', '800'))", + target = "streamline.{{this.identifier}}" + ) ) -) }} +}} WITH blocks AS ( -- CTE to identify blocks that doesn't have tx_results ingested for mainnet 18 diff --git a/models/silver/streamline/core/history/transactions/batch/streamline__get_batch_transactions_mainnet_18.sql b/models/silver/streamline/core/history/transactions/batch/streamline__get_batch_transactions_mainnet_18.sql index c869e0a..dcb2c35 100644 --- a/models/silver/streamline/core/history/transactions/batch/streamline__get_batch_transactions_mainnet_18.sql +++ b/models/silver/streamline/core/history/transactions/batch/streamline__get_batch_transactions_mainnet_18.sql @@ -1,19 +1,11 @@ {{ config ( materialized = "view", - post_hook = if_data_call_function_v2( - func = 'udf_bulk_grpc', - target = "streamline.{{this.identifier}}", - params = { - "node_url":"access-001.mainnet18.nodes.onflow.org:9000", - "external_table": "transactions_mainnet_18", - "sql_limit": "188000", - "producer_batch_size": "14000", - "worker_batch_size": "100", - "sql_source": "{{this.identifier}}", - "concurrent_requests": "850" - } + post_hook = if_data_call_function( + func = "streamline.udf_bulk_grpc(object_construct('node_url','access-001.mainnet18.nodes.onflow.org:9000', 'external_table', 'transactions_mainnet_18', 'sql_limit', '188000', 'producer_batch_size', '14000', 'worker_batch_size', '100', 'sql_source', '{{this.identifier}}', 'concurrent_requests', '850'))", + target = "streamline.{{this.identifier}}" + ) ) -) }} +}} WITH collection_transactions AS ( diff --git a/models/silver/streamline/core/history/transactions/batch/streamline__get_batch_transactions_mainnet_19.sql b/models/silver/streamline/core/history/transactions/batch/streamline__get_batch_transactions_mainnet_19.sql index 33a853f..ab51d29 100644 --- a/models/silver/streamline/core/history/transactions/batch/streamline__get_batch_transactions_mainnet_19.sql +++ b/models/silver/streamline/core/history/transactions/batch/streamline__get_batch_transactions_mainnet_19.sql @@ -1,19 +1,11 @@ {{ config ( materialized = "view", - post_hook = if_data_call_function_v2( - func = 'udf_bulk_grpc', - target = "streamline.{{this.identifier}}", - params = { - "node_url":"access-001.mainnet19.nodes.onflow.org:9000", - "external_table": "transactions_mainnet_19", - "sql_limit": "188000", - "producer_batch_size": "14000", - "worker_batch_size": "100", - "sql_source": "{{this.identifier}}", - "concurrent_requests": "800" - } + post_hook = if_data_call_function( + func = "streamline.udf_bulk_grpc(object_construct('node_url','access-001.mainnet19.nodes.onflow.org:9000', 'external_table', 'transactions_mainnet_19', 'sql_limit', '188000', 'producer_batch_size', '14000', 'worker_batch_size', '100', 'sql_source', '{{this.identifier}}', 'concurrent_requests', '800'))", + target = "streamline.{{this.identifier}}" + ) ) -) }} +}} WITH collection_transactions AS (