From 0d3128f868672d4ea6514ba6cd329ca1aba87716 Mon Sep 17 00:00:00 2001 From: Shah Newaz Khan Date: Mon, 4 Mar 2024 13:33:25 -0800 Subject: [PATCH] Stream 798/tx batch tuning (#280) * adds tx result batch make directive | bronze make directive * batch tx_results success * revert collections bronze scope * revert tx results history bronze --------- Co-authored-by: shah --- Makefile | 17 ++++- ...transaction_results_history_mainnet_18.sql | 66 +++++++++++++++++++ ...transaction_results_history_mainnet_18.sql | 15 ++++- package-lock.yml | 30 ++++----- packages.yml | 2 +- 5 files changed, 109 insertions(+), 21 deletions(-) create mode 100644 models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_18.sql diff --git a/Makefile b/Makefile index 6d3192c..fb40cd9 100644 --- a/Makefile +++ b/Makefile @@ -83,7 +83,15 @@ tx_history: tx_results_history: dbt run \ --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \ - -m 1+models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_22.sql \ + -m 1+models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql \ + --profile flow \ + --target $(DBT_TARGET) \ + --profiles-dir ~/.dbt + +tx_results_batch_history: + dbt run \ + --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \ + -m 1+models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_18.sql \ --profile flow \ --target $(DBT_TARGET) \ --profiles-dir ~/.dbt @@ -96,4 +104,9 @@ lq_overloads: --profiles-dir ~/.dbt \ --vars '{"UPDATE_EPHEMERAL_UDFS":True}' - +bronze: + dbt run \ + -s bronze__streamline_transaction_results_history \ + --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \ + --profiles-dir ~/.dbt \ + --target $(DBT_TARGET) diff --git a/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_18.sql b/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_18.sql new file mode 100644 index 0000000..718cd15 --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_18.sql @@ -0,0 +1,66 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'udf_bulk_grpc_us_east_2', + target = "{{this.schema}}.{{this.identifier}}", + params = { + "node_url":"access-001.mainnet18.nodes.onflow.org:9000", + "external_table": "transaction_results_batch_mainnet_18", + "sql_limit": "100", + "producer_batch_size": "100", + "worker_batch_size": "100", + "sql_source": "{{this.identifier}}" + } + ) +) }} + +WITH blocks AS ( +-- CTE to identify blocks that doesn't have tx_results ingested for mainnet 18 + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 31735955 + AND 35858810 + EXCEPT + SELECT + distinct block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results_history") }} + WHERE + block_number BETWEEN 31735955 + AND 35858810 +), +block_ids AS ( +-- CTE to generate the block_ids for the missing block transactions + SELECT + data:id::STRING as block_id, + block_number + FROM + {{ ref("streamline__complete_get_blocks_history")}} + WHERE + block_number BETWEEN 31735955 + AND 35858810 +) +-- Generate the requests based on the missing block transactions +SELECT + OBJECT_CONSTRUCT( + 'grpc', + 'proto3', + 'method', + 'get_transaction_results_by_block_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'block_id', + block_id + ) + ) AS request +FROM + blocks +JOIN + block_ids on blocks.block_height = block_ids.block_number +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql index f45e519..3a2e2e0 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql @@ -1,8 +1,17 @@ {{ config ( materialized = "view", - post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet18.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_18', 'sql_limit', '6000000', 'producer_batch_size','1000', 'worker_batch_size','2', 'batch_call_limit', {{var('batch_call_limit','1')}}))", - target = "{{this.schema}}.{{this.identifier}}" + + post_hook = fsc_utils.if_data_call_function_v2( + func = 'udf_bulk_grpc_us_east_2', + target = "{{this.schema}}.{{this.identifier}}", + params = { + "node_url":"access-001.mainnet18.nodes.onflow.org:9000", + "external_table": "transaction_results_mainnet_18", + "sql_limit": "25000", + "producer_batch_size": "1000", + "worker_batch_size": "200", + "sql_source": "{{this.identifier}}" + } ) ) }} diff --git a/package-lock.yml b/package-lock.yml index 34171f2..2de10b9 100644 --- a/package-lock.yml +++ b/package-lock.yml @@ -1,16 +1,16 @@ packages: - - package: calogica/dbt_expectations - version: 0.8.0 - - package: dbt-labs/dbt_external_tables - version: 0.8.0 - - package: dbt-labs/dbt_utils - version: 1.0.0 - - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: 3a45311f3e94c8a695443f77aab433a05b894f90 - - package: get-select/dbt_snowflake_query_tags - version: 2.3.2 - - package: calogica/dbt_date - version: 0.7.2 - - git: https://github.com/FlipsideCrypto/livequery-models.git - revision: bca494102fbd2d621d32746e9a7fe780678044f8 -sha1_hash: 82abbdb1baa6b6b88e7d0e40f3ee8aaf2850e052 +- package: calogica/dbt_expectations + version: 0.8.0 +- package: dbt-labs/dbt_external_tables + version: 0.8.0 +- package: dbt-labs/dbt_utils + version: 1.0.0 +- git: https://github.com/FlipsideCrypto/fsc-utils.git + revision: ec7703e90ad70dfd997abe57717a07400aae2f17 +- package: get-select/dbt_snowflake_query_tags + version: 2.3.3 +- package: calogica/dbt_date + version: 0.7.2 +- git: https://github.com/FlipsideCrypto/livequery-models.git + revision: bca494102fbd2d621d32746e9a7fe780678044f8 +sha1_hash: bb079c5efcbe9e6e5f17498d0c787df276e58f67 diff --git a/packages.yml b/packages.yml index 97f06d3..5c945f5 100644 --- a/packages.yml +++ b/packages.yml @@ -6,6 +6,6 @@ packages: - package: dbt-labs/dbt_utils version: 1.0.0 - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: "v1.9.3" + revision: "v1.17.2" - package: get-select/dbt_snowflake_query_tags version: [">=2.0.0", "<3.0.0"]