From 6266dfed2663b21575e96c4b8ff2562943c8f922 Mon Sep 17 00:00:00 2001 From: Austin <93135983+austinFlipside@users.noreply.github.com> Date: Fri, 1 Nov 2024 14:25:49 -0400 Subject: [PATCH] lower producer sizes (#64) * lower producer sizes * limits * 100K -> 25K --------- Co-authored-by: Eric Laurello --- .../silver/core/realtime/streamline__evm_blocks_realtime.sql | 4 ++-- .../core/realtime/streamline__evm_receipts_realtime.sql | 2 +- .../core/realtime/streamline__evm_transactions_realtime.sql | 4 ++-- .../streamline/core/realtime/streamline__blocks_realtime.sql | 2 +- .../core/realtime/streamline__transactions_realtime.sql | 2 +- .../core/realtime/streamline__tx_counts_realtime.sql | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/models/evm/streamline/silver/core/realtime/streamline__evm_blocks_realtime.sql b/models/evm/streamline/silver/core/realtime/streamline__evm_blocks_realtime.sql index d410d5b..e5511cd 100644 --- a/models/evm/streamline/silver/core/realtime/streamline__evm_blocks_realtime.sql +++ b/models/evm/streamline/silver/core/realtime/streamline__evm_blocks_realtime.sql @@ -5,8 +5,8 @@ target = "{{this.schema}}.{{this.identifier}}", params ={ "external_table" :"evm_blocks", "sql_limit" :"25000", - "producer_batch_size" :"100000", - "worker_batch_size" :"10000", + "producer_batch_size" :"25000", + "worker_batch_size" :"15000", "sql_source" :"{{this.identifier}}" } ), tags = ['streamline_core_evm_realtime'] diff --git a/models/evm/streamline/silver/core/realtime/streamline__evm_receipts_realtime.sql b/models/evm/streamline/silver/core/realtime/streamline__evm_receipts_realtime.sql index 8836583..97e8a8f 100644 --- a/models/evm/streamline/silver/core/realtime/streamline__evm_receipts_realtime.sql +++ b/models/evm/streamline/silver/core/realtime/streamline__evm_receipts_realtime.sql @@ -5,7 +5,7 @@ target = "{{this.schema}}.{{this.identifier}}", params ={ "external_table" :"evm_receipts", "sql_limit" :"25000", - "producer_batch_size" :"100000", + "producer_batch_size" :"25000", "worker_batch_size" :"10000", "sql_source" :"{{this.identifier}}", "exploded_key": tojson(["result"]) } diff --git a/models/evm/streamline/silver/core/realtime/streamline__evm_transactions_realtime.sql b/models/evm/streamline/silver/core/realtime/streamline__evm_transactions_realtime.sql index c6d4e39..de165df 100644 --- a/models/evm/streamline/silver/core/realtime/streamline__evm_transactions_realtime.sql +++ b/models/evm/streamline/silver/core/realtime/streamline__evm_transactions_realtime.sql @@ -5,8 +5,8 @@ target = "{{this.schema}}.{{this.identifier}}", params ={ "external_table" :"evm_transactions", "sql_limit" :"25000", - "producer_batch_size" :"100000", - "worker_batch_size" :"10000", + "producer_batch_size" :"25000", + "worker_batch_size" :"15000", "sql_source" :"{{this.identifier}}", "exploded_key": tojson(["result.transactions"]) } ), diff --git a/models/streamline/core/realtime/streamline__blocks_realtime.sql b/models/streamline/core/realtime/streamline__blocks_realtime.sql index 29c90b8..9c35595 100644 --- a/models/streamline/core/realtime/streamline__blocks_realtime.sql +++ b/models/streamline/core/realtime/streamline__blocks_realtime.sql @@ -4,7 +4,7 @@ func = 'streamline.udf_bulk_rest_api_v2', target = "{{this.schema}}.{{this.identifier}}", params ={ "external_table" :"blocks_v2", - "sql_limit" :"100000", + "sql_limit" :"25000", "producer_batch_size" :"2000", "worker_batch_size" :"500", "sql_source" :"{{this.identifier}}" } diff --git a/models/streamline/core/realtime/streamline__transactions_realtime.sql b/models/streamline/core/realtime/streamline__transactions_realtime.sql index 306d835..3e72dbf 100644 --- a/models/streamline/core/realtime/streamline__transactions_realtime.sql +++ b/models/streamline/core/realtime/streamline__transactions_realtime.sql @@ -4,7 +4,7 @@ func = 'streamline.udf_bulk_rest_api_v2', target = "{{this.schema}}.{{this.identifier}}", params ={ "external_table" :"transactions_v2", - "sql_limit" :"100000", + "sql_limit" :"25000", "producer_batch_size" :"400", "worker_batch_size" :"100", "exploded_key": tojson(["result.txs"]), diff --git a/models/streamline/core/realtime/streamline__tx_counts_realtime.sql b/models/streamline/core/realtime/streamline__tx_counts_realtime.sql index 81e6e0d..d6ccde3 100644 --- a/models/streamline/core/realtime/streamline__tx_counts_realtime.sql +++ b/models/streamline/core/realtime/streamline__tx_counts_realtime.sql @@ -4,7 +4,7 @@ func = 'streamline.udf_bulk_rest_api_v2', target = "{{this.schema}}.{{this.identifier}}", params ={ "external_table" :"txcount_v2", - "sql_limit" :"100000", + "sql_limit" :"25000", "producer_batch_size" :"2000", "worker_batch_size" :"500", "sql_source" :"{{this.identifier}}" }