From ec247316468d88ba2f5f077fc30854a93f8ba119 Mon Sep 17 00:00:00 2001 From: xiuy001 Date: Mon, 27 Nov 2023 15:21:15 -0500 Subject: [PATCH 1/3] updated --- .../realtime/streamline__blocks_realtime.sql | 56 +++++++++++-------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/models/silver/streamline/realtime/streamline__blocks_realtime.sql b/models/silver/streamline/realtime/streamline__blocks_realtime.sql index 555807b..cf427d8 100644 --- a/models/silver/streamline/realtime/streamline__blocks_realtime.sql +++ b/models/silver/streamline/realtime/streamline__blocks_realtime.sql @@ -6,22 +6,24 @@ ) ) }} -WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} +WITH last_3_days AS ( - SELECT - 0 AS block_number - {% else %} SELECT MAX(block_number) - 50000 AS block_number --aprox 3 days FROM {{ ref("streamline__blocks") }} - {% endif %}), - tbl AS ( - SELECT - block_number, - block_number_hex - FROM - {{ ref("streamline__blocks") }} +), +tbl AS ( + SELECT + block_number, + block_number_hex + FROM + {{ ref("streamline__blocks") }} + + {% if var('STREAMLINE_RUN_HISTORY') %} + WHERE + block_number IS NOT NULL + {% else %} WHERE ( block_number >= ( @@ -32,16 +34,22 @@ WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} ) ) AND block_number IS NOT NULL - EXCEPT - SELECT - block_number, - REPLACE( - concat_ws('', '0x', to_char(block_number, 'XXXXXXXX')), - ' ', - '' - ) AS block_number_hex - FROM - {{ ref("streamline__complete_blocks") }} + {% endif %} + EXCEPT + SELECT + block_number, + REPLACE( + concat_ws('', '0x', to_char(block_number, 'XXXXXXXX')), + ' ', + '' + ) AS block_number_hex + FROM + {{ ref("streamline__complete_blocks") }} + + {% if var('STREAMLINE_RUN_HISTORY') %} + WHERE + block_number IS NOT NULL + {% else %} WHERE ( block_number >= ( @@ -52,7 +60,8 @@ WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} ) ) AND block_number IS NOT NULL - ) + {% endif %} +) SELECT block_number, 'eth_getBlockByNumber' AS method, @@ -61,4 +70,5 @@ SELECT '_-_', 'false' ) AS params -FROM tbl \ No newline at end of file +FROM + tbl From f53fed101d62e49ffece30547c0f7df14a904b58 Mon Sep 17 00:00:00 2001 From: xiuy001 Date: Mon, 27 Nov 2023 15:39:56 -0500 Subject: [PATCH 2/3] updated the range of the blocks --- .../realtime/streamline__blocks_realtime.sql | 56 ++++++++----------- .../streamline__transactions_realtime.sql | 2 +- .../streamline__tx_receipts_realtime.sql | 2 +- 3 files changed, 25 insertions(+), 35 deletions(-) diff --git a/models/silver/streamline/realtime/streamline__blocks_realtime.sql b/models/silver/streamline/realtime/streamline__blocks_realtime.sql index cf427d8..349a917 100644 --- a/models/silver/streamline/realtime/streamline__blocks_realtime.sql +++ b/models/silver/streamline/realtime/streamline__blocks_realtime.sql @@ -6,24 +6,22 @@ ) ) }} -WITH last_3_days AS ( +WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} SELECT - MAX(block_number) - 50000 AS block_number --aprox 3 days - FROM - {{ ref("streamline__blocks") }} -), -tbl AS ( + 0 AS block_number + {% else %} SELECT - block_number, - block_number_hex + MAX(block_number) - 500000 AS block_number --aprox 3 days FROM {{ ref("streamline__blocks") }} - - {% if var('STREAMLINE_RUN_HISTORY') %} - WHERE - block_number IS NOT NULL - {% else %} + {% endif %}), + tbl AS ( + SELECT + block_number, + block_number_hex + FROM + {{ ref("streamline__blocks") }} WHERE ( block_number >= ( @@ -34,22 +32,16 @@ tbl AS ( ) ) AND block_number IS NOT NULL - {% endif %} - EXCEPT - SELECT - block_number, - REPLACE( - concat_ws('', '0x', to_char(block_number, 'XXXXXXXX')), - ' ', - '' - ) AS block_number_hex - FROM - {{ ref("streamline__complete_blocks") }} - - {% if var('STREAMLINE_RUN_HISTORY') %} - WHERE - block_number IS NOT NULL - {% else %} + EXCEPT + SELECT + block_number, + REPLACE( + concat_ws('', '0x', to_char(block_number, 'XXXXXXXX')), + ' ', + '' + ) AS block_number_hex + FROM + {{ ref("streamline__complete_blocks") }} WHERE ( block_number >= ( @@ -60,8 +52,7 @@ tbl AS ( ) ) AND block_number IS NOT NULL - {% endif %} -) + ) SELECT block_number, 'eth_getBlockByNumber' AS method, @@ -70,5 +61,4 @@ SELECT '_-_', 'false' ) AS params -FROM - tbl +FROM tbl \ No newline at end of file diff --git a/models/silver/streamline/realtime/streamline__transactions_realtime.sql b/models/silver/streamline/realtime/streamline__transactions_realtime.sql index 0ac17bb..7a59483 100644 --- a/models/silver/streamline/realtime/streamline__transactions_realtime.sql +++ b/models/silver/streamline/realtime/streamline__transactions_realtime.sql @@ -12,7 +12,7 @@ WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} 0 AS block_number {% else %} SELECT - MAX(block_number) - 50000 AS block_number --aprox 3 days + MAX(block_number) - 500000 AS block_number --aprox 3 days FROM {{ ref("streamline__blocks") }} {% endif %}), diff --git a/models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql b/models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql index ae28001..be26800 100644 --- a/models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql +++ b/models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql @@ -12,7 +12,7 @@ WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} 0 AS block_number {% else %} SELECT - MAX(block_number) - 50000 AS block_number -- aprox 3 days + MAX(block_number) - 500000 AS block_number -- aprox 3 days FROM {{ ref("streamline__complete_transactions") }} {% endif %}), From d7216e23fd46d2f6d40fad3479ad5c707a63a311 Mon Sep 17 00:00:00 2001 From: xiuy001 Date: Mon, 27 Nov 2023 15:42:03 -0500 Subject: [PATCH 3/3] optimized the params --- .../silver/streamline/realtime/streamline__blocks_realtime.sql | 2 +- .../streamline/realtime/streamline__transactions_realtime.sql | 2 +- .../streamline/realtime/streamline__tx_receipts_realtime.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/silver/streamline/realtime/streamline__blocks_realtime.sql b/models/silver/streamline/realtime/streamline__blocks_realtime.sql index 349a917..e7c68bd 100644 --- a/models/silver/streamline/realtime/streamline__blocks_realtime.sql +++ b/models/silver/streamline/realtime/streamline__blocks_realtime.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table','blocks', 'producer_batch_size',500000, 'producer_limit_size', 20000000, 'worker_batch_size',5000))", + func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table','blocks', 'producer_batch_size',10000, 'producer_limit_size',2000000, 'worker_batch_size',100))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} diff --git a/models/silver/streamline/realtime/streamline__transactions_realtime.sql b/models/silver/streamline/realtime/streamline__transactions_realtime.sql index 7a59483..09cd7cd 100644 --- a/models/silver/streamline/realtime/streamline__transactions_realtime.sql +++ b/models/silver/streamline/realtime/streamline__transactions_realtime.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'transactions', 'exploded_key','[\"result\", \"transactions\"]', 'producer_batch_size',500000, 'producer_limit_size', 20000000, 'worker_batch_size',5000))", + func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'transactions', 'exploded_key','[\"result\", \"transactions\"]', 'producer_batch_size',10000, 'producer_limit_size',2000000, 'worker_batch_size',100))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} diff --git a/models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql b/models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql index be26800..906a62a 100644 --- a/models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql +++ b/models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'tx_receipts', 'producer_batch_size',200000, 'producer_limit_size', 20000000, 'worker_batch_size',400))", + func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'tx_receipts', 'producer_batch_size',10000, 'producer_limit_size',2000000, 'worker_batch_size',100))", target = "{{this.schema}}.{{this.identifier}}" ) ) }}