From ec3ded7f81511ea483056d0119c59faff0a1a203 Mon Sep 17 00:00:00 2001 From: tarikceric Date: Tue, 19 Nov 2024 15:43:43 -0800 Subject: [PATCH] remove macros, add target db, update sl blocks --- macros/create_udfs.sql | 3 --- macros/get_base_table_udtf.sql | 22 ------------------- macros/streamline/api_integrations.sql | 21 ------------------ .../streamline__blocks_tx_realtime.sql | 19 +++++----------- ...streamline__transaction_batch_realtime.sql | 8 +++---- models/streamline/core/streamline__blocks.sql | 5 ++++- .../streamline/core/streamline__chainhead.sql | 2 +- 7 files changed, 13 insertions(+), 67 deletions(-) delete mode 100644 macros/get_base_table_udtf.sql delete mode 100644 macros/streamline/api_integrations.sql diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql index 99134c5..5f0688b 100644 --- a/macros/create_udfs.sql +++ b/macros/create_udfs.sql @@ -2,9 +2,6 @@ {% if var("UPDATE_UDFS_AND_SPS") %} {% set sql %} CREATE schema if NOT EXISTS silver; -{{ create_udtf_get_base_table( - schema = "streamline" - ) }} {{ create_udf_bulk_rest_api_v2() }} {% endset %} {% do run_query(sql) %} diff --git a/macros/get_base_table_udtf.sql b/macros/get_base_table_udtf.sql deleted file mode 100644 index f19ea69..0000000 --- a/macros/get_base_table_udtf.sql +++ /dev/null @@ -1,22 +0,0 @@ -{% macro create_udtf_get_base_table(schema) %} - CREATE - OR REPLACE FUNCTION {{ schema }}.udtf_get_base_table( - max_height INTEGER - ) returns TABLE ( - height NUMBER - ) AS $$ WITH base AS ( - SELECT - ROW_NUMBER() over ( - ORDER BY - SEQ4() - ) AS id - FROM - TABLE(GENERATOR(rowcount => 100000000)) - ) -SELECT - id AS height -FROM - base -WHERE - id <= max_height $$; -{% endmacro %} \ No newline at end of file diff --git a/macros/streamline/api_integrations.sql b/macros/streamline/api_integrations.sql deleted file mode 100644 index 44d057c..0000000 --- a/macros/streamline/api_integrations.sql +++ /dev/null @@ -1,21 +0,0 @@ -{% macro create_aws_aptos_api() %} - {{ log( - "Creating integration for target:" ~ target - ) }} - - {% if target.name == "prod" %} - {% set sql %} - CREATE api integration IF NOT EXISTS aws_aptos_api api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::490041342817:role/aptos-api-prod-rolesnowflakeudfsAF733095-wEyotLQyFEIl' api_allowed_prefixes = ( - 'https://dedvhh9fi1.execute-api.us-east-1.amazonaws.com/prod/' - ) enabled = TRUE; -{% endset %} - {% do run_query(sql) %} - {% elif target.name == "dev" %} - {% set sql %} - CREATE api integration IF NOT EXISTS aws_aptos_api_dev api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::704693948482:role/aptos-api-stg-rolesnowflakeudfsAF733095-k23uBmqxZRsN' api_allowed_prefixes = ( - 'https://9v6g64rv1e.execute-api.us-east-1.amazonaws.com/stg/' - ) enabled = TRUE; -{% endset %} - {% do run_query(sql) %} - {% endif %} -{% endmacro %} diff --git a/models/streamline/core/realtime/streamline__blocks_tx_realtime.sql b/models/streamline/core/realtime/streamline__blocks_tx_realtime.sql index 53720a5..22a6780 100644 --- a/models/streamline/core/realtime/streamline__blocks_tx_realtime.sql +++ b/models/streamline/core/realtime/streamline__blocks_tx_realtime.sql @@ -4,10 +4,11 @@ func = 'streamline.udf_bulk_rest_api_v2', target = "{{this.schema}}.{{this.identifier}}", params ={ "external_table" :"blocks_tx_v2", - "sql_limit" :"1200000", + "sql_limit" :"10", "producer_batch_size" :"300000", "worker_batch_size" :"50000", - "sql_source" :"{{this.identifier}}" } + "sql_source" :"{{this.identifier}}", + "order_by_column": "block_number" } ), tags = ['streamline_core_realtime'] ) }} @@ -22,13 +23,6 @@ WITH blocks AS ( block_number FROM {{ ref("streamline__complete_blocks_tx") }} - WHERE - block_number < 252143860 - EXCEPT - SELECT - block_number - FROM - {{ ref("streamline__complete_blocks_tx") }} ) SELECT ROUND( @@ -36,7 +30,7 @@ SELECT -3 ) :: INT AS partition_key, block_number, - aptos_dev.live.udf_api( + {{ target.database }}.live.udf_api( 'GET', '{service}/{Authentication}/v1/blocks/by_height/' || block_number || '?with_transactions=true', object_construct( @@ -49,7 +43,4 @@ SELECT 'Vault/prod/aptos/node/mainnet' ) AS request FROM - blocks -ORDER BY - block_number -limit 10 \ No newline at end of file + blocks \ No newline at end of file diff --git a/models/streamline/core/realtime/streamline__transaction_batch_realtime.sql b/models/streamline/core/realtime/streamline__transaction_batch_realtime.sql index c3ae383..3528c07 100644 --- a/models/streamline/core/realtime/streamline__transaction_batch_realtime.sql +++ b/models/streamline/core/realtime/streamline__transaction_batch_realtime.sql @@ -8,7 +8,8 @@ "sql_limit": "1200000", "producer_batch_size": "300000", "worker_batch_size": "50000", - "sql_source": "{{this.identifier}}" } + "sql_source": "{{this.identifier}}", + "order_by_column": "block_number" } ), tags = ['streamline_core_realtime'] ) }} @@ -72,7 +73,7 @@ SELECT block_number, -3 ) :: INT AS partition_key, - aptos_dev.live.udf_api( + {{ target.database }}.live.udf_api( 'GET', '{service}/{Authentication}/v1/transactions?start=' || tx_version || '&limit=100', object_construct( @@ -86,6 +87,3 @@ SELECT ) AS request FROM work -ORDER BY - block_number -limit 10 diff --git a/models/streamline/core/streamline__blocks.sql b/models/streamline/core/streamline__blocks.sql index 4be223a..b613bf0 100644 --- a/models/streamline/core/streamline__blocks.sql +++ b/models/streamline/core/streamline__blocks.sql @@ -16,4 +16,7 @@ WHERE MAX(block_number) FROM {{ ref('streamline__chainhead') }} - ) \ No newline at end of file + ) +UNION ALL +SELECT + 0 AS block_number \ No newline at end of file diff --git a/models/streamline/core/streamline__chainhead.sql b/models/streamline/core/streamline__chainhead.sql index 0e4d8d9..c7afbe3 100644 --- a/models/streamline/core/streamline__chainhead.sql +++ b/models/streamline/core/streamline__chainhead.sql @@ -4,7 +4,7 @@ ) }} SELECT - aptos.live.udf_api( + {{ target.database }}.live.udf_api( 'GET', '{service}/{Authentication}/v1', OBJECT_CONSTRUCT(