From ce6c632dc18d9d03e2ff06f0a2ba39fc3e0abdd3 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Mon, 3 Jun 2024 10:29:49 -0400 Subject: [PATCH] upgrade keys change batche sizes --- macros/snowflake_api/get_blockchain_api.sql | 4 ++-- .../core/realtime/streamline__balances_realtime.sql | 6 +++--- .../silver/core/realtime/streamline__blocks_realtime.sql | 8 ++++---- .../core/realtime/streamline__pool_balances_realtime.sql | 8 ++++++-- .../core/realtime/streamline__transactions_realtime.sql | 4 ++-- .../core/realtime/streamline__tx_counts_realtime.sql | 8 ++++---- models/streamline/silver/core/streamline__chainhead.sql | 4 ++-- 7 files changed, 23 insertions(+), 19 deletions(-) diff --git a/macros/snowflake_api/get_blockchain_api.sql b/macros/snowflake_api/get_blockchain_api.sql index 6102dff..8e277c7 100644 --- a/macros/snowflake_api/get_blockchain_api.sql +++ b/macros/snowflake_api/get_blockchain_api.sql @@ -108,13 +108,13 @@ SELECT call, {{ target.database }}.live.udf_api ( 'POST', - '{service}/{Authentication}', + '{Service}/{Authentication}', OBJECT_CONSTRUCT( 'Content-Type', 'application/json' ), call, - 'Vault/prod/osmosis/allthatnode/mainnet-archive/rpc' + 'Vault/prod/osmosis/atn/mainnet' ) AS DATA, SYSDATE() FROM diff --git a/models/streamline/silver/core/realtime/streamline__balances_realtime.sql b/models/streamline/silver/core/realtime/streamline__balances_realtime.sql index 49ce49a..9ca13c3 100644 --- a/models/streamline/silver/core/realtime/streamline__balances_realtime.sql +++ b/models/streamline/silver/core/realtime/streamline__balances_realtime.sql @@ -5,7 +5,7 @@ target = "{{this.schema}}.{{this.identifier}}", params ={ "external_table" :"balances_v2", "sql_limit" :"100000", - "producer_batch_size" :"400", + "producer_batch_size" :"200", "worker_batch_size" :"100", "sql_source" :"{{this.identifier}}" } ) @@ -32,7 +32,7 @@ SELECT ) AS partition_key, {{ target.database }}.live.udf_api( 'GET', - '{service}/{Authentication}/cosmos/bank/v1beta1/balances/' || address || '?pagination.limit=10000', + 'https://osmosis-mainnet.g.allthatnode.com/archive/rest/{Authentication}/cosmos/bank/v1beta1/balances/' || address || '?pagination.limit=10000', OBJECT_CONSTRUCT( 'Content-Type', 'application/json', @@ -40,7 +40,7 @@ SELECT block_number :: STRING ), PARSE_JSON('{}'), - 'vault/prod/osmosis/allthatnode/mainnet-archive/rest' + 'vault/prod/osmosis/atn/mainnet' ) AS request, block_number, address diff --git a/models/streamline/silver/core/realtime/streamline__blocks_realtime.sql b/models/streamline/silver/core/realtime/streamline__blocks_realtime.sql index d662a9b..b41f46f 100644 --- a/models/streamline/silver/core/realtime/streamline__blocks_realtime.sql +++ b/models/streamline/silver/core/realtime/streamline__blocks_realtime.sql @@ -5,8 +5,8 @@ target = "{{this.schema}}.{{this.identifier}}", params ={ "external_table" :"blocks_v2", "sql_limit" :"100000", - "producer_batch_size" :"1000", - "worker_batch_size" :"500", + "producer_batch_size" :"500", + "worker_batch_size" :"250", "sql_source" :"{{this.identifier}}" } ) ) }} @@ -30,7 +30,7 @@ SELECT ) :: INT AS partition_key, {{ target.database }}.live.udf_api( 'POST', - '{service}/{Authentication}', + '{Service}/{Authentication}', OBJECT_CONSTRUCT( 'Content-Type', 'application/json' @@ -47,7 +47,7 @@ SELECT block_number :: STRING ) ), - 'vault/prod/osmosis/allthatnode/mainnet-archive/rpc' + 'vault/prod/osmosis/atn/mainnet' ) AS request FROM blocks diff --git a/models/streamline/silver/core/realtime/streamline__pool_balances_realtime.sql b/models/streamline/silver/core/realtime/streamline__pool_balances_realtime.sql index 27e07ef..a1b5b22 100644 --- a/models/streamline/silver/core/realtime/streamline__pool_balances_realtime.sql +++ b/models/streamline/silver/core/realtime/streamline__pool_balances_realtime.sql @@ -45,7 +45,11 @@ SELECT ) AS partition_key, {{ target.database }}.live.udf_api( 'GET', - '{service}/{Authentication}/osmosis/gamm/v1beta1/pools?pagination.limit=10000', + REPLACE( + 'https://osmosis-mainnet.g.allthatnode.com/archive/rest/{Authentication}/osmosis/gamm/v1beta1/pools?pagination.limit=10000', + 'tendermint', + 'rest' + ), OBJECT_CONSTRUCT( 'Content-Type', 'application/json', @@ -53,7 +57,7 @@ SELECT block_number :: STRING ), PARSE_JSON('{}'), - 'vault/prod/osmosis/allthatnode/mainnet-archive/rest' + 'vault/prod/osmosis/atn/mainnet' ) AS request, block_number FROM diff --git a/models/streamline/silver/core/realtime/streamline__transactions_realtime.sql b/models/streamline/silver/core/realtime/streamline__transactions_realtime.sql index 31c128f..cf7b3be 100644 --- a/models/streamline/silver/core/realtime/streamline__transactions_realtime.sql +++ b/models/streamline/silver/core/realtime/streamline__transactions_realtime.sql @@ -86,7 +86,7 @@ numbers AS ( ) :: INT AS partition_key, live.udf_api( 'POST', - '{service}/{Authentication}', + '{Service}/{Authentication}', OBJECT_CONSTRUCT( 'Content-Type', 'application/json' @@ -107,7 +107,7 @@ numbers AS ( 'asc' ) ), - 'vault/prod/osmosis/allthatnode/mainnet-archive/rpc' + 'vault/prod/osmosis/atn/mainnet' ) AS request, page_number, block_number AS block_number_requested diff --git a/models/streamline/silver/core/realtime/streamline__tx_counts_realtime.sql b/models/streamline/silver/core/realtime/streamline__tx_counts_realtime.sql index 5a2c575..bc54f53 100644 --- a/models/streamline/silver/core/realtime/streamline__tx_counts_realtime.sql +++ b/models/streamline/silver/core/realtime/streamline__tx_counts_realtime.sql @@ -5,8 +5,8 @@ target = "{{this.schema}}.{{this.identifier}}", params ={ "external_table" :"txcount_v2", "sql_limit" :"100000", - "producer_batch_size" :"1000", - "worker_batch_size" :"500", + "producer_batch_size" :"300", + "worker_batch_size" :"100", "sql_source" :"{{this.identifier}}" } ) ) }} @@ -30,7 +30,7 @@ SELECT ) :: INT AS partition_key, {{ target.database }}.live.udf_api( 'POST', - '{service}/{Authentication}', + '{Service}/{Authentication}', OBJECT_CONSTRUCT( 'Content-Type', 'application/json' @@ -51,7 +51,7 @@ SELECT 'asc' ) ), - 'vault/prod/osmosis/allthatnode/mainnet-archive/rpc' + 'vault/prod/osmosis/atn/mainnet' ) AS request FROM blocks diff --git a/models/streamline/silver/core/streamline__chainhead.sql b/models/streamline/silver/core/streamline__chainhead.sql index 462aa30..0f0b358 100644 --- a/models/streamline/silver/core/streamline__chainhead.sql +++ b/models/streamline/silver/core/streamline__chainhead.sql @@ -6,7 +6,7 @@ SELECT {{ target.database }}.live.udf_api( 'POST', - '{service}/{Authentication}', + '{Service}/{Authentication}', OBJECT_CONSTRUCT( 'Content-Type', 'application/json', @@ -23,5 +23,5 @@ SELECT 'params', [] ), - 'vault/prod/osmosis/allthatnode/mainnet-archive/rpc' + 'vault/prod/osmosis/atn/mainnet' ) :data :result :sync_info :latest_block_height :: INT AS block_number