From b6be3b8e64ada9a13ba8aec8fe3e0d5c346434c2 Mon Sep 17 00:00:00 2001 From: shah Date: Wed, 9 Oct 2024 12:17:53 -0700 Subject: [PATCH 01/50] STREAM-1051 added ez_native_transfers live view deployment macro --- macros/evm/evm.yaml.sql | 18 ++++ macros/evm/evm_abstractions.sql | 147 +++++++++++++++++++++++++++++++- 2 files changed, 164 insertions(+), 1 deletion(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index 32bd2eb..3d9e9e6 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -362,8 +362,26 @@ COMMENT = $$Returns the latest decoded events emitted by multiple contracts within the last `lookback` blocks. Submit missing ABIs [here](https://science.flipsidecrypto.xyz/abi-requestor/). *Please note there are RPC limitations on this method.* $$ sql: | {{ evm_latest_contract_events_decoded_ai(schema, blockchain, network) | indent(4) -}} + +- name: {{ schema -}}.tf_live_view_ez_native_transfers + signature: + - [block_height, INTEGER, The start block height to get the transfers from] + - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] + - [native_token_address, STRING, The address of the native token to get the transfers of] + return_type: + - "TABLE (tx_hash STRING, block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_position INTEGER, trace_index INTEGER, identifier STRING, origin_from_address STRING, origin_to_address STRING, origin_function_signature STRING, from_address STRING, to_address STRING, amount FLOAT, amount_precise_raw NUMBER, amount_precise FLOAT, amount_usd FLOAT, ez_native_transfers_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + COMMENT = $$Returns the native transfers for a given block height. If to_latest is true, it will continue fetching transfers until the latest block. Otherwise, it will fetch transfers until the block height is reached.$$ + sql: | + {{ evm_live_view_ez_native_transfers(schema, blockchain, network) | indent(4) -}} + {%- endmacro -%} + + {% macro config_eth_high_level_abstractions(blockchain, network) -%} {# This macro is used to generate high level abstractions for Ethereum mainnet only. diff --git a/macros/evm/evm_abstractions.sql b/macros/evm/evm_abstractions.sql index cea18f6..9ef3a70 100644 --- a/macros/evm/evm_abstractions.sql +++ b/macros/evm/evm_abstractions.sql @@ -1733,4 +1733,149 @@ from {{ ref('_eth__decoded_logs') }} where contract_address = (select contract_address from inputs) and block_number >= min_block and block_number <= (select min_block_no from chainhead) -{% endmacro %} \ No newline at end of file +{% endmacro %} + +{% macro evm_live_view_ez_native_transfers(schema, blockchain, network) %} +WITH heights AS ( + SELECT + livequery_dev.live.udf_api( + '{service}/{Authentication}', + livequery_Dev.utils.udf_json_rpc_call( + 'eth_blockNumber', + [] + ) + ):data AS result, + livequery_dev.utils.udf_hex_to_int(result:result)::integer as latest_block_height, + coalesce( + block_height, + latest_block_height + ) as min_height, + iff( + coalesce(to_latest, false), + latest_block_height, + min_height + ) as max_height +), +spine as ( + select + row_number() over ( + order by + null + ) -1 + coalesce(block_height, 0)::integer as spine_block_number, + min_height, + iff( + coalesce(to_latest, false), + latest_block_height, + min_height + ) as max_height, + latest_block_height + from + table(generator(ROWCOUNT => 5)), + heights + qualify spine_block_number between min_height and max_height +), +raw_block_data AS ( + SELECT + s.spine_block_number AS block_number, + livequery_dev.live.udf_api( + '{service}/{Authentication}', + livequery_Dev.utils.udf_json_rpc_call( + 'eth_getBlockByNumber', + [livequery_dev.utils.udf_int_to_hex(s.spine_block_number), true] + ) + ):data AS block_data, + b.value AS tx_data, + TO_TIMESTAMP_NTZ(livequery_dev.utils.udf_hex_to_int(block_data:result:timestamp::string)) AS block_timestamp, + tx_data:hash::string AS tx_hash, + tx_data:from::string AS from_address, + tx_data:to_address::string AS to_address, + TRY_TO_NUMBER(livequery_dev.utils.udf_hex_to_int(tx_data:value::string), 38, 0) / 1e18 AS eth_value, + TRY_TO_NUMBER(livequery_dev.utils.udf_hex_to_int(tx_data:value::string), 38, 0) AS eth_value_precise_raw, + TRY_TO_NUMBER(livequery_dev.utils.udf_hex_to_int(tx_data:value::string), 38, 0) / 1e18 AS eth_value_precise, + tx_data:input::string AS input, + livequery_dev.utils.udf_hex_to_int(tx_data:transactionIndex::string)::INTEGER AS tx_position, + 'CALL' AS TYPE, + 'SUCCESS' AS tx_status, + 'SUCCESS' AS trace_status, + CASE + WHEN LEFT(input, 10) = '0x' THEN SUBSTRING(input, 1, 10) + ELSE NULL + END AS origin_function_signature, + 'native_transfer' AS identifier, + NULL AS trace_index + FROM + spine s, + LATERAL FLATTEN(input => block_data:result:transactions) b + WHERE + TRY_TO_NUMBER(livequery_dev.utils.udf_hex_to_int(tx_data:value::string), 38, 0) > 0 +), +eth_base AS ( + SELECT + tx_hash, + block_number, + block_timestamp, + identifier, + from_address, + to_address, + eth_value AS amount, + eth_value_precise_raw AS amount_precise_raw, + eth_value_precise AS amount_precise, + tx_position, + trace_index + FROM + raw_block_data + WHERE + eth_value > 0 + AND tx_status = 'SUCCESS' + AND trace_status = 'SUCCESS' + AND TYPE NOT IN ('DELEGATECALL', 'STATICCALL') +), +tx_table AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + from_address AS origin_from_address, + to_address AS origin_to_address, + origin_function_signature + FROM + raw_block_data + WHERE + tx_hash IN (SELECT DISTINCT tx_hash FROM eth_base) +), +price_data AS ( + SELECT + DATE_TRUNC('hour', e.block_timestamp) AS hour, + AVG(p.price) AS price + FROM + eth_base e + JOIN ETHEREUM.PRICE.EZ_PRICES_HOURLY p + ON DATE_TRUNC('hour', e.block_timestamp) = p.hour + AND p.token_address = native_token_address + GROUP BY 1 +) +SELECT + A.tx_hash, + A.block_number, + A.block_timestamp, + A.tx_position, + A.trace_index, + A.identifier, + T.origin_from_address, + T.origin_to_address, + T.origin_function_signature, + A.from_address, + A.to_address, + A.amount::FLOAT, + A.amount_precise_raw::NUMBER(38,0), + A.amount_precise::FLOAT, + ROUND(A.amount * P.price, 2)::FLOAT AS amount_usd, + MD5(CONCAT(A.tx_hash, '|', COALESCE(A.trace_index::STRING, ''))) AS ez_native_transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + eth_base A + LEFT JOIN price_data P ON DATE_TRUNC('hour', A.block_timestamp) = P.hour + JOIN tx_table T ON A.tx_hash = T.tx_hash AND A.block_number = T.block_number +{% endmacro %} + From d50042e98e4d651c5735ad80e3350dfa825bee0d Mon Sep 17 00:00:00 2001 From: shah Date: Wed, 9 Oct 2024 13:26:28 -0700 Subject: [PATCH 02/50] STREAM-1051 add deploy eth_sepolia make directive --- Makefile | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4f9ddd5 --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ +SHELL := /bin/bash + +deploy_evm_eth_sepolia: + dbt run \ + -s livequery_models.deploy.evm.ethereum__sepolia \ + --vars '{UPDATE_UDFS_AND_SPS: true}' \ + --profiles-dir ~/.dbt \ + --profile livequery \ + --target dev \ No newline at end of file From f4ed98e6f000e54c081bbcc455e22c45fbae4f12 Mon Sep 17 00:00:00 2001 From: shah Date: Wed, 9 Oct 2024 16:21:53 -0700 Subject: [PATCH 03/50] rm Makefile --- Makefile | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index 4f9ddd5..0000000 --- a/Makefile +++ /dev/null @@ -1,9 +0,0 @@ -SHELL := /bin/bash - -deploy_evm_eth_sepolia: - dbt run \ - -s livequery_models.deploy.evm.ethereum__sepolia \ - --vars '{UPDATE_UDFS_AND_SPS: true}' \ - --profiles-dir ~/.dbt \ - --profile livequery \ - --target dev \ No newline at end of file From f825fadcca8ef196a04d7c7847959e490cbee954 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Thu, 10 Oct 2024 10:24:50 +0900 Subject: [PATCH 04/50] update evm live views tf --- macros/evm/evm.yaml.sql | 30 +- macros/evm/evm_abstractions.sql | 379 ++++++++----------------- macros/evm/evm_live_views.sql | 482 ++++++++++++++++++++++++++++++++ 3 files changed, 621 insertions(+), 270 deletions(-) create mode 100644 macros/evm/evm_live_views.sql diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index 3d9e9e6..8309a21 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -24,7 +24,7 @@ COMMENT = $$Returns the native asset balance at the latest block for a given address.$$ sql: | {{ evm_latest_native_balance_string(schema, blockchain, network) | indent(4) -}} - + - name: {{ schema -}}.tf_latest_native_balance signature: - [wallets, ARRAY, An array of addresses string to get the balance of at the latest block] @@ -41,7 +41,7 @@ - name: {{ schema -}}.tf_latest_token_balance signature: - [wallet, STRING, The address to get the balance of at the latest block] - - [token, STRING, The address of the token to get the balance of] + - [token, STRING, The address of the token to get the balance of] return_type: - "TABLE(status STRING, blockchain STRING, network STRING, wallet_address STRING, token_address STRING, symbol STRING, raw_balance STRING, balance FLOAT)" options: | @@ -55,7 +55,7 @@ - name: {{ schema -}}.tf_latest_token_balance signature: - [wallet, STRING, The address to get the balance of at the latest block] - - [tokens, ARRAY, An array of address strings of the tokens to get the balance of] + - [tokens, ARRAY, An array of address strings of the tokens to get the balance of] return_type: - "TABLE(status STRING, blockchain STRING, network STRING, wallet_address STRING, token_address STRING, symbol STRING, raw_balance STRING, balance FLOAT)" options: | @@ -83,7 +83,7 @@ - name: {{ schema -}}.tf_latest_token_balance signature: - [wallets, ARRAY, An array of addresses string to get the balance of at the latest block] - - [tokens, ARRAY, An array of address strings of the tokens to get the balance of] + - [tokens, ARRAY, An array of address strings of the tokens to get the balance of] return_type: - "TABLE(status STRING, blockchain STRING, network STRING, wallet_address STRING, token_address STRING, symbol STRING, raw_balance STRING, balance FLOAT)" options: | @@ -281,7 +281,7 @@ COMMENT = $$Returns the latest events emitted by a contract within the last `lookback` blocks. *Please note there are RPC limitations on this method.*$$ sql: | {{ evm_latest_contract_events_si(schema, blockchain, network) | indent(4) -}} - + - name: {{ schema -}}.tf_latest_contract_events signature: - [addresses, ARRAY, The addresses of the contracts to get the events of] @@ -378,8 +378,22 @@ sql: | {{ evm_live_view_ez_native_transfers(schema, blockchain, network) | indent(4) -}} -{%- endmacro -%} +- name: {{ schema -}}.tf_live_view_ez_token_transfers + signature: + - [block_height, INTEGER, The start block height to get the transfers from] + - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] + - [ez_token_transfers_id, STRING, The topic of the token transfers to get] + return_type: + - "TABLE (block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, event_index INTEGER, origin_funcion_signature STRING, origin_from_address STRING, origin_to_address STRING, contract_address STRING, from_address STRING, to_address STRING, raw_amount_precise NUMBER, raw_amount FLOAT, amount_precise STRING, amount FLOAT, amount_usd FLOAT, decimals INTEGER, symbol STRING, token_price FLOAT, has_decimal STRING, has_price STRING, ez_token_transfers_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + COMMENT = $$Returns the token transfers for a given block height. If to_latest is true, it will continue fetching transfers until the latest block. Otherwise, it will fetch transfers until the block height is reached.$$ + sql: | + {{ evm_live_view_ez_token_transfers(schema, blockchain, network) | indent(4) -}} +{%- endmacro -%} {% macro config_eth_high_level_abstractions(blockchain, network) -%} @@ -411,7 +425,7 @@ NOT NULL RETURNS NULL ON NULL INPUT VOLATILE - COMMENT = $$Returns the decoded events emitted by a contract from a specific block to the latest block. Submit missing ABIs [here](https://science.flipsidecrypto.xyz/abi-requestor/).$$ + COMMENT = $$Returns the decoded events emitted by a contract from a specific block to the latest block. Submit missing ABIs [here](https://science.flipsidecrypto.xyz/abi-requestor/).$$ sql: | {{ evm_contract_events_decoded(schema, blockchain, network) | indent(4) -}} -{%- endmacro -%} \ No newline at end of file +{%- endmacro -%} diff --git a/macros/evm/evm_abstractions.sql b/macros/evm/evm_abstractions.sql index 9ef3a70..e0416ec 100644 --- a/macros/evm/evm_abstractions.sql +++ b/macros/evm/evm_abstractions.sql @@ -1,10 +1,10 @@ {% macro evm_latest_native_balance_string(schema, blockchain, network) %} with base as (select lower(wallet) AS wallet_address) SELECT - case - when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') + case + when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') then 'Success' - else 'Error - Invalid Input' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -33,10 +33,10 @@ node_call AS ( FROM flat_addresses ) SELECT - case - when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') - then 'Success' - else 'Error - Invalid Input' + case + when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -76,11 +76,11 @@ node_call AS ( and blockchain = '{{blockchain}}' ) SELECT - case - when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') - and REGEXP_LIKE(token_address, '^0x([a-fA-F0-9]{40})$') - then 'Success' - else 'Error - Invalid Input' + case + when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') + and REGEXP_LIKE(token_address, '^0x([a-fA-F0-9]{40})$') + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -118,16 +118,16 @@ final AS ( raw_balance::INT / POW(10, ifnull(decimals,0)) AS balance FROM flat_rows - LEFT JOIN {{ ref('_evm__contracts_map') }} + LEFT JOIN {{ ref('_evm__contracts_map') }} ON token_address = address and blockchain = '{{blockchain}}' ) SELECT - case - when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') - and REGEXP_LIKE(token_address, '^0x([a-fA-F0-9]{40})$') - then 'Success' - else 'Error - Invalid Input' + case + when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') + and REGEXP_LIKE(token_address, '^0x([a-fA-F0-9]{40})$') + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -165,16 +165,16 @@ final AS ( raw_balance::INT / POW(10, ifnull(decimals,0)) AS balance FROM flat_rows - LEFT JOIN {{ ref('_evm__contracts_map') }} + LEFT JOIN {{ ref('_evm__contracts_map') }} ON token_address = address and blockchain = '{{blockchain}}' ) SELECT - case - when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') - and REGEXP_LIKE(token_address, '^0x([a-fA-F0-9]{40})$') - then 'Success' - else 'Error - Invalid Input' + case + when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') + and REGEXP_LIKE(token_address, '^0x([a-fA-F0-9]{40})$') + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -214,16 +214,16 @@ final AS ( raw_balance::INT / POW(10, ifnull(decimals,0)) AS balance FROM flat_rows - LEFT JOIN {{ ref('_evm__contracts_map') }} + LEFT JOIN {{ ref('_evm__contracts_map') }} ON token_address = address and blockchain = '{{blockchain}}' ) SELECT - case - when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') - and REGEXP_LIKE(token_address, '^0x([a-fA-F0-9]{40})$') - then 'Success' - else 'Error - Invalid Input' + case + when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') + and REGEXP_LIKE(token_address, '^0x([a-fA-F0-9]{40})$') + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -262,12 +262,12 @@ WITH inputs AS ( AND blockchain = '{{blockchain}}' ) SELECT - case - when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') + case + when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') and REGEXP_LIKE(token_address, '^0x([a-fA-F0-9]{40})$') and is_integer(block_number) - then 'Success' - else 'Error - Invalid Input' + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -316,12 +316,12 @@ final AS ( AND blockchain = '{{blockchain}}' ) SELECT - case - when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') + case + when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') and REGEXP_LIKE(token_address, '^0x([a-fA-F0-9]{40})$') and is_integer(block_number) - then 'Success' - else 'Error - Invalid Input' + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -370,12 +370,12 @@ final AS ( AND blockchain = '{{blockchain}}' ) SELECT - case - when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') + case + when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') and REGEXP_LIKE(token_address, '^0x([a-fA-F0-9]{40})$') and is_integer(block_number) - then 'Success' - else 'Error - Invalid Input' + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -433,12 +433,12 @@ final AS ( AND blockchain = '{{blockchain}}' ) SELECT - case - when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') + case + when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') and REGEXP_LIKE(token_address, '^0x([a-fA-F0-9]{40})$') and is_integer(block_number) - then 'Success' - else 'Error - Invalid Input' + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -488,12 +488,12 @@ final AS ( AND blockchain = '{{blockchain}}' ) SELECT - case - when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') + case + when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') and REGEXP_LIKE(token_address, '^0x([a-fA-F0-9]{40})$') and is_integer(block_number) - then 'Success' - else 'Error - Invalid Input' + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -552,12 +552,12 @@ final AS ( AND blockchain = '{{blockchain}}' ) SELECT - case - when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') + case + when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') and REGEXP_LIKE(token_address, '^0x([a-fA-F0-9]{40})$') and is_integer(block_number) - then 'Success' - else 'Error - Invalid Input' + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -616,12 +616,12 @@ final AS ( AND blockchain = '{{blockchain}}' ) SELECT - case - when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') + case + when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') and REGEXP_LIKE(token_address, '^0x([a-fA-F0-9]{40})$') and is_integer(block_number) - then 'Success' - else 'Error - Invalid Input' + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -689,12 +689,12 @@ final AS ( AND blockchain = '{{blockchain}}' ) SELECT - case - when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') + case + when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') and REGEXP_LIKE(token_address, '^0x([a-fA-F0-9]{40})$') and is_integer(block_number) - then 'Success' - else 'Error - Invalid Input' + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -710,11 +710,11 @@ FROM final {% macro evm_historical_native_balance_si(schema, blockchain, network) %} with base as (select lower(wallet) AS wallet_address, CONCAT('0x', TRIM(TO_CHAR(block_number, 'XXXXXXXXXX'))) as hex_block) SELECT - case - when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') + case + when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') and is_integer(block_number) - then 'Success' - else 'Error - Invalid Input' + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -723,7 +723,7 @@ SELECT block_number, utils.udf_hex_to_int({{schema}}.udf_rpc_eth_get_balance(wallet_address,hex_block)::string) AS raw_balance, (raw_balance / POW(10,18))::float AS balance -FROM base +FROM base LEFT JOIN {{ ref('_evm__native_symbol_map') }} on '{{blockchain}}' = blockchain and '{{network}}' = network @@ -746,11 +746,11 @@ inputs AS ( FROM blocks ) SELECT - case - when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') + case + when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') and is_integer(block_number) - then 'Success' - else 'Error - Invalid Input' + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -782,11 +782,11 @@ inputs AS ( FROM flat_wallets ) SELECT - case - when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') + case + when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') and is_integer(block_number) - then 'Success' - else 'Error - Invalid Input' + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -819,11 +819,11 @@ and '{{network}}' = network FROM flat_wallets ) SELECT - case - when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') + case + when REGEXP_LIKE(wallet_address, '^0x([a-fA-F0-9]{40})$') and is_integer(block_number) - then 'Success' - else 'Error - Invalid Input' + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -866,10 +866,10 @@ and '{{network}}' = network LATERAL FLATTEN(input => eth_getLogs) ) SELECT - case - when REGEXP_LIKE(contract_address, '^0x([a-fA-F0-9]{40})$') - then 'Success' - else 'Error - Invalid Input' + case + when REGEXP_LIKE(contract_address, '^0x([a-fA-F0-9]{40})$') + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -910,11 +910,11 @@ and '{{network}}' = network LATERAL FLATTEN(input => eth_getLogs) ) SELECT - case - when REGEXP_LIKE(contract_address, '^0x([a-fA-F0-9]{40})$') + case + when REGEXP_LIKE(contract_address, '^0x([a-fA-F0-9]{40})$') and is_integer(lookback) - then 'Success' - else 'Error - Invalid Input' + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -958,10 +958,10 @@ and '{{network}}' = network LATERAL FLATTEN(input => eth_getLogs) ) SELECT - case - when REGEXP_LIKE(contract_address, '^0x([a-fA-F0-9]{40})$') - then 'Success' - else 'Error - Invalid Input' + case + when REGEXP_LIKE(contract_address, '^0x([a-fA-F0-9]{40})$') + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -1005,11 +1005,11 @@ and '{{network}}' = network LATERAL FLATTEN(input => eth_getLogs) ) SELECT - case - when REGEXP_LIKE(contract_address, '^0x([a-fA-F0-9]{40})$') + case + when REGEXP_LIKE(contract_address, '^0x([a-fA-F0-9]{40})$') and is_integer(lookback) - then 'Success' - else 'Error - Invalid Input' + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -1125,10 +1125,10 @@ final AS ( transformed ) SELECT - case - when REGEXP_LIKE(n.contract_address, '^0x([a-fA-F0-9]{40})$') then 'Success' + case + when REGEXP_LIKE(n.contract_address, '^0x([a-fA-F0-9]{40})$') then 'Success' when f.event_name is null then 'Error - Contract ABI Not Found, submit ABIs [here](https://science.flipsidecrypto.xyz/abi-requestor/)' - else 'Error - Invalid Input' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -1253,10 +1253,10 @@ final AS ( transformed ) SELECT - case - when REGEXP_LIKE(n.contract_address, '^0x([a-fA-F0-9]{40})$') and is_integer(n.lookback) then 'Success' + case + when REGEXP_LIKE(n.contract_address, '^0x([a-fA-F0-9]{40})$') and is_integer(n.lookback) then 'Success' when f.event_name is null then 'Error - Contract ABI Not Found, submit ABIs [here](https://science.flipsidecrypto.xyz/abi-requestor/)' - else 'Error - Invalid Input' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -1380,10 +1380,10 @@ final AS ( transformed ) SELECT - case - when REGEXP_LIKE(n.contract_address, '^0x([a-fA-F0-9]{40})$') then 'Success' + case + when REGEXP_LIKE(n.contract_address, '^0x([a-fA-F0-9]{40})$') then 'Success' when f.event_name is null then 'Error - Contract ABI Not Found, submit ABIs [here](https://science.flipsidecrypto.xyz/abi-requestor/)' - else 'Error - Invalid Input' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -1510,10 +1510,10 @@ final AS ( transformed ) SELECT - case - when REGEXP_LIKE(n.contract_address, '^0x([a-fA-F0-9]{40})$') and is_integer(n.lookback) then 'Success' + case + when REGEXP_LIKE(n.contract_address, '^0x([a-fA-F0-9]{40})$') and is_integer(n.lookback) then 'Success' when f.event_name is null then 'Error - Contract ABI Not Found, submit ABIs [here](https://science.flipsidecrypto.xyz/abi-requestor/)' - else 'Error - Invalid Input' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -1561,10 +1561,10 @@ and n.event_index = f.event_index LATERAL FLATTEN(input => eth_getLogs) ) SELECT - case - when REGEXP_LIKE(contract_address, '^0x([a-fA-F0-9]{40})$') - then 'Success' - else 'Error - Invalid Input' + case + when REGEXP_LIKE(contract_address, '^0x([a-fA-F0-9]{40})$') + then 'Success' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -1577,7 +1577,7 @@ and n.event_index = f.event_index FROM node_flat UNION ALL SELECT - 'Success' as status, + 'Success' as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, tx_hash, @@ -1588,7 +1588,7 @@ and n.event_index = f.event_index data as event_data from {{ ref('_eth__logs') }} where contract_address = (select contract_address from node_call) - and block_number >= min_block + and block_number >= min_block and block_number <= (select min_block_no from chainhead) {% endmacro %} @@ -1696,10 +1696,10 @@ final AS ( transformed ) SELECT - case - when REGEXP_LIKE(n.contract_address, '^0x([a-fA-F0-9]{40})$') and is_integer(min_block) then 'Success' + case + when REGEXP_LIKE(n.contract_address, '^0x([a-fA-F0-9]{40})$') and is_integer(min_block) then 'Success' when f.event_name is null then 'Error - Contract ABI Not Found, submit ABIs [here](https://science.flipsidecrypto.xyz/abi-requestor/)' - else 'Error - Invalid Input' + else 'Error - Invalid Input' end as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -1717,7 +1717,7 @@ on n.block_number = f.block_number and n.tx_hash = f.tx_hash and n.event_index = f.event_index union all -select +select 'Success' as status, '{{blockchain}}' AS blockchain, '{{network}}' AS network, @@ -1734,148 +1734,3 @@ from {{ ref('_eth__decoded_logs') }} and block_number >= min_block and block_number <= (select min_block_no from chainhead) {% endmacro %} - -{% macro evm_live_view_ez_native_transfers(schema, blockchain, network) %} -WITH heights AS ( - SELECT - livequery_dev.live.udf_api( - '{service}/{Authentication}', - livequery_Dev.utils.udf_json_rpc_call( - 'eth_blockNumber', - [] - ) - ):data AS result, - livequery_dev.utils.udf_hex_to_int(result:result)::integer as latest_block_height, - coalesce( - block_height, - latest_block_height - ) as min_height, - iff( - coalesce(to_latest, false), - latest_block_height, - min_height - ) as max_height -), -spine as ( - select - row_number() over ( - order by - null - ) -1 + coalesce(block_height, 0)::integer as spine_block_number, - min_height, - iff( - coalesce(to_latest, false), - latest_block_height, - min_height - ) as max_height, - latest_block_height - from - table(generator(ROWCOUNT => 5)), - heights - qualify spine_block_number between min_height and max_height -), -raw_block_data AS ( - SELECT - s.spine_block_number AS block_number, - livequery_dev.live.udf_api( - '{service}/{Authentication}', - livequery_Dev.utils.udf_json_rpc_call( - 'eth_getBlockByNumber', - [livequery_dev.utils.udf_int_to_hex(s.spine_block_number), true] - ) - ):data AS block_data, - b.value AS tx_data, - TO_TIMESTAMP_NTZ(livequery_dev.utils.udf_hex_to_int(block_data:result:timestamp::string)) AS block_timestamp, - tx_data:hash::string AS tx_hash, - tx_data:from::string AS from_address, - tx_data:to_address::string AS to_address, - TRY_TO_NUMBER(livequery_dev.utils.udf_hex_to_int(tx_data:value::string), 38, 0) / 1e18 AS eth_value, - TRY_TO_NUMBER(livequery_dev.utils.udf_hex_to_int(tx_data:value::string), 38, 0) AS eth_value_precise_raw, - TRY_TO_NUMBER(livequery_dev.utils.udf_hex_to_int(tx_data:value::string), 38, 0) / 1e18 AS eth_value_precise, - tx_data:input::string AS input, - livequery_dev.utils.udf_hex_to_int(tx_data:transactionIndex::string)::INTEGER AS tx_position, - 'CALL' AS TYPE, - 'SUCCESS' AS tx_status, - 'SUCCESS' AS trace_status, - CASE - WHEN LEFT(input, 10) = '0x' THEN SUBSTRING(input, 1, 10) - ELSE NULL - END AS origin_function_signature, - 'native_transfer' AS identifier, - NULL AS trace_index - FROM - spine s, - LATERAL FLATTEN(input => block_data:result:transactions) b - WHERE - TRY_TO_NUMBER(livequery_dev.utils.udf_hex_to_int(tx_data:value::string), 38, 0) > 0 -), -eth_base AS ( - SELECT - tx_hash, - block_number, - block_timestamp, - identifier, - from_address, - to_address, - eth_value AS amount, - eth_value_precise_raw AS amount_precise_raw, - eth_value_precise AS amount_precise, - tx_position, - trace_index - FROM - raw_block_data - WHERE - eth_value > 0 - AND tx_status = 'SUCCESS' - AND trace_status = 'SUCCESS' - AND TYPE NOT IN ('DELEGATECALL', 'STATICCALL') -), -tx_table AS ( - SELECT - block_number, - block_timestamp, - tx_hash, - from_address AS origin_from_address, - to_address AS origin_to_address, - origin_function_signature - FROM - raw_block_data - WHERE - tx_hash IN (SELECT DISTINCT tx_hash FROM eth_base) -), -price_data AS ( - SELECT - DATE_TRUNC('hour', e.block_timestamp) AS hour, - AVG(p.price) AS price - FROM - eth_base e - JOIN ETHEREUM.PRICE.EZ_PRICES_HOURLY p - ON DATE_TRUNC('hour', e.block_timestamp) = p.hour - AND p.token_address = native_token_address - GROUP BY 1 -) -SELECT - A.tx_hash, - A.block_number, - A.block_timestamp, - A.tx_position, - A.trace_index, - A.identifier, - T.origin_from_address, - T.origin_to_address, - T.origin_function_signature, - A.from_address, - A.to_address, - A.amount::FLOAT, - A.amount_precise_raw::NUMBER(38,0), - A.amount_precise::FLOAT, - ROUND(A.amount * P.price, 2)::FLOAT AS amount_usd, - MD5(CONCAT(A.tx_hash, '|', COALESCE(A.trace_index::STRING, ''))) AS ez_native_transfers_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp -FROM - eth_base A - LEFT JOIN price_data P ON DATE_TRUNC('hour', A.block_timestamp) = P.hour - JOIN tx_table T ON A.tx_hash = T.tx_hash AND A.block_number = T.block_number -{% endmacro %} - diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql new file mode 100644 index 0000000..e8ff2b3 --- /dev/null +++ b/macros/evm/evm_live_views.sql @@ -0,0 +1,482 @@ +{% macro evm_live_view_latest_block_height(blockchain, network) %} + SELECT + live.udf_api( + '{service}/{Authentication}', + utils.udf_json_rpc_call( + 'eth_blockNumber', + [] + ) + ):data AS result, + utils.udf_hex_to_int(result:result)::integer AS latest_block_height, + COALESCE( + block_height, + latest_block_height + ) AS min_height, + iff( + COALESCE(to_latest, false), + latest_block_height, + min_height + ) AS max_height +{% endmacro %} + +{% macro evm_live_view_target_blocks(blockchain, network) %} + WITH heights AS ( + {{ evm_live_view_latest_block_height(blockchain, network) }} + ) + SELECT + ROW_NUMBER() OVER ( + ORDER BY + NULL + ) - 1 + COALESCE(block_height, 0)::integer AS block_number, + min_height, + iff( + COALESCE(to_latest, false), + latest_block_height, + min_height + ) AS max_height, + latest_block_height + FROM + TABLE(generator(ROWCOUNT => 500)), + heights + QUALIFY block_number BETWEEN min_height + AND max_height +{% endmacro %} + +-- Get Raw EVM chain data +{% macro evm_live_view_bronze_blocks(table_name) %} +SELECT + block_number, + live.udf_api( + '{service}/{Authentication}', + utils.udf_json_rpc_call( + 'eth_getBlockByNumber', + [utils.udf_int_to_hex(block_number), true] + ) + ):data.result AS DATA +FROM + {{ table_name }} +{% endmacro %} + +{% macro evm_live_view_bronze_receipts(table_name) %} +SELECT + latest_block_height, + block_number, + live.udf_api( + '{service}/{Authentication}', + utils.udf_json_rpc_call( + 'eth_getBlockReceipts', + [utils.udf_int_to_hex(block_number)] + ) + ):data.result AS result, + v.value AS DATA +FROM + {{ table_name }}, + LATERAL FLATTEN(result) v +{% endmacro %} + +{% macro evm_live_view_bronze_logs(table_name) %} +SELECT + r.block_number, + v.value +from + {{ table_name }} r, + lateral flatten(r.data:logs) v +{% endmacro %} + +{% macro evm_live_view_bronze_transactions(table_name) %} +SELECT + block_number, + v.value as DATA +from + {{ table_name }} r, + lateral flatten(r.data:transactions) v +{% endmacro %} + +-- Transformation macro for EVM chains +{% macro evm_live_view_silver_blocks(table_name) %} +SELECT + block_number, + utils.udf_hex_to_int(DATA:baseFeePerGas::STRING)::INT AS base_fee_per_gas, + utils.udf_hex_to_int(DATA:difficulty::STRING)::INT AS difficulty, + DATA:extraData::STRING AS extra_data, + utils.udf_hex_to_int(DATA:gasLimit::STRING)::INT AS gas_limit, + utils.udf_hex_to_int(DATA:gasUsed::STRING)::INT AS gas_used, + DATA:hash::STRING AS HASH, + DATA:logsBloom::STRING AS logs_bloom, + DATA:miner::STRING AS miner, + utils.udf_hex_to_int(DATA:nonce::STRING)::INT AS nonce, + utils.udf_hex_to_int(DATA:number::STRING)::INT AS NUMBER, + DATA:parentHash::STRING AS parent_hash, + DATA:receiptsRoot::STRING AS receipts_root, + DATA:sha3Uncles::STRING AS sha3_uncles, + utils.udf_hex_to_int(DATA:size::STRING)::INT AS SIZE, + DATA:stateRoot::STRING AS state_root, + utils.udf_hex_to_int(DATA:timestamp::STRING)::TIMESTAMP AS block_timestamp, + utils.udf_hex_to_int(DATA:totalDifficulty::STRING)::INT AS total_difficulty, + ARRAY_SIZE(DATA:transactions) AS tx_count, + DATA:transactionsRoot::STRING AS transactions_root, + DATA:uncles AS uncles, + DATA:withdrawals AS withdrawals, + DATA:withdrawalsRoot::STRING AS withdrawals_root, + md5( + CAST( + COALESCE( + CAST(block_number AS TEXT), + '_dbt_utils_surrogate_key_null_' + ) AS TEXT + ) + ) AS blocks_id, + utils.udf_hex_to_int(DATA:blobGasUsed::STRING)::INT AS blob_gas_used, + utils.udf_hex_to_int(DATA:excessBlobGas::STRING)::INT AS excess_blob_gas +FROM + {{ table_name }} +{% endmacro %} + +{% macro evm_live_view_silver_receipts(table_name) %} +select + latest_block_height, + block_number, + DATA :blockHash::STRING AS block_hash, + utils.udf_hex_to_int(DATA :blockNumber::STRING)::INT AS blockNumber, + utils.udf_hex_to_int(DATA :cumulativeGasUsed::STRING)::INT AS cumulative_gas_used, + utils.udf_hex_to_int(DATA :effectiveGasPrice::STRING)::INT / pow(10, 9) AS effective_gas_price, + DATA :from::STRING AS from_address, + utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS gas_used, + DATA :logs AS logs, + DATA :logsBloom::STRING AS logs_bloom, + utils.udf_hex_to_int(DATA :status::STRING)::INT AS status, + CASE + WHEN status = 1 THEN TRUE + ELSE FALSE + END AS tx_success, + CASE + WHEN status = 1 THEN 'SUCCESS' + ELSE 'FAIL' + END AS tx_status, + DATA :to::STRING AS to_address1, + CASE + WHEN to_address1 = '' THEN NULL + ELSE to_address1 + END AS to_address, + DATA :transactionHash::STRING AS tx_hash, + utils.udf_hex_to_int(DATA :transactionIndex::STRING)::INT AS POSITION, + utils.udf_hex_to_int(DATA :type::STRING)::INT AS TYPE, + utils.udf_hex_to_int(DATA :effectiveGasPrice::STRING)::INT AS blob_gas_price, + utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS blob_gas_used +from + {{ table_name }} +{% endmacro %} + +{% macro evm_live_view_silver_logs(table_name) %} +SELECT + latest_block_height, + block_number, + DATA:blockHash::STRING AS block_hash, + utils.udf_hex_to_int(DATA:blockNumber::STRING)::INT AS block_number, + utils.udf_hex_to_int(DATA:cumulativeGasUsed::STRING)::INT AS cumulative_gas_used, + utils.udf_hex_to_int(DATA:effectiveGasPrice::STRING)::INT / POW(10, 9) AS effective_gas_price, + DATA:from::STRING AS from_address, + utils.udf_hex_to_int(DATA:gasUsed::STRING)::INT AS gas_used, + DATA:logs AS logs, + DATA:logsBloom::STRING AS logs_bloom, + utils.udf_hex_to_int(DATA:status::STRING)::INT AS status, + CASE + WHEN status = 1 THEN TRUE + ELSE FALSE + END AS tx_success, + CASE + WHEN status = 1 THEN 'SUCCESS' + ELSE 'FAIL' + END AS tx_status, + DATA:to::STRING AS to_address1, + CASE + WHEN to_address1 = '' THEN NULL + ELSE to_address1 + END AS to_address, + DATA:transactionHash::STRING AS tx_hash, + utils.udf_hex_to_int(DATA:transactionIndex::STRING)::INT AS position, + utils.udf_hex_to_int(DATA:type::STRING)::INT AS type, + utils.udf_hex_to_int(DATA:effectiveGasPrice::STRING)::INT AS blob_gas_price, + utils.udf_hex_to_int(DATA:gasUsed::STRING)::INT AS blob_gas_used +FROM + {{ table_name }} +{% endmacro %} + +{% macro evm_live_view_silver_transactions(bronze_transactions, silver_blocks, silver_logs) %} +select + A.block_number AS block_number, + A.data :blockHash::STRING AS block_hash, + utils.udf_hex_to_int(A.data :blockNumber::STRING)::INT AS blockNumber, + utils.udf_hex_to_int(A.data :chainId::STRING)::INT AS chain_id, + A.data :from::STRING AS from_address, + utils.udf_hex_to_int(A.data :gas::STRING)::INT AS gas, + utils.udf_hex_to_int(A.data :gasPrice::STRING)::INT / pow(10, 9) AS gas_price, + A.data :hash::STRING AS tx_hash, + A.data :input::STRING AS input_data, + SUBSTR(input_data, 1, 10) AS origin_function_signature, + utils.udf_hex_to_int(A.data :maxFeePerGas::STRING)::INT / pow(10, 9) AS max_fee_per_gas, + utils.udf_hex_to_int( + A.data :maxPriorityFeePerGas::STRING + )::INT / pow(10, 9) AS max_priority_fee_per_gas, + utils.udf_hex_to_int(A.data :nonce::STRING)::INT AS nonce, + A.data :r::STRING AS r, + A.data :s::STRING AS s, + A.data :to::STRING AS to_address1, + utils.udf_hex_to_int(A.data :transactionIndex::STRING)::INT AS POSITION, + A.data :type::STRING AS TYPE, + A.data :v::STRING AS v, + utils.udf_hex_to_int(A.data :value::STRING) AS value_precise_raw, + value_precise_raw * power(10, -18) AS value_precise, + value_precise::FLOAT AS VALUE, + A.data :accessList AS access_list, + A.data, + A.data: blobVersionedHashes::ARRAY AS blob_versioned_hashes, + utils.udf_hex_to_int(A.data: maxFeePerGas::STRING)::INT AS max_fee_per_blob_gas, + block_timestamp, + CASE + WHEN block_timestamp IS NULL + OR tx_status IS NULL THEN TRUE + ELSE FALSE + END AS is_pending, + r.gas_used, + tx_success, + tx_status, + cumulative_gas_used, + effective_gas_price, + utils.udf_hex_to_int(A.data :gasPrice) * power(10, -18) * r.gas_used AS tx_fee_precise, + COALESCE(tx_fee_precise::FLOAT, 0) AS tx_fee, + r.type as tx_type, + r.blob_gas_used, + r.blob_gas_price, +from + {{ bronze_transactions }} A + left join {{ silver_blocks }} b on b.block_number = A.block_number + left join {{ silver_logs }} as r on r.tx_hash = A.data :hash::STRING +{% endmacro %} + +-- Get EVM chain fact data +{% macro evm_live_view_fact_blocks(blockchain, network) %} + WITH spine AS ( + {{ evm_live_view_target_blocks(blockchain, network) }} + ), + raw_block_txs AS ( + {{ evm_live_view_bronze_blocks(spine) }} + ) + {{ evm_live_view_silver_blocks(raw_block_txs) }} +{% endmacro %} + +{% macro evm_live_view_fact_logs(blockchain, network) %} + WITH spine AS ( + {{ evm_live_view_target_blocks(blockchain, network) }} + ), + raw_receipts AS ( + {{ evm_live_view_bronze_receipts(spine) }} + ), + raw_logs AS ( + {{ evm_live_view_bronze_logs(raw_receipts) }} + ) + {{ evm_live_view_silver_logs(raw_logs) }} +{% endmacro %} + +-- Get EVM chain ez data +{% macro evm_live_view_ez_token_transfers(schema, blockchain, network) %} +WITH spine AS ( + {{ evm_live_view_target_blocks(blockchain, network) }} +), +raw_block_txs AS ( + {{ evm_live_view_bronze_blocks(spine) }} +), +raw_receipts AS ( + {{ evm_live_view_bronze_receipts(spine) }} +), +raw_logs AS ( + {{ evm_live_view_bronze_logs(raw_receipts) }} +), +raw_transactions AS ( + {{ evm_live_view_bronze_transactions(raw_receipts) }} +), +blocks AS ( + {{ evm_live_view_silver_blocks(raw_block_txs) }} +), +receipts AS ( + {{ evm_live_view_silver_receipts(raw_receipts) }} +), +transactions AS ( + {{ evm_live_view_silver_transactions(raw_transactions, blocks, receipts) }} +), +logs AS ( + {{ evm_live_view_silver_logs(raw_logs) }} +) + +SELECT + block_number, + block_timestamp, + tx_hash, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + contract_address::STRING AS contract_address, + CONCAT('0x', SUBSTR(topics [1], 27, 40))::STRING AS from_address, + CONCAT('0x', SUBSTR(topics [2], 27, 40))::STRING AS to_address, + utils.udf_hex_to_int(SUBSTR(DATA, 3, 64)) AS raw_amount_precise, + raw_amount_precise::FLOAT AS raw_amount, + IFF( + C.decimals IS NOT NULL, + raw_amount_precise * power(10, C.decimals * -1), + NULL + ) AS amount_precise, + amount_precise::FLOAT AS amount, + IFF( + C.decimals IS NOT NULL + AND price IS NOT NULL, + amount * price, + NULL + ) AS amount_usd, + C.decimals AS decimals, + C.symbol AS symbol, + price AS token_price, + CASE + WHEN C.decimals IS NULL THEN 'false' + ELSE 'true' + END AS has_decimal, + CASE + WHEN price IS NULL THEN 'false' + ELSE 'true' + END AS has_price, + md5( + cast( + coalesce( + cast(tx_hash as TEXT), + '_dbt_utils_surrogate_key_null_' + ) || '-' || coalesce( + cast(event_index as TEXT), + '_dbt_utils_surrogate_key_null_' + ) as TEXT + ) + ) as ez_token_transfers_id, + sysdate() as inserted_timestamp, + sysdate() as modified_timestamp +FROM + logs l + LEFT JOIN {{ schema }}.price.EZ_PRICES_HOURLY p ON l.contract_address = p.token_address + AND DATE_TRUNC('hour', l.block_timestamp) = HOUR + LEFT JOIN {{ schema }}.core.DIM_CONTRACTS C ON l.contract_address = C.address +WHERE + topics [0]::STRING = ez_token_transfers_id + AND tx_status = 'SUCCESS' + and raw_amount IS NOT NULL + AND to_address IS NOT NULL +AND from_address IS NOT NULL +{% endmacro %} + +{% macro evm_live_view_ez_native_transfers(schema, blockchain, network) %} +WITH spine AS ( + {{ evm_live_view_target_blocks(blockchain, network) }} +), + +raw_block_data AS ( + SELECT + s.spine_block_number AS block_number, + live.udf_api( + '{service}/{Authentication}', + utils.udf_json_rpc_call( + 'eth_getBlockByNumber', + [utils.udf_int_to_hex(s.spine_block_number), true] + ) + ):data AS block_data, + b.value AS tx_data, + TO_TIMESTAMP_NTZ(utils.udf_hex_to_int(block_data:result:timestamp::string)) AS block_timestamp, + tx_data:hash::string AS tx_hash, + tx_data:from::string AS from_address, + tx_data:to_address::string AS to_address, + TRY_TO_NUMBER(utils.udf_hex_to_int(tx_data:value::string), 38, 0) / 1e18 AS eth_value, + TRY_TO_NUMBER(utils.udf_hex_to_int(tx_data:value::string), 38, 0) AS eth_value_precise_raw, + TRY_TO_NUMBER(utils.udf_hex_to_int(tx_data:value::string), 38, 0) / 1e18 AS eth_value_precise, + tx_data:input::string AS input, + utils.udf_hex_to_int(tx_data:transactionIndex::string)::INTEGER AS tx_position, + 'CALL' AS TYPE, + 'SUCCESS' AS tx_status, + 'SUCCESS' AS trace_status, + CASE + WHEN LEFT(input, 10) = '0x' THEN SUBSTRING(input, 1, 10) + ELSE NULL + END AS origin_function_signature, + 'native_transfer' AS identifier, + NULL AS trace_index + FROM + spine s, + LATERAL FLATTEN(input => block_data:result:transactions) b + WHERE + TRY_TO_NUMBER(utils.udf_hex_to_int(tx_data:value::string), 38, 0) > 0 +), +eth_base AS ( + SELECT + tx_hash, + block_number, + block_timestamp, + identifier, + from_address, + to_address, + eth_value AS amount, + eth_value_precise_raw AS amount_precise_raw, + eth_value_precise AS amount_precise, + tx_position, + trace_index + FROM + raw_block_data + WHERE + eth_value > 0 + AND tx_status = 'SUCCESS' + AND trace_status = 'SUCCESS' + AND TYPE NOT IN ('DELEGATECALL', 'STATICCALL') +), +tx_table AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + from_address AS origin_from_address, + to_address AS origin_to_address, + origin_function_signature + FROM + raw_block_data + WHERE + tx_hash IN (SELECT DISTINCT tx_hash FROM eth_base) +), +price_data AS ( + SELECT + DATE_TRUNC('hour', e.block_timestamp) AS hour, + AVG(p.price) AS price + FROM + eth_base e + JOIN {{ schema }}.PRICE.EZ_PRICES_HOURLY p + ON DATE_TRUNC('hour', e.block_timestamp) = p.hour + AND p.token_address = native_token_address + GROUP BY 1 +) +SELECT + A.tx_hash, + A.block_number, + A.block_timestamp, + A.tx_position, + A.trace_index, + A.identifier, + T.origin_from_address, + T.origin_to_address, + T.origin_function_signature, + A.from_address, + A.to_address, + A.amount::FLOAT, + A.amount_precise_raw::NUMBER(38,0), + A.amount_precise::FLOAT, + ROUND(A.amount * P.price, 2)::FLOAT AS amount_usd, + MD5(CONCAT(A.tx_hash, '|', COALESCE(A.trace_index::STRING, ''))) AS ez_native_transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + eth_base A + LEFT JOIN price_data P ON DATE_TRUNC('hour', A.block_timestamp) = P.hour + JOIN tx_table T ON A.tx_hash = T.tx_hash AND A.block_number = T.block_number +{% endmacro %} + From 3b37b90a01d25868f4b3cec29f99662f54a73aca Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Thu, 10 Oct 2024 14:16:57 +0900 Subject: [PATCH 05/50] [STREAM-1050] fix macros for bronze & silver --- macros/evm/evm.yaml.sql | 4 +- macros/evm/evm_live_views.sql | 134 +++++++++++++++------------------- 2 files changed, 62 insertions(+), 76 deletions(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index 8309a21..97113e6 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -369,7 +369,7 @@ - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] - [native_token_address, STRING, The address of the native token to get the transfers of] return_type: - - "TABLE (tx_hash STRING, block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_position INTEGER, trace_index INTEGER, identifier STRING, origin_from_address STRING, origin_to_address STRING, origin_function_signature STRING, from_address STRING, to_address STRING, amount FLOAT, amount_precise_raw NUMBER, amount_precise FLOAT, amount_usd FLOAT, ez_native_transfers_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + - "TABLE(tx_hash STRING, block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_position INTEGER, trace_index INTEGER, identifier STRING, origin_from_address STRING, origin_to_address STRING, origin_function_signature STRING, from_address STRING, to_address STRING, amount FLOAT, amount_precise_raw NUMBER, amount_precise FLOAT, amount_usd FLOAT, ez_native_transfers_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" options: | NOT NULL RETURNS NULL ON NULL INPUT @@ -384,7 +384,7 @@ - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] - [ez_token_transfers_id, STRING, The topic of the token transfers to get] return_type: - - "TABLE (block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, event_index INTEGER, origin_funcion_signature STRING, origin_from_address STRING, origin_to_address STRING, contract_address STRING, from_address STRING, to_address STRING, raw_amount_precise NUMBER, raw_amount FLOAT, amount_precise STRING, amount FLOAT, amount_usd FLOAT, decimals INTEGER, symbol STRING, token_price FLOAT, has_decimal STRING, has_price STRING, ez_token_transfers_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, event_index INTEGER, origin_funcion_signature STRING, origin_from_address STRING, origin_to_address STRING, contract_address STRING, from_address STRING, to_address STRING, raw_amount_precise NUMBER, raw_amount FLOAT, amount_precise STRING, amount FLOAT, amount_usd FLOAT, decimals INTEGER, symbol STRING, token_price FLOAT, has_decimal STRING, has_price STRING, ez_token_transfers_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" options: | NOT NULL RETURNS NULL ON NULL INPUT diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index e8ff2b3..15b6d10 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -29,17 +29,16 @@ NULL ) - 1 + COALESCE(block_height, 0)::integer AS block_number, min_height, - iff( + IFF( COALESCE(to_latest, false), - latest_block_height, + block_height, min_height ) AS max_height, latest_block_height FROM TABLE(generator(ROWCOUNT => 500)), - heights - QUALIFY block_number BETWEEN min_height - AND max_height + heights qualify block_number BETWEEN min_height + AND max_height {% endmacro %} -- Get Raw EVM chain data @@ -54,7 +53,7 @@ SELECT ) ):data.result AS DATA FROM - {{ table_name }} + {{ table_name | replace('\'', '')}} {% endmacro %} {% macro evm_live_view_bronze_receipts(table_name) %} @@ -167,39 +166,31 @@ from {{ table_name }} {% endmacro %} -{% macro evm_live_view_silver_logs(table_name) %} -SELECT - latest_block_height, - block_number, - DATA:blockHash::STRING AS block_hash, - utils.udf_hex_to_int(DATA:blockNumber::STRING)::INT AS block_number, - utils.udf_hex_to_int(DATA:cumulativeGasUsed::STRING)::INT AS cumulative_gas_used, - utils.udf_hex_to_int(DATA:effectiveGasPrice::STRING)::INT / POW(10, 9) AS effective_gas_price, - DATA:from::STRING AS from_address, - utils.udf_hex_to_int(DATA:gasUsed::STRING)::INT AS gas_used, - DATA:logs AS logs, - DATA:logsBloom::STRING AS logs_bloom, - utils.udf_hex_to_int(DATA:status::STRING)::INT AS status, +{% macro evm_live_view_silver_logs(silver_receipts, silver_transactions) %} +select + r.latest_block_height, + r.block_number, + r.tx_hash, + txs.block_timestamp, + txs.origin_function_signature, + r.from_address AS origin_from_address, + r.to_address AS origin_to_address, + r.tx_status, + v.VALUE :address::STRING AS contract_address, + v.VALUE :blockHash::STRING AS block_hash, + v.VALUE :data::STRING AS DATA, + utils.udf_hex_to_int(v.VALUE :logIndex::STRING)::INT AS event_index, + v.VALUE :removed::BOOLEAN AS event_removed, + v.VALUE :topics AS topics, CASE - WHEN status = 1 THEN TRUE + WHEN txs.block_timestamp IS NULL + OR txs.origin_function_signature IS NULL THEN TRUE ELSE FALSE - END AS tx_success, - CASE - WHEN status = 1 THEN 'SUCCESS' - ELSE 'FAIL' - END AS tx_status, - DATA:to::STRING AS to_address1, - CASE - WHEN to_address1 = '' THEN NULL - ELSE to_address1 - END AS to_address, - DATA:transactionHash::STRING AS tx_hash, - utils.udf_hex_to_int(DATA:transactionIndex::STRING)::INT AS position, - utils.udf_hex_to_int(DATA:type::STRING)::INT AS type, - utils.udf_hex_to_int(DATA:effectiveGasPrice::STRING)::INT AS blob_gas_price, - utils.udf_hex_to_int(DATA:gasUsed::STRING)::INT AS blob_gas_used -FROM - {{ table_name }} + END AS is_pending, +from + {{ bronze_receipts }} r + left join {{ silver_transactions }} txs on txs.tx_hash = r.tx_hash, + lateral flatten(r.logs) v {% endmacro %} {% macro evm_live_view_silver_transactions(bronze_transactions, silver_blocks, silver_logs) %} @@ -260,52 +251,47 @@ from {{ evm_live_view_target_blocks(blockchain, network) }} ), raw_block_txs AS ( - {{ evm_live_view_bronze_blocks(spine) }} + {{ evm_live_view_bronze_blocks('spine') }} ) - {{ evm_live_view_silver_blocks(raw_block_txs) }} + {{ evm_live_view_silver_blocks('raw_block_txs') }} {% endmacro %} {% macro evm_live_view_fact_logs(blockchain, network) %} WITH spine AS ( {{ evm_live_view_target_blocks(blockchain, network) }} ), + raw_block_txs AS ( + {{ evm_live_view_bronze_blocks('spine') }} + ), raw_receipts AS ( - {{ evm_live_view_bronze_receipts(spine) }} + {{ evm_live_view_bronze_receipts('spine') }} ), raw_logs AS ( - {{ evm_live_view_bronze_logs(raw_receipts) }} + {{ evm_live_view_bronze_logs('raw_receipts') }} + ), + raw_transactions AS ( + {{ evm_live_view_bronze_transactions('raw_receipts') }} + ), + blocks AS ( + {{ evm_live_view_silver_blocks('raw_block_txs') }} + ), + receipts AS ( + {{ evm_live_view_silver_receipts('raw_receipts') }} + ), + transactions AS ( + {{ evm_live_view_silver_transactions('raw_transactions', 'blocks', 'receipts') }} + ), + logs AS ( + {{ evm_live_view_silver_logs('receipts', 'transactions') }} ) - {{ evm_live_view_silver_logs(raw_logs) }} + + SELECT * FROM logs {% endmacro %} -- Get EVM chain ez data {% macro evm_live_view_ez_token_transfers(schema, blockchain, network) %} -WITH spine AS ( - {{ evm_live_view_target_blocks(blockchain, network) }} -), -raw_block_txs AS ( - {{ evm_live_view_bronze_blocks(spine) }} -), -raw_receipts AS ( - {{ evm_live_view_bronze_receipts(spine) }} -), -raw_logs AS ( - {{ evm_live_view_bronze_logs(raw_receipts) }} -), -raw_transactions AS ( - {{ evm_live_view_bronze_transactions(raw_receipts) }} -), -blocks AS ( - {{ evm_live_view_silver_blocks(raw_block_txs) }} -), -receipts AS ( - {{ evm_live_view_silver_receipts(raw_receipts) }} -), -transactions AS ( - {{ evm_live_view_silver_transactions(raw_transactions, blocks, receipts) }} -), -logs AS ( - {{ evm_live_view_silver_logs(raw_logs) }} +WITH fact_logs AS ( + {{ evm_live_view_fact_logs(blockchain, network) }} ) SELECT @@ -358,10 +344,10 @@ SELECT sysdate() as inserted_timestamp, sysdate() as modified_timestamp FROM - logs l - LEFT JOIN {{ schema }}.price.EZ_PRICES_HOURLY p ON l.contract_address = p.token_address + fact_logs l + LEFT JOIN {{ blockchain }}.price.EZ_PRICES_HOURLY p ON l.contract_address = p.token_address AND DATE_TRUNC('hour', l.block_timestamp) = HOUR - LEFT JOIN {{ schema }}.core.DIM_CONTRACTS C ON l.contract_address = C.address + LEFT JOIN {{ blockchain }}.core.DIM_CONTRACTS C ON l.contract_address = C.address WHERE topics [0]::STRING = ez_token_transfers_id AND tx_status = 'SUCCESS' @@ -377,12 +363,12 @@ WITH spine AS ( raw_block_data AS ( SELECT - s.spine_block_number AS block_number, + s.block_number AS block_number, live.udf_api( '{service}/{Authentication}', utils.udf_json_rpc_call( 'eth_getBlockByNumber', - [utils.udf_int_to_hex(s.spine_block_number), true] + [utils.udf_int_to_hex(s.block_number), true] ) ):data AS block_data, b.value AS tx_data, @@ -450,7 +436,7 @@ price_data AS ( AVG(p.price) AS price FROM eth_base e - JOIN {{ schema }}.PRICE.EZ_PRICES_HOURLY p + JOIN {{ blockchain }}.PRICE.EZ_PRICES_HOURLY p ON DATE_TRUNC('hour', e.block_timestamp) = p.hour AND p.token_address = native_token_address GROUP BY 1 From 00764d7b0ca0c747b887634fd65eee021587b811 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Thu, 10 Oct 2024 14:41:40 +0900 Subject: [PATCH 06/50] [STREAM-1050] fix alias in macros --- macros/evm/evm.yaml.sql | 2 +- macros/evm/evm_live_views.sql | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index 97113e6..f22d06b 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -384,7 +384,7 @@ - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] - [ez_token_transfers_id, STRING, The topic of the token transfers to get] return_type: - - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, event_index INTEGER, origin_funcion_signature STRING, origin_from_address STRING, origin_to_address STRING, contract_address STRING, from_address STRING, to_address STRING, raw_amount_precise NUMBER, raw_amount FLOAT, amount_precise STRING, amount FLOAT, amount_usd FLOAT, decimals INTEGER, symbol STRING, token_price FLOAT, has_decimal STRING, has_price STRING, ez_token_transfers_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, event_index INTEGER, origin_funcion_signature STRING, origin_from_address STRING, origin_to_address STRING, contract_address STRING, from_address STRING, to_address STRING, raw_amount_precise STRING, raw_amount FLOAT, amount_precise FLOAT, amount FLOAT, amount_usd FLOAT, decimals INTEGER, symbol STRING, token_price FLOAT, has_decimal STRING, has_price STRING, ez_token_transfers_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" options: | NOT NULL RETURNS NULL ON NULL INPUT diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 15b6d10..75ca094 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -21,7 +21,7 @@ {% macro evm_live_view_target_blocks(blockchain, network) %} WITH heights AS ( - {{ evm_live_view_latest_block_height(blockchain, network) }} + {{ evm_live_view_latest_block_height(blockchain, network) | indent(4) -}} ) SELECT ROW_NUMBER() OVER ( @@ -188,7 +188,7 @@ select ELSE FALSE END AS is_pending, from - {{ bronze_receipts }} r + {{ silver_receipts }} r left join {{ silver_transactions }} txs on txs.tx_hash = r.tx_hash, lateral flatten(r.logs) v {% endmacro %} @@ -248,41 +248,41 @@ from -- Get EVM chain fact data {% macro evm_live_view_fact_blocks(blockchain, network) %} WITH spine AS ( - {{ evm_live_view_target_blocks(blockchain, network) }} + {{ evm_live_view_target_blocks(blockchain, network) | indent(4) -}} ), raw_block_txs AS ( - {{ evm_live_view_bronze_blocks('spine') }} + {{ evm_live_view_bronze_blocks('spine') | indent(4) -}} ) - {{ evm_live_view_silver_blocks('raw_block_txs') }} + {{ evm_live_view_silver_blocks('raw_block_txs') | indent(4) -}} {% endmacro %} {% macro evm_live_view_fact_logs(blockchain, network) %} WITH spine AS ( - {{ evm_live_view_target_blocks(blockchain, network) }} + {{ evm_live_view_target_blocks(blockchain, network) | indent(4) -}} ), raw_block_txs AS ( - {{ evm_live_view_bronze_blocks('spine') }} + {{ evm_live_view_bronze_blocks('spine') | indent(4) -}} ), raw_receipts AS ( - {{ evm_live_view_bronze_receipts('spine') }} + {{ evm_live_view_bronze_receipts('spine') | indent(4) -}} ), raw_logs AS ( - {{ evm_live_view_bronze_logs('raw_receipts') }} + {{ evm_live_view_bronze_logs('raw_receipts') | indent(4) -}} ), raw_transactions AS ( - {{ evm_live_view_bronze_transactions('raw_receipts') }} + {{ evm_live_view_bronze_transactions('raw_receipts') | indent(4) -}} ), blocks AS ( - {{ evm_live_view_silver_blocks('raw_block_txs') }} + {{ evm_live_view_silver_blocks('raw_block_txs') | indent(4) -}} ), receipts AS ( - {{ evm_live_view_silver_receipts('raw_receipts') }} + {{ evm_live_view_silver_receipts('raw_receipts') | indent(4) -}} ), transactions AS ( - {{ evm_live_view_silver_transactions('raw_transactions', 'blocks', 'receipts') }} + {{ evm_live_view_silver_transactions('raw_transactions', 'blocks', 'receipts') | indent(4) -}} ), logs AS ( - {{ evm_live_view_silver_logs('receipts', 'transactions') }} + {{ evm_live_view_silver_logs('receipts', 'transactions') | indent(4) -}} ) SELECT * FROM logs @@ -291,7 +291,7 @@ from -- Get EVM chain ez data {% macro evm_live_view_ez_token_transfers(schema, blockchain, network) %} WITH fact_logs AS ( - {{ evm_live_view_fact_logs(blockchain, network) }} + {{ evm_live_view_fact_logs(blockchain, network) | indent(4) -}} ) SELECT From c965d5e3cddb470aeff079d688a702ff12a2a6ee Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Fri, 11 Oct 2024 14:32:29 +0900 Subject: [PATCH 07/50] [STREAM-1052] add live view fact blocks --- macros/evm/evm.yaml.sql | 28 +++++ macros/evm/evm_live_views.sql | 212 +++++++++++++++++++++++++--------- 2 files changed, 185 insertions(+), 55 deletions(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index f22d06b..e4f5ca0 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -363,6 +363,34 @@ sql: | {{ evm_latest_contract_events_decoded_ai(schema, blockchain, network) | indent(4) -}} +- name: {{ schema -}}.tf_live_view_fact_blocks + signature: + - [block_height, INTEGER, The start block height to get the transfers from] + - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] + return_type: + - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, network STRING, blockchain STRING, tx_count INTEGER, difficulty INTEGER, total_difficulty INTEGER, extra_data STRING, gas_limit INTEGER, gas_used INTEGER, hash STRING, parent_hash STRING, miner STRING, nonce INTEGER, receipts_root STRING, sha3_uncles STRING, size INTEGER, uncle_blocks VARIANT, block_header_json OBJECT, excess_blob_gas INTEGER, blob_gas_used INTEGER, fact_blocks_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ, withdrawals VARIANT, withdrawals_root STRING)" + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + COMMENT = $$Returns the block data for a given block height. If to_latest is true, it will continue fetching blocks until the latest block. Otherwise, it will fetch blocks until the block height is reached.$$ + sql: | + {{ evm_live_view_fact_blocks(schema, blockchain, network) | indent(4) -}} + +- name: {{ schema -}}.tf_live_view_fact_logs + signature: + - [block_height, INTEGER, The start block height to get the logs from] + - [to_latest, BOOLEAN, Whether to continue fetching logs until the latest block or not] + return_type: + - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, origin_function_signature STRING, origin_from_address STRING, origin_to_address STRING, event_index INTEGER, contract_address STRING, topics VARIANT, data STRING, event_removed BOOLEAN, tx_status STRING, _log_id STRING, fact_event_logs_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + COMMENT = $$Returns the logs for a given block height. If to_latest is true, it will continue fetching logs until the latest block. Otherwise, it will fetch logs until the block height is reached.$$ + sql: | + {{ evm_live_view_fact_logs(schema, blockchain, network) | indent(4) -}} + - name: {{ schema -}}.tf_live_view_ez_native_transfers signature: - [block_height, INTEGER, The start block height to get the transfers from] diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 75ca094..bcc9dec 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -53,7 +53,7 @@ SELECT ) ):data.result AS DATA FROM - {{ table_name | replace('\'', '')}} + {{ table_name }} {% endmacro %} {% macro evm_live_view_bronze_receipts(table_name) %} @@ -77,8 +77,8 @@ FROM SELECT r.block_number, v.value -from - {{ table_name }} r, +FROM + {{ table_name }} AS r, lateral flatten(r.data:logs) v {% endmacro %} @@ -86,8 +86,8 @@ from SELECT block_number, v.value as DATA -from - {{ table_name }} r, +FROM + {{ table_name }} AS r, lateral flatten(r.data:transactions) v {% endmacro %} @@ -95,6 +95,7 @@ from {% macro evm_live_view_silver_blocks(table_name) %} SELECT block_number, + utils.udf_hex_to_int(DATA:timestamp::STRING)::TIMESTAMP AS block_timestamp, utils.udf_hex_to_int(DATA:baseFeePerGas::STRING)::INT AS base_fee_per_gas, utils.udf_hex_to_int(DATA:difficulty::STRING)::INT AS difficulty, DATA:extraData::STRING AS extra_data, @@ -110,7 +111,6 @@ SELECT DATA:sha3Uncles::STRING AS sha3_uncles, utils.udf_hex_to_int(DATA:size::STRING)::INT AS SIZE, DATA:stateRoot::STRING AS state_root, - utils.udf_hex_to_int(DATA:timestamp::STRING)::TIMESTAMP AS block_timestamp, utils.udf_hex_to_int(DATA:totalDifficulty::STRING)::INT AS total_difficulty, ARRAY_SIZE(DATA:transactions) AS tx_count, DATA:transactionsRoot::STRING AS transactions_root, @@ -132,7 +132,7 @@ FROM {% endmacro %} {% macro evm_live_view_silver_receipts(table_name) %} -select +SELECT latest_block_height, block_number, DATA :blockHash::STRING AS block_hash, @@ -162,39 +162,38 @@ select utils.udf_hex_to_int(DATA :type::STRING)::INT AS TYPE, utils.udf_hex_to_int(DATA :effectiveGasPrice::STRING)::INT AS blob_gas_price, utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS blob_gas_used -from +FROM {{ table_name }} {% endmacro %} {% macro evm_live_view_silver_logs(silver_receipts, silver_transactions) %} -select - r.latest_block_height, +SELECT r.block_number, - r.tx_hash, txs.block_timestamp, + r.tx_hash, txs.origin_function_signature, r.from_address AS origin_from_address, r.to_address AS origin_to_address, - r.tx_status, + utils.udf_hex_to_int(v.VALUE :logIndex::STRING)::INT AS event_index, v.VALUE :address::STRING AS contract_address, v.VALUE :blockHash::STRING AS block_hash, v.VALUE :data::STRING AS DATA, - utils.udf_hex_to_int(v.VALUE :logIndex::STRING)::INT AS event_index, v.VALUE :removed::BOOLEAN AS event_removed, v.VALUE :topics AS topics, + r.tx_status, CASE WHEN txs.block_timestamp IS NULL OR txs.origin_function_signature IS NULL THEN TRUE ELSE FALSE END AS is_pending, -from - {{ silver_receipts }} r - left join {{ silver_transactions }} txs on txs.tx_hash = r.tx_hash, +FROM + {{ silver_receipts }} AS r + LEFT JOIN {{ silver_transactions }} AS txs on txs.tx_hash = r.tx_hash, lateral flatten(r.logs) v {% endmacro %} {% macro evm_live_view_silver_transactions(bronze_transactions, silver_blocks, silver_logs) %} -select +SELECT A.block_number AS block_number, A.data :blockHash::STRING AS block_hash, utils.udf_hex_to_int(A.data :blockNumber::STRING)::INT AS blockNumber, @@ -239,59 +238,162 @@ select r.type as tx_type, r.blob_gas_used, r.blob_gas_price, -from - {{ bronze_transactions }} A - left join {{ silver_blocks }} b on b.block_number = A.block_number - left join {{ silver_logs }} as r on r.tx_hash = A.data :hash::STRING +FROM + {{ bronze_transactions }} AS A + LEFT JOIN {{ silver_blocks }} AS b on b.block_number = A.block_number + LEFT JOIN {{ silver_logs }} AS r on r.tx_hash = A.data :hash::STRING {% endmacro %} -- Get EVM chain fact data -{% macro evm_live_view_fact_blocks(blockchain, network) %} - WITH spine AS ( +{% macro evm_live_view_fact_blocks(schema, blockchain, network) %} +WITH spine AS ( {{ evm_live_view_target_blocks(blockchain, network) | indent(4) -}} ), raw_block_txs AS ( {{ evm_live_view_bronze_blocks('spine') | indent(4) -}} + ), + silver_blocks AS ( + {{ evm_live_view_silver_blocks('raw_block_txs') | indent(4) -}} ) - {{ evm_live_view_silver_blocks('raw_block_txs') | indent(4) -}} + select + block_number, + block_timestamp, + '{{ network }}' AS network, + '{{ blockchain }}' AS blockchain, + tx_count, + difficulty, + total_difficulty, + extra_data, + gas_limit, + gas_used, + HASH, + parent_hash, + miner, + nonce, + receipts_root, + sha3_uncles, + SIZE, + uncles AS uncle_blocks, + OBJECT_CONSTRUCT( + 'baseFeePerGas', + base_fee_per_gas, + 'difficulty', + difficulty, + 'extraData', + extra_data, + 'gasLimit', + gas_limit, + 'gasUsed', + gas_used, + 'hash', + HASH, + 'logsBloom', + logs_bloom, + 'miner', + miner, + 'nonce', + nonce, + 'number', + NUMBER, + 'parentHash', + parent_hash, + 'receiptsRoot', + receipts_root, + 'sha3Uncles', + sha3_uncles, + 'size', + SIZE, + 'stateRoot', + state_root, + 'timestamp', + block_timestamp, + 'totalDifficulty', + total_difficulty, + 'transactionsRoot', + transactions_root, + 'uncles', + uncles, + 'excessBlobGas', + excess_blob_gas, + 'blobGasUsed', + blob_gas_used + ) AS block_header_json, + excess_blob_gas, + blob_gas_used, + block_number::STRING AS fact_blocks_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + withdrawals, + withdrawals_root + from silver_blocks {% endmacro %} -{% macro evm_live_view_fact_logs(blockchain, network) %} - WITH spine AS ( - {{ evm_live_view_target_blocks(blockchain, network) | indent(4) -}} - ), - raw_block_txs AS ( - {{ evm_live_view_bronze_blocks('spine') | indent(4) -}} - ), - raw_receipts AS ( - {{ evm_live_view_bronze_receipts('spine') | indent(4) -}} - ), - raw_logs AS ( - {{ evm_live_view_bronze_logs('raw_receipts') | indent(4) -}} - ), - raw_transactions AS ( - {{ evm_live_view_bronze_transactions('raw_receipts') | indent(4) -}} - ), - blocks AS ( - {{ evm_live_view_silver_blocks('raw_block_txs') | indent(4) -}} - ), - receipts AS ( - {{ evm_live_view_silver_receipts('raw_receipts') | indent(4) -}} - ), - transactions AS ( - {{ evm_live_view_silver_transactions('raw_transactions', 'blocks', 'receipts') | indent(4) -}} - ), - logs AS ( - {{ evm_live_view_silver_logs('receipts', 'transactions') | indent(4) -}} - ) - - SELECT * FROM logs +{% macro evm_live_view_fact_logs(schema, blockchain, network) %} +WITH spine AS ( + {{ evm_live_view_target_blocks(blockchain, network) | indent(4) -}} +), +raw_block_txs AS ( + {{ evm_live_view_bronze_blocks('spine') | indent(4) -}} +), +raw_receipts AS ( + {{ evm_live_view_bronze_receipts('spine') | indent(4) -}} +), +raw_logs AS ( + {{ evm_live_view_bronze_logs('raw_receipts') | indent(4) -}} +), +raw_transactions AS ( + {{ evm_live_view_bronze_transactions('raw_receipts') | indent(4) -}} +), +blocks AS ( + {{ evm_live_view_silver_blocks('raw_block_txs') | indent(4) -}} +), +receipts AS ( + {{ evm_live_view_silver_receipts('raw_receipts') | indent(4) -}} +), +transactions AS ( + {{ evm_live_view_silver_transactions('raw_transactions', 'blocks', 'receipts') | indent(4) -}} +), +logs AS ( + {{ evm_live_view_silver_logs('receipts', 'transactions') | indent(4) -}} +) +SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + event_index, + contract_address, + topics, + DATA, + event_removed, + tx_status, + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + md5( + cast( + coalesce( + cast(tx_hash as TEXT), + '_dbt_utils_surrogate_key_null_' + ) || '-' || coalesce( + cast(event_index as TEXT), + '_dbt_utils_surrogate_key_null_' + ) as TEXT + ) + ) AS fact_event_logs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM logs {% endmacro %} -- Get EVM chain ez data {% macro evm_live_view_ez_token_transfers(schema, blockchain, network) %} WITH fact_logs AS ( - {{ evm_live_view_fact_logs(blockchain, network) | indent(4) -}} + {{ evm_live_view_fact_logs(schema, blockchain, network) | indent(4) -}} ) SELECT From ee208b89372a6b1e9f28798d874433e5f157d1b9 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Fri, 11 Oct 2024 15:15:00 +0900 Subject: [PATCH 08/50] [STREAM-1049] fix tx's source table error --- macros/evm/evm_live_views.sql | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index bcc9dec..a7218d1 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -1,7 +1,8 @@ {% macro evm_live_view_latest_block_height(blockchain, network) %} SELECT live.udf_api( - '{service}/{Authentication}', + {# '{service}/{Authentication}', #} + 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', utils.udf_json_rpc_call( 'eth_blockNumber', [] @@ -46,7 +47,8 @@ SELECT block_number, live.udf_api( - '{service}/{Authentication}', + {# '{service}/{Authentication}', #} + 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', utils.udf_json_rpc_call( 'eth_getBlockByNumber', [utils.udf_int_to_hex(block_number), true] @@ -61,7 +63,8 @@ SELECT latest_block_height, block_number, live.udf_api( - '{service}/{Authentication}', + {# '{service}/{Authentication}', #} + 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', utils.udf_json_rpc_call( 'eth_getBlockReceipts', [utils.udf_int_to_hex(block_number)] @@ -192,7 +195,7 @@ FROM lateral flatten(r.logs) v {% endmacro %} -{% macro evm_live_view_silver_transactions(bronze_transactions, silver_blocks, silver_logs) %} +{% macro evm_live_view_silver_transactions(bronze_transactions, silver_blocks, silver_receipts) %} SELECT A.block_number AS block_number, A.data :blockHash::STRING AS block_hash, @@ -241,7 +244,7 @@ SELECT FROM {{ bronze_transactions }} AS A LEFT JOIN {{ silver_blocks }} AS b on b.block_number = A.block_number - LEFT JOIN {{ silver_logs }} AS r on r.tx_hash = A.data :hash::STRING + LEFT JOIN {{ silver_receipts }} AS r on r.tx_hash = A.data :hash::STRING {% endmacro %} -- Get EVM chain fact data @@ -342,7 +345,7 @@ raw_logs AS ( {{ evm_live_view_bronze_logs('raw_receipts') | indent(4) -}} ), raw_transactions AS ( - {{ evm_live_view_bronze_transactions('raw_receipts') | indent(4) -}} + {{ evm_live_view_bronze_transactions('raw_block_txs') | indent(4) -}} ), blocks AS ( {{ evm_live_view_silver_blocks('raw_block_txs') | indent(4) -}} From f4b9680c6650661e44e439894fc7251e73f56b85 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Fri, 11 Oct 2024 15:18:32 +0900 Subject: [PATCH 09/50] [STREAM-1052] update generator to get 1000 rows/blocks per call --- macros/evm/evm_live_views.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index a7218d1..1011274 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -37,7 +37,7 @@ ) AS max_height, latest_block_height FROM - TABLE(generator(ROWCOUNT => 500)), + TABLE(generator(ROWCOUNT => 1000)), heights qualify block_number BETWEEN min_height AND max_height {% endmacro %} From c9699bb69701c216c6d8adfc7f5fa3b8fcba4fec Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Fri, 11 Oct 2024 15:20:50 +0900 Subject: [PATCH 10/50] [STREAM-1052] fix typo --- macros/evm/evm_live_views.sql | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 1011274..d0ae939 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -1,8 +1,7 @@ {% macro evm_live_view_latest_block_height(blockchain, network) %} SELECT live.udf_api( - {# '{service}/{Authentication}', #} - 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', + '{service}/{Authentication}', utils.udf_json_rpc_call( 'eth_blockNumber', [] @@ -47,8 +46,7 @@ SELECT block_number, live.udf_api( - {# '{service}/{Authentication}', #} - 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', + '{service}/{Authentication}', utils.udf_json_rpc_call( 'eth_getBlockByNumber', [utils.udf_int_to_hex(block_number), true] @@ -63,8 +61,7 @@ SELECT latest_block_height, block_number, live.udf_api( - {# '{service}/{Authentication}', #} - 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', + '{service}/{Authentication}', utils.udf_json_rpc_call( 'eth_getBlockReceipts', [utils.udf_int_to_hex(block_number)] From 7d3b4769acaf9dc19527b8784f39d77410a735c4 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Fri, 11 Oct 2024 15:22:46 +0900 Subject: [PATCH 11/50] [STREAM-1052] rename table functions --- macros/evm/evm.yaml.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index e4f5ca0..d2a9327 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -363,7 +363,7 @@ sql: | {{ evm_latest_contract_events_decoded_ai(schema, blockchain, network) | indent(4) -}} -- name: {{ schema -}}.tf_live_view_fact_blocks +- name: {{ schema -}}.tf_fact_blocks signature: - [block_height, INTEGER, The start block height to get the transfers from] - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] @@ -377,7 +377,7 @@ sql: | {{ evm_live_view_fact_blocks(schema, blockchain, network) | indent(4) -}} -- name: {{ schema -}}.tf_live_view_fact_logs +- name: {{ schema -}}.tf_fact_logs signature: - [block_height, INTEGER, The start block height to get the logs from] - [to_latest, BOOLEAN, Whether to continue fetching logs until the latest block or not] @@ -391,7 +391,7 @@ sql: | {{ evm_live_view_fact_logs(schema, blockchain, network) | indent(4) -}} -- name: {{ schema -}}.tf_live_view_ez_native_transfers +- name: {{ schema -}}.tf_ez_native_transfers signature: - [block_height, INTEGER, The start block height to get the transfers from] - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] @@ -406,7 +406,7 @@ sql: | {{ evm_live_view_ez_native_transfers(schema, blockchain, network) | indent(4) -}} -- name: {{ schema -}}.tf_live_view_ez_token_transfers +- name: {{ schema -}}.tf_ez_token_transfers signature: - [block_height, INTEGER, The start block height to get the transfers from] - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] From abc1bfb82fb732f42f465439428faaffea3d70fb Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Fri, 11 Oct 2024 20:47:13 +0900 Subject: [PATCH 12/50] [STREAM-1052] fix min heights --- macros/evm/evm_live_views.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index d0ae939..886af6a 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -27,7 +27,7 @@ ROW_NUMBER() OVER ( ORDER BY NULL - ) - 1 + COALESCE(block_height, 0)::integer AS block_number, + ) - 1 + COALESCE(block_height, latest_block_height)::integer AS block_number, min_height, IFF( COALESCE(to_latest, false), From f05c5ae3517b57f33b3784c344a628e5107e2019 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Sat, 12 Oct 2024 01:07:27 +0900 Subject: [PATCH 13/50] [STREAM-1055] add demo plotcard app to show latest block number with the udtf --- .gitignore | 6 +- apps/README.md | 45 ++++++++ apps/__init__.py | 0 apps/app.py | 111 ++++++++++++++++++++ apps/dashboards/__init__.py | 0 apps/dashboards/realtime_monitoring.py | 139 +++++++++++++++++++++++++ apps/env.sample | 7 ++ apps/table_list.yaml | 35 +++++++ requirements.txt | 5 +- 9 files changed, 346 insertions(+), 2 deletions(-) create mode 100644 apps/README.md create mode 100644 apps/__init__.py create mode 100644 apps/app.py create mode 100644 apps/dashboards/__init__.py create mode 100644 apps/dashboards/realtime_monitoring.py create mode 100644 apps/env.sample create mode 100644 apps/table_list.yaml diff --git a/.gitignore b/.gitignore index 5094746..b127376 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,8 @@ dbt-env/ .env .* # KEEP -!.github/ \ No newline at end of file +!.github/ + +# Ignore Python bytecode files +*.pyc +__pycache__/ diff --git a/apps/README.md b/apps/README.md new file mode 100644 index 0000000..e7b1599 --- /dev/null +++ b/apps/README.md @@ -0,0 +1,45 @@ +## Development + +1. cd your directory + + ```bash + cd apps + ``` + +2. Setup a virtual environment + + ```bash + conda create -n "livequery-models" python=3.9 + ``` + +3. Activate the virtual environment + + ```bash + conda activate livequery-models + ``` + +4. Create a `.env` file from the env.sample file and run, + + ```base + set -a; source .env; set +a + ``` + +5. Install + + ```bash + pip install -r ../requirements.txt + ``` + +6. Add PYTHONPATH + + ```bash + export PYTHONPATH=. + ``` + +7. Run the app + + ```bash + wave run app.py + ``` + +7. Visit the demo at [http://localhost:10101/demo](http://127.0.0.1:10101/demo) diff --git a/apps/__init__.py b/apps/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/app.py b/apps/app.py new file mode 100644 index 0000000..e717c3f --- /dev/null +++ b/apps/app.py @@ -0,0 +1,111 @@ +import time +from h2o_wave import app, data, ui, Q, main + +from dashboards.realtime_monitoring import execute_query +import logging + + +logging.basicConfig(level=logging.INFO) + + +light_theme_colors = "$red $pink $purple $violet $indigo $blue $azure $cyan $teal $mint $green $amber $orange $tangerine".split() # noqa: E501 +dark_theme_colors = "$red $pink $blue $azure $cyan $teal $mint $green $lime $yellow $amber $orange $tangerine".split() + +_color_index = -1 +colors = dark_theme_colors + + +def next_color(): + global _color_index + _color_index += 1 + return colors[_color_index % len(colors)] + + +_curve_index = -1 +curves = "linear smooth step step-after step-before".split() + + +def next_curve(): + global _curve_index + _curve_index += 1 + return curves[_curve_index % len(curves)] + +async def update_realtime_dataset(q): + async for data_table in execute_query(): + try: + latest_block_number = max( + row["BLOCK_NUMBER"] for row in data_table.to_pylist() + ) + q.page["block_number_card"].value = str(latest_block_number) + except Exception as e: + print(e) + + # try: + # import pyarrow as pa + # import pyarrow.compute as pc + + # # Extract block_timestamp column + # block_timestamps = data_table.column("BLOCK_TIMESTAMP") + + # # Convert timestamps to minutes + # block_timestamps_minute = pc.strftime(block_timestamps, format="%Y-%m-%d %H:%M") + + # # Group by minute and count + # table = pa.table([block_timestamps_minute], names=["minute"]) + # grouped_data = table.group_by("minute").aggregate([("minute", "count")]) + + # # Convert grouped data to list and assign it back + # existing_data = list(q.page["line_plot"].data) # Convert Ref to list + # new_data = existing_data + [[row["minute"], row["minute_count"]] for row in grouped_data.to_pylist()] # Append new entries + # q.page["line_plot"].data = new_data # Assign back to Ref + # except Exception as e: + # print(e) + + # try: + # rows = [ + # ui.table_row( + # name=str(i), + # cells=[current_time, str(len(row))] + # ) + # for i, row in enumerate(data_table.to_pylist()) + # ] + # q.page["data_table"].rows = rows + # except Exception as e: + # print(e) + + await q.page.save() + +@app("/demo", mode='broadcast') +async def create_dashboard(q: Q): + q.page.drop() + line_plot = ui.plot_card( + box="1 3 4 4", + title="Token Transfers Data Size Over Time", + data=data("time length", -100), + plot=ui.plot( + [ui.mark(type="line", x="=time", y="=length", curve="smooth")] + ), + ) + + block_number_card = ui.small_stat_card(box="1 1 2 2", title="Latest Block Number", value="0") + + # data_table = ui.table( + # name='data_table', + # columns=[ + # ui.table_column(name='time', label='Time', sortable=True), + # ui.table_column(name='length', label='Data Length', sortable=True), + # ], + # rows=[], + # pagination=ui.table_pagination(total_rows=100, rows_per_page=100) + # ) + + # q.page["line_plot"] = line_plot + q.page["block_number_card"] = block_number_card + # q.page["data_table"] = data_table + + await q.page.save() + + try: + await update_realtime_dataset(q) + finally: + await q.close() diff --git a/apps/dashboards/__init__.py b/apps/dashboards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/dashboards/realtime_monitoring.py b/apps/dashboards/realtime_monitoring.py new file mode 100644 index 0000000..2aba2a5 --- /dev/null +++ b/apps/dashboards/realtime_monitoring.py @@ -0,0 +1,139 @@ +from h2o_wave import ui, data, Q +import pyarrow as pa +from pyarrow import compute as pc +import asyncio +import time +from concurrent.futures import ThreadPoolExecutor +import snowflake.connector +from snowflake.connector import ProgrammingError +import os +import yaml + +async def pull_data_from_snowflake(query: str): + with snowflake.connector.connect( + user=os.getenv("SNOWFLAKE_USER"), + password=os.getenv("SNOWFLAKE_PASSWORD"), + account=os.getenv("SNOWFLAKE_ACCOUNT"), + database=os.getenv("SNOWFLAKE_DATABASE"), + schema=os.getenv("SNOWFLAKE_SCHEMA"), + warehouse=os.getenv("SNOWFLAKE_WAREHOUSE"), + role=os.getenv("SNOWFLAKE_ROLE"), + session_parameters={ + "QUERY_TAG": "realtime_monitoring", + }, + ) as conn: + cur = conn.cursor() + try: + cur.execute_async(query) + query_id = cur.sfqid + + try: + while conn.is_still_running(conn.get_query_status_throw_if_error(query_id)): + time.sleep(1) + except ProgrammingError as err: + print('Programming Error: {0}'.format(err)) + + results = [] + cur.get_results_from_sfqid(query_id) + + #TODO: work with PyArrow and yield in batches to avoid OOM + for row in cur.fetch_arrow_batches(): + results.extend(row.to_pylist()) + + return results + finally: + cur.close() + + + +QUERY = """ + SELECT * + FROM TABLE({SCHEMA}.{TABLE_FUNCTION_NAME}( + {table_function_args} + )); +""" + +def load_table_list_from_yaml(yaml_file): + with open(yaml_file, 'r') as file: + data = yaml.safe_load(file) + return data + +# table_lists = load_table_list_from_yaml('apps/table_list.yaml') + +async def fetch_data_for_schema_table(schema, table, args): + latest_block_height = 0 + query = QUERY.format( + DATABASE=os.getenv("SNOWFLAKE_DATABASE"), + SCHEMA=schema, + TABLE_FUNCTION_NAME=table, + table_function_args=args, + ) + results = await pull_data_from_snowflake(query) + if results: + results_table = pa.Table.from_pylist(results) + latest_block_height = pc.max(results_table.column('BLOCK_NUMBER')).as_py() + + return (latest_block_height, results_table) + +# async def gather_tasks_and_parse_args(q: Q): +# """ +# Gather tasks and parse args for each table and network. +# """ +# table_lists = load_table_list_from_yaml('apps/table_list.yaml') +# tables = table_lists['tables'] + +# loop = asyncio.get_event_loop() +# with ThreadPoolExecutor() as executor: +# tasks = [] +# for table in tables: +# blockchain = table['blockchain'] +# for network in table['networks']: +# for network_name, network_data in network.items(): +# schema = f"{blockchain}_{network_name}" +# for table_data in network_data['tables']: +# table_name = table_data['name'] +# args = {**network_data['default_args'], **table_data.get('extra_args', {})} +# formatted_args = ', '.join(f"{key}={arg['value']}::{arg['type']}" for key, arg in args.items()) +# tasks.append( +# loop.run_in_executor( +# executor, fetch_data_for_schema_table, q, schema, table_name, formatted_args +# ) +# ) +# await asyncio.gather(*tasks) + +async def execute_query() -> pa.Table: + table_lists = { + "schema": "ethereum_mainnet", + "table": "tf_ez_token_transfers", + "args": [ + { + "block_height": "NULL::INTEGER", + "to_latest": "TRUE::BOOLEAN", + "ez_token_transfers_id": "'0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef'::STRING" + } + ] + } + + initial_args = ', '.join(f"{value}" for arg in table_lists['args'] for _, value in arg.items()) + latest_block_height, data_table = await fetch_data_for_schema_table(table_lists['schema'], table_lists['table'], initial_args) + + # Forever loop to fetch new data + while True: + table_lists['args'][0]['block_height'] = f"{latest_block_height}::INTEGER" + updated_args = ', '.join(f"{value}" for arg in table_lists['args'] for _, value in arg.items()) + latest_block_height, data_table = await fetch_data_for_schema_table(table_lists['schema'], table_lists['table'], updated_args) + + yield data_table + +if __name__ == "__main__": + # data = asyncio.run(execute_query()) + import logging + + logging.basicConfig(level=logging.INFO) + + async def log_data(): + async for data_table in execute_query(): + data_array = data_table.to_pylist() + logging.info(data_array) + + asyncio.run(log_data()) diff --git a/apps/env.sample b/apps/env.sample new file mode 100644 index 0000000..c7ba6fd --- /dev/null +++ b/apps/env.sample @@ -0,0 +1,7 @@ +SNOWFLAKE_ACCOUNT='vna27887.us-east-1' +SNOWFLAKE_ROLE='ACCOUNTADMIN' +SNOWFLAKE_USER='<>' +SNOWFLAKE_PASSWORD='<>' +SNOWFLAKE_REGION='us-east-1' +SNOWFLAKE_DATABASE='LIVEQUERY_DEV' +SNOWFLAKE_WAREHOUSE='INTERNAL_DEV' diff --git a/apps/table_list.yaml b/apps/table_list.yaml new file mode 100644 index 0000000..4adb06b --- /dev/null +++ b/apps/table_list.yaml @@ -0,0 +1,35 @@ +tables: + - blockchain: ethereum + networks: + - mainnet: + tables: + - name: tf_fact_blocks + - name: tf_fact_logs + - name: tf_ez_token_transfers + extra_args: + ez_token_transfers_id: + value: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' + type: STRING + default_args: + block_height: + value: NULL + type: INTEGER + to_latest: + value: TRUE + type: BOOLEAN + - sepolia: + tables: + - name: tf_fact_blocks + - name: tf_fact_logs + - name: tf_ez_token_transfers + extra_args: + ez_token_transfers_id: + value: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' + type: STRING + default_args: + block_height: + value: NULL + type: INTEGER + to_latest: + value: TRUE + type: BOOLEAN diff --git a/requirements.txt b/requirements.txt index ff8e7ca..29a3fa6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,4 @@ -dbt-snowflake~=1.5.0 \ No newline at end of file +dbt-snowflake~=1.5.0 +snowflake-connector-python~=3.12.2 +pyarrow~=17.0.0 +h2o-wave~=1.5.1 From 5bc156eb4c0b10af6c52899fcb94fb1e42ef0440 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Tue, 15 Oct 2024 00:46:02 +0900 Subject: [PATCH 14/50] [STREAM-1061] Add EVM Decoded Logs udtf --- macros/evm/evm.yaml.sql | 14 +++++ macros/evm/evm_live_views.sql | 101 ++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index d2a9327..4b207bd 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -391,6 +391,20 @@ sql: | {{ evm_live_view_fact_logs(schema, blockchain, network) | indent(4) -}} +- name: {{ schema -}}.tf_fact_decoded_event_logs + signature: + - [block_height, INTEGER, The start block height to get the logs from] + - [to_latest, BOOLEAN, Whether to continue fetching logs until the latest block or not] + return_type: + - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, event_index INTEGER, contract_address STRING, contract_name STRING, event_name STRING, decoded_log OBJECT, full_decoded_log VARIANT, origin_function_signature STRING, origin_from_address STRING, origin_to_address STRING, topics VARIANT, data STRING, event_removed STRING, tx_status STRING, _log_id STRING, fact_event_logs_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + COMMENT = $$Returns the decoded event logs data for a given block height. If to_latest is true, it will continue fetching blocks until the latest block. Otherwise, it will fetch blocks until the block height is reached.$$ + sql: | + {{ evm_live_view_fact_decoded_event_logs(schema, blockchain, network) | indent(4) -}} + - name: {{ schema -}}.tf_ez_native_transfers signature: - [block_height, INTEGER, The start block height to get the transfers from] diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 886af6a..5605f06 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -390,6 +390,107 @@ SELECT FROM logs {% endmacro %} +{% macro evm_live_view_fact_decoded_event_logs(schema, blockchain, network) %} +WITH _fact_event_logs AS ( + {{ evm_live_view_fact_logs(schema, blockchain, network) | indent(4) -}} +), + +_silver_decoded_logs AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + event_index, + topics, + DATA, + contract_address, + OBJECT_CONSTRUCT('topics', topics, 'data', data, 'address', contract_address) AS event_data, + abi, + utils.udf_evm_decode_log(abi, event_data)[0] AS decoded_data, + event_removed, + decoded_data:name::string AS event_name, + {{ blockchain }}.utils.udf_transform_logs(decoded_data) AS transformed, + _log_id, + inserted_timestamp, + tx_status + FROM + _fact_event_logs + JOIN + {{ blockchain }}.core.dim_contract_abis + USING + (contract_address) + WHERE + tx_status = 'SUCCESS' +), + +_flatten_logs AS ( + SELECT + b.tx_hash, + b.block_number, + b.event_index, + b.event_name, + b.contract_address, + b.decoded_data, + b.transformed, + b._log_id, + b.inserted_timestamp, + OBJECT_AGG( + DISTINCT CASE + WHEN v.value :name = '' THEN CONCAT( + 'anonymous_', + v.index + ) + ELSE v.value :name + END, + v.value :value + ) AS decoded_flat + FROM + _silver_decoded_logs b, + LATERAL FLATTEN( + input => b.transformed :data + ) v + GROUP BY + b.tx_hash, + b.block_number, + b.event_index, + b.event_name, + b.contract_address, + b.decoded_data, + b.transformed, + b._log_id, + b.inserted_timestamp +) + +SELECT + block_number, + C.block_timestamp, + B.tx_hash, + B.event_index, + B.contract_address, + D.name AS contract_name, + B.event_name, + B.decoded_flat AS decoded_log, + B.decoded_data AS full_decoded_log, + C.origin_function_signature, + C.origin_from_address, + C.origin_to_address, + C.topics, + C.DATA, + C.event_removed :: STRING AS event_removed, + C.tx_status, + _log_id, + md5(_log_id) AS fact_event_logs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM _flatten_logs AS B +LEFT JOIN _silver_decoded_logs AS C USING (block_number, _log_id) +LEFT JOIN {{ blockchain }}.core.dim_contracts AS D + ON B.contract_address = D.address +{% endmacro %} + -- Get EVM chain ez data {% macro evm_live_view_ez_token_transfers(schema, blockchain, network) %} WITH fact_logs AS ( From 5f956ce61d48b162f799e5cf9dbbc4ffa825fe80 Mon Sep 17 00:00:00 2001 From: shah Date: Mon, 14 Oct 2024 18:24:23 -0700 Subject: [PATCH 15/50] STREAM-1051 add ez_native_transfers sql --- macros/evm/evm.yaml.sql | 3 +- macros/evm/evm_live_views.sql | 683 +++++++++++++++++++++++++++++----- 2 files changed, 586 insertions(+), 100 deletions(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index 4b207bd..152fc32 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -409,9 +409,8 @@ signature: - [block_height, INTEGER, The start block height to get the transfers from] - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] - - [native_token_address, STRING, The address of the native token to get the transfers of] return_type: - - "TABLE(tx_hash STRING, block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_position INTEGER, trace_index INTEGER, identifier STRING, origin_from_address STRING, origin_to_address STRING, origin_function_signature STRING, from_address STRING, to_address STRING, amount FLOAT, amount_precise_raw NUMBER, amount_precise FLOAT, amount_usd FLOAT, ez_native_transfers_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + - "TABLE(tx_hash STRING, block_number NUMBER(38,0), block_timestamp TIMESTAMP_NTZ(9), tx_position NUMBER(38,0), trace_index NUMBER(19,0), identifier STRING, origin_from_address STRING, origin_to_address STRING, origin_function_signature STRING, from_address STRING, to_address STRING, amount FLOAT, amount_precise_raw STRING, amount_precise STRING, amount_usd FLOAT, ez_native_transfers_id STRING, inserted_timestamp TIMESTAMP_NTZ(9), modified_timestamp TIMESTAMP_NTZ(9))" options: | NOT NULL RETURNS NULL ON NULL INPUT diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 5605f06..ff4b03b 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -560,112 +560,599 @@ AND from_address IS NOT NULL {% endmacro %} {% macro evm_live_view_ez_native_transfers(schema, blockchain, network) %} -WITH spine AS ( - {{ evm_live_view_target_blocks(blockchain, network) }} -), - -raw_block_data AS ( +WITH heights AS ( SELECT - s.block_number AS block_number, - live.udf_api( - '{service}/{Authentication}', - utils.udf_json_rpc_call( - 'eth_getBlockByNumber', - [utils.udf_int_to_hex(s.block_number), true] + livequery_dev.live.udf_api( + 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', + livequery_Dev.utils.udf_json_rpc_call( + 'eth_blockNumber', + [] ) - ):data AS block_data, - b.value AS tx_data, - TO_TIMESTAMP_NTZ(utils.udf_hex_to_int(block_data:result:timestamp::string)) AS block_timestamp, - tx_data:hash::string AS tx_hash, - tx_data:from::string AS from_address, - tx_data:to_address::string AS to_address, - TRY_TO_NUMBER(utils.udf_hex_to_int(tx_data:value::string), 38, 0) / 1e18 AS eth_value, - TRY_TO_NUMBER(utils.udf_hex_to_int(tx_data:value::string), 38, 0) AS eth_value_precise_raw, - TRY_TO_NUMBER(utils.udf_hex_to_int(tx_data:value::string), 38, 0) / 1e18 AS eth_value_precise, - tx_data:input::string AS input, - utils.udf_hex_to_int(tx_data:transactionIndex::string)::INTEGER AS tx_position, - 'CALL' AS TYPE, - 'SUCCESS' AS tx_status, - 'SUCCESS' AS trace_status, - CASE - WHEN LEFT(input, 10) = '0x' THEN SUBSTRING(input, 1, 10) - ELSE NULL - END AS origin_function_signature, - 'native_transfer' AS identifier, - NULL AS trace_index + ):data AS result, + livequery_dev.utils.udf_hex_to_int(result:result)::integer as latest_block_height, + coalesce( + arg_block_height, + latest_block_height + ) as min_height, + iff( + coalesce(arg_to_latest, false), + latest_block_height, + min_height + ) as max_height + ), + spine as ( + select + row_number() over ( + order by + null + ) -1 + coalesce(arg_block_height, 0)::integer as block_number, + min_height, + iff( + coalesce(arg_to_latest, false), + latest_block_height, + min_height + ) as max_height, + latest_block_height + from + table(generator(ROWCOUNT => 500)), + heights + qualify block_number between min_height and max_height + ), + raw_receipts as ( + SELECT + latest_block_height, + block_number, + livequery_dev.live.udf_api( + 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', + livequery_Dev.utils.udf_json_rpc_call( + 'eth_getBlockReceipts', + [livequery_dev.utils.udf_int_to_hex(block_number)] + ) + ):data.result AS result, + v.value as DATA + from + spine, + lateral flatten(result) v + ), + raw_block_txs as ( + SELECT + block_number, + livequery_dev.live.udf_api( + 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', + livequery_Dev.utils.udf_json_rpc_call( + 'eth_getBlockByNumber', + [livequery_dev.utils.udf_int_to_hex(block_number), true] + ) + ):data.result AS DATA + from + spine + ), + raw_txs as ( + SELECT + block_number, + v.value as DATA + from + raw_block_txs r, + lateral flatten(r.data:transactions) v + ), + blocks as ( + select + block_number, + livequery_dev.utils.udf_hex_to_int(DATA :baseFeePerGas::STRING)::INT AS base_fee_per_gas, + livequery_dev.utils.udf_hex_to_int(DATA :difficulty::STRING)::INT AS difficulty, + DATA :extraData::STRING AS extra_data, + livequery_dev.utils.udf_hex_to_int(DATA :gasLimit::STRING)::INT AS gas_limit, + livequery_dev.utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS gas_used, + DATA :hash::STRING AS HASH, + DATA :logsBloom::STRING AS logs_bloom, + DATA :miner::STRING AS miner, + livequery_dev.utils.udf_hex_to_int(DATA :nonce::STRING)::INT AS nonce, + livequery_dev.utils.udf_hex_to_int(DATA :number::STRING)::INT AS NUMBER, + DATA :parentHash::STRING AS parent_hash, + DATA :receiptsRoot::STRING AS receipts_root, + DATA :sha3Uncles::STRING AS sha3_uncles, + livequery_dev.utils.udf_hex_to_int(DATA :size::STRING)::INT AS SIZE, + DATA :stateRoot::STRING AS state_root, + livequery_dev.utils.udf_hex_to_int(DATA :timestamp::STRING)::TIMESTAMP AS block_timestamp, + livequery_dev.utils.udf_hex_to_int(DATA :totalDifficulty::STRING)::INT AS total_difficulty, + ARRAY_SIZE(DATA :transactions) AS tx_count, + DATA :transactionsRoot::STRING AS transactions_root, + DATA :uncles AS uncles, + DATA :withdrawals AS withdrawals, + DATA :withdrawalsRoot::STRING AS withdrawals_root, + md5( + cast( + coalesce( + cast(block_number as TEXT), + '_dbt_utils_surrogate_key_null_' + ) as TEXT + ) + ) AS blocks_id, + livequery_dev.utils.udf_hex_to_int(DATA: blobGasUsed::STRING)::INT AS blob_gas_used, + livequery_dev.utils.udf_hex_to_int(DATA: excessBlobGas::STRING)::INT AS excess_blob_gas, + from + raw_block_txs + ), + receipts as ( + select + latest_block_height, + block_number, + DATA :blockHash::STRING AS block_hash, + livequery_dev.utils.udf_hex_to_int(DATA :blockNumber::STRING)::INT AS blockNumber, + livequery_dev.utils.udf_hex_to_int(DATA :cumulativeGasUsed::STRING)::INT AS cumulative_gas_used, + livequery_dev.utils.udf_hex_to_int(DATA :effectiveGasPrice::STRING)::INT / pow(10, 9) AS effective_gas_price, + DATA :from::STRING AS from_address, + livequery_dev.utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS gas_used, + DATA :logs AS logs, + DATA :logsBloom::STRING AS logs_bloom, + livequery_dev.utils.udf_hex_to_int(DATA :status::STRING)::INT AS status, + CASE + WHEN status = 1 THEN TRUE + ELSE FALSE + END AS tx_success, + CASE + WHEN status = 1 THEN 'SUCCESS' + ELSE 'FAIL' + END AS tx_status, + DATA :to::STRING AS to_address1, + CASE + WHEN to_address1 = '' THEN NULL + ELSE to_address1 + END AS to_address, + DATA :transactionHash::STRING AS tx_hash, + livequery_dev.utils.udf_hex_to_int(DATA :transactionIndex::STRING)::INT AS POSITION, + livequery_dev.utils.udf_hex_to_int(DATA :type::STRING)::INT AS TYPE, + livequery_dev.utils.udf_hex_to_int(DATA :effectiveGasPrice::STRING)::INT AS blob_gas_price, + livequery_dev.utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS blob_gas_used + from + raw_receipts + ), + txs as ( + select + A.block_number AS block_number, + A.data :blockHash::STRING AS block_hash, + livequery_dev.utils.udf_hex_to_int(A.data :blockNumber::STRING)::INT AS blockNumber, + livequery_dev.utils.udf_hex_to_int(A.data :chainId::STRING)::INT AS chain_id, + A.data :from::STRING AS from_address, + livequery_dev.utils.udf_hex_to_int(A.data :gas::STRING)::INT AS gas, + livequery_dev.utils.udf_hex_to_int(A.data :gasPrice::STRING)::INT / pow(10, 9) AS gas_price, + A.data :hash::STRING AS tx_hash, + A.data :input::STRING AS input_data, + SUBSTR(input_data, 1, 10) AS origin_function_signature, + livequery_dev.utils.udf_hex_to_int(A.data :maxFeePerGas::STRING)::INT / pow(10, 9) AS max_fee_per_gas, + livequery_dev.utils.udf_hex_to_int( + A.data :maxPriorityFeePerGas::STRING + )::INT / pow(10, 9) AS max_priority_fee_per_gas, + livequery_dev.utils.udf_hex_to_int(A.data :nonce::STRING)::INT AS nonce, + A.data :r::STRING AS r, + A.data :s::STRING AS s, + A.data :to::STRING AS to_address1, + livequery_dev.utils.udf_hex_to_int(A.data :transactionIndex::STRING)::INT AS POSITION, + A.data :type::STRING AS TYPE, + A.data :v::STRING AS v, + livequery_dev.utils.udf_hex_to_int(A.data :value::STRING) AS value_precise_raw, + value_precise_raw * power(10, -18) AS value_precise, + value_precise::FLOAT AS VALUE, + A.data :accessList AS access_list, + A.data, + A.data: blobVersionedHashes::ARRAY AS blob_versioned_hashes, + livequery_dev.utils.udf_hex_to_int(A.data: maxFeePerGas::STRING)::INT AS max_fee_per_blob_gas, + block_timestamp, + CASE + WHEN block_timestamp IS NULL + OR tx_status IS NULL THEN TRUE + ELSE FALSE + END AS is_pending, + r.gas_used, + tx_success, + tx_status, + cumulative_gas_used, + effective_gas_price, + livequery_dev.utils.udf_hex_to_int(A.data :gasPrice) * power(10, -18) * r.gas_used AS tx_fee_precise, + COALESCE(tx_fee_precise::FLOAT, 0) AS tx_fee, + r.type as tx_type, + r.blob_gas_used, + r.blob_gas_price, + from + raw_txs A + left join blocks b on b.block_number = A.block_number + left join receipts as r on r.tx_hash = A.data :hash::STRING + ), + raw_traces AS ( + SELECT + s.block_number, + v.index::INT AS tx_position, + v.value:result AS full_traces, + SYSDATE() AS _inserted_timestamp + FROM spine s, + LATERAL FLATTEN(input => PARSE_JSON( + livequery_dev.live.udf_api( + 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', + livequery_Dev.utils.udf_json_rpc_call( + 'debug_traceBlockByNumber', + [livequery_dev.utils.udf_int_to_hex(s.block_number), {'tracer': 'callTracer'}] + ) + ):data.result + )) v + ), + flatten_traces AS ( + SELECT + block_number, + tx_position, + IFF( + path IN ( + 'result', + 'result.value', + 'result.type', + 'result.to', + 'result.input', + 'result.gasUsed', + 'result.gas', + 'result.from', + 'result.output', + 'result.error', + 'result.revertReason', + 'gasUsed', + 'gas', + 'type', + 'to', + 'from', + 'value', + 'input', + 'error', + 'output', + 'revertReason' + ), + 'ORIGIN', + REGEXP_REPLACE(REGEXP_REPLACE(path, '[^0-9]+', '_'), '^_|_$', '') + ) AS trace_address, + _inserted_timestamp, + OBJECT_AGG( + key, + VALUE + ) AS trace_json, + CASE + WHEN trace_address = 'ORIGIN' THEN NULL + WHEN POSITION( + '_' IN trace_address + ) = 0 THEN 'ORIGIN' + ELSE REGEXP_REPLACE( + trace_address, + '_[0-9]+$', + '', + 1, + 1 + ) + END AS parent_trace_address, + SPLIT( + trace_address, + '_' + ) AS str_array + FROM + raw_traces, + TABLE( + FLATTEN( + input => PARSE_JSON(full_traces), + recursive => TRUE + ) + ) f + WHERE + f.index IS NULL + AND f.key != 'calls' + AND f.path != 'result' + GROUP BY + block_number, + tx_position, + trace_address, + _inserted_timestamp + ), + sub_traces AS ( + SELECT + block_number, + tx_position, + parent_trace_address, + COUNT(*) AS sub_traces + FROM + flatten_traces + GROUP BY + block_number, + tx_position, + parent_trace_address + ), + num_array AS ( + SELECT + block_number, + tx_position, + trace_address, + ARRAY_AGG(flat_value) AS num_array + FROM + ( + SELECT + block_number, + tx_position, + trace_address, + IFF( + VALUE :: STRING = 'ORIGIN', + -1, + VALUE :: INT + ) AS flat_value + FROM + flatten_traces, + LATERAL FLATTEN ( + input => str_array + ) + ) + GROUP BY + block_number, + tx_position, + trace_address + ), + cleaned_traces AS ( + SELECT + b.block_number, + b.tx_position, + b.trace_address, + IFNULL( + sub_traces, + 0 + ) AS sub_traces, + num_array, + ROW_NUMBER() over ( + PARTITION BY b.block_number, + b.tx_position + ORDER BY + num_array ASC + ) - 1 AS trace_index, + trace_json, + b._inserted_timestamp + FROM + flatten_traces b + LEFT JOIN sub_traces s + ON b.block_number = s.block_number + AND b.tx_position = s.tx_position + AND b.trace_address = s.parent_trace_address + JOIN num_array n + ON b.block_number = n.block_number + AND b.tx_position = n.tx_position + AND b.trace_address = n.trace_address + ), + final_traces AS ( + SELECT + tx_position, + trace_index, + block_number, + trace_address, + trace_json :error :: STRING AS error_reason, + trace_json :from :: STRING AS from_address, + trace_json :to :: STRING AS to_address, + IFNULL( + utils.udf_hex_to_int( + trace_json :value :: STRING + ), + '0' + ) AS eth_value_precise_raw, + ethereum.utils.udf_decimal_adjust( + eth_value_precise_raw, + 18 + ) AS eth_value_precise, + eth_value_precise :: FLOAT AS eth_value, + utils.udf_hex_to_int( + trace_json :gas :: STRING + ) :: INT AS gas, + utils.udf_hex_to_int( + trace_json :gasUsed :: STRING + ) :: INT AS gas_used, + trace_json :input :: STRING AS input, + trace_json :output :: STRING AS output, + trace_json :type :: STRING AS TYPE, + concat_ws( + '_', + TYPE, + trace_address + ) AS identifier, + concat_ws( + '-', + block_number, + tx_position, + identifier + ) AS _call_id, + _inserted_timestamp, + trace_json AS DATA, + sub_traces + FROM + cleaned_traces + ), + new_records AS ( + SELECT + f.block_number, + t.tx_hash, + t.block_timestamp, + t.tx_status, + f.tx_position, + f.trace_index, + f.from_address, + f.to_address, + f.eth_value_precise_raw, + f.eth_value_precise, + f.eth_value, + f.gas, + f.gas_used, + f.input, + f.output, + f.type, + f.identifier, + f.sub_traces, + f.error_reason, + IFF( + f.error_reason IS NULL, + 'SUCCESS', + 'FAIL' + ) AS trace_status, + f.data, + IFF( + t.tx_hash IS NULL + OR t.block_timestamp IS NULL + OR t.tx_status IS NULL, + TRUE, + FALSE + ) AS is_pending, + f._call_id, + f._inserted_timestamp FROM - spine s, - LATERAL FLATTEN(input => block_data:result:transactions) b - WHERE - TRY_TO_NUMBER(utils.udf_hex_to_int(tx_data:value::string), 38, 0) > 0 -), -eth_base AS ( + final_traces f + LEFT OUTER JOIN ethereum.silver.transactions t + ON f.tx_position = t.position + AND f.block_number = t.block_number + ), + traces_final AS ( + SELECT + block_number, + tx_hash, + block_timestamp, + tx_status, + tx_position, + trace_index, + from_address, + to_address, + eth_value_precise_raw, + eth_value_precise, + eth_value, + gas, + gas_used, + input, + output, + TYPE, + identifier, + sub_traces, + error_reason, + trace_status, + DATA, + is_pending, + _call_id, + _inserted_timestamp + FROM + new_records + ), + eth_base AS ( + SELECT + tx_hash, + block_number, + block_timestamp, + identifier, + from_address, + to_address, + eth_value AS amount, + _call_id, + _inserted_timestamp, + eth_value_precise_raw AS amount_precise_raw, + eth_value_precise AS amount_precise, + tx_position, + trace_index + FROM + traces_final + WHERE + eth_value > 0 + AND tx_status = 'SUCCESS' + AND trace_status = 'SUCCESS' + AND TYPE NOT IN ( + 'DELEGATECALL', + 'STATICCALL' + ) + ), + tx_table AS ( + SELECT + block_number, + tx_hash, + from_address AS origin_from_address, + to_address1 AS origin_to_address, + origin_function_signature + FROM + txs + WHERE + tx_hash IN ( + SELECT + DISTINCT tx_hash + FROM + eth_base + ) + ), + native_transfers AS ( + SELECT + e.tx_hash, + e.block_number, + e.block_timestamp, + e.identifier, + t.origin_from_address, + t.origin_to_address, + t.origin_function_signature, + e.from_address, + e.to_address, + e.amount, + e.amount_precise_raw, + e.amount_precise, + ROUND( + e.amount * p.price, + 2 + ) AS amount_usd, + e._call_id, + e._inserted_timestamp, + e.tx_position, + e.trace_index, + md5( + cast( + coalesce(cast(e.tx_hash as TEXT), '_dbt_utils_surrogate_key_null_') + || '-' || coalesce(cast(e.trace_index as TEXT), '_dbt_utils_surrogate_key_null_') + as TEXT + ) + ) as native_transfers_id, + SYSDATE() as inserted_timestamp, + SYSDATE() as modified_timestamp + FROM + eth_base e + JOIN tx_table t ON e.tx_hash = t.tx_hash AND e.block_number = t.block_number + LEFT JOIN ETHEREUM.price.EZ_PRICES_HOURLY p + ON DATE_TRUNC('hour', e.block_timestamp) = p.HOUR + AND p.token_address = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' -- WETH address + ) SELECT tx_hash, block_number, block_timestamp, - identifier, - from_address, - to_address, - eth_value AS amount, - eth_value_precise_raw AS amount_precise_raw, - eth_value_precise AS amount_precise, tx_position, - trace_index + trace_index, + identifier AS trace_type, + origin_from_address, + origin_to_address, + origin_function_signature, + from_address AS trace_from_address, + to_address AS trace_to_address, + amount, + amount_precise_raw, + amount_precise, + amount_usd, + COALESCE( + native_transfers_id, + md5( + cast( + coalesce(cast(tx_hash as TEXT), '_dbt_utils_surrogate_key_null_') + || '-' || coalesce(cast(trace_index as TEXT), '_dbt_utils_surrogate_key_null_') + as TEXT + ) + ) + ) AS ez_native_transfers_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM - raw_block_data - WHERE - eth_value > 0 - AND tx_status = 'SUCCESS' - AND trace_status = 'SUCCESS' - AND TYPE NOT IN ('DELEGATECALL', 'STATICCALL') -), -tx_table AS ( - SELECT - block_number, - block_timestamp, - tx_hash, - from_address AS origin_from_address, - to_address AS origin_to_address, - origin_function_signature - FROM - raw_block_data - WHERE - tx_hash IN (SELECT DISTINCT tx_hash FROM eth_base) -), -price_data AS ( - SELECT - DATE_TRUNC('hour', e.block_timestamp) AS hour, - AVG(p.price) AS price - FROM - eth_base e - JOIN {{ blockchain }}.PRICE.EZ_PRICES_HOURLY p - ON DATE_TRUNC('hour', e.block_timestamp) = p.hour - AND p.token_address = native_token_address - GROUP BY 1 -) -SELECT - A.tx_hash, - A.block_number, - A.block_timestamp, - A.tx_position, - A.trace_index, - A.identifier, - T.origin_from_address, - T.origin_to_address, - T.origin_function_signature, - A.from_address, - A.to_address, - A.amount::FLOAT, - A.amount_precise_raw::NUMBER(38,0), - A.amount_precise::FLOAT, - ROUND(A.amount * P.price, 2)::FLOAT AS amount_usd, - MD5(CONCAT(A.tx_hash, '|', COALESCE(A.trace_index::STRING, ''))) AS ez_native_transfers_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp -FROM - eth_base A - LEFT JOIN price_data P ON DATE_TRUNC('hour', A.block_timestamp) = P.hour - JOIN tx_table T ON A.tx_hash = T.tx_hash AND A.block_number = T.block_number + native_transfers + QUALIFY (ROW_NUMBER() OVER ( + PARTITION BY block_number, tx_position, trace_index + ORDER BY _inserted_timestamp DESC + )) = 1 {% endmacro %} From 6657352df864e858aca645d4439f72818a4ba314 Mon Sep 17 00:00:00 2001 From: shah Date: Mon, 14 Oct 2024 19:36:50 -0700 Subject: [PATCH 16/50] STREAM-1051 Rename UDTF args --- macros/evm/evm_live_views.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index ff4b03b..9926cf0 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -571,11 +571,11 @@ WITH heights AS ( ):data AS result, livequery_dev.utils.udf_hex_to_int(result:result)::integer as latest_block_height, coalesce( - arg_block_height, + block_height, latest_block_height ) as min_height, iff( - coalesce(arg_to_latest, false), + coalesce(to_latest, false), latest_block_height, min_height ) as max_height @@ -585,10 +585,10 @@ WITH heights AS ( row_number() over ( order by null - ) -1 + coalesce(arg_block_height, 0)::integer as block_number, + ) -1 + coalesce(block_height, 0)::integer as block_number, min_height, iff( - coalesce(arg_to_latest, false), + coalesce(to_latest, false), latest_block_height, min_height ) as max_height, From e980409944365c18de1657b69a9a0e65a2e393e9 Mon Sep 17 00:00:00 2001 From: shah Date: Tue, 15 Oct 2024 16:26:17 -0700 Subject: [PATCH 17/50] STREAM-1062 add fact_traces UDTF --- macros/evm/evm.yaml.sql | 13 + macros/evm/evm_live_views.sql | 515 +++++++++++++++++++++++++++++++++- 2 files changed, 527 insertions(+), 1 deletion(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index 152fc32..a930ff7 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -404,6 +404,19 @@ COMMENT = $$Returns the decoded event logs data for a given block height. If to_latest is true, it will continue fetching blocks until the latest block. Otherwise, it will fetch blocks until the block height is reached.$$ sql: | {{ evm_live_view_fact_decoded_event_logs(schema, blockchain, network) | indent(4) -}} +- name: {{ schema -}}.tf_fact_traces + signature: + - [block_height, INTEGER, The start block height to get the transfers from] + - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] + return_type: + - "TABLE(tx_hash STRING, block_number NUMBER, block_timestamp TIMESTAMP_NTZ(9), from_address STRING, to_address STRING, eth_value FLOAT, eth_value_precise_raw STRING, eth_value_precise STRING, gas NUMBER, gas_used NUMBER, input STRING, output STRING, TYPE STRING, identifier STRING, DATA OBJECT, tx_status STRING, sub_traces NUMBER, trace_status STRING, error_reason STRING, trace_index NUMBER, fact_traces_id STRING, inserted_timestamp TIMESTAMP_NTZ(9), modified_timestamp TIMESTAMP_NTZ(9))" + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + COMMENT = $$Returns the traces for a given block height. If to_latest is true, it will continue fetching traces until the latest block. Otherwise, it will fetch traces until the block height is reached.$$ + sql: | + {{ evm_live_view_fact_traces(schema, blockchain, network) | indent(4) -}} - name: {{ schema -}}.tf_ez_native_transfers signature: diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 9926cf0..c871b81 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -491,6 +491,520 @@ LEFT JOIN {{ blockchain }}.core.dim_contracts AS D ON B.contract_address = D.address {% endmacro %} +{% macro evm_live_view_fact_traces(schema, blockchain, network) %} +WITH heights AS ( + SELECT + livequery_dev.live.udf_api( + 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', + livequery_Dev.utils.udf_json_rpc_call( + 'eth_blockNumber', + [] + ) + ):data AS result, + livequery_dev.utils.udf_hex_to_int(result:result)::integer as latest_block_height, + coalesce( + block_height, + latest_block_height + ) as min_height, + iff( + coalesce(to_latest, false), + latest_block_height, + min_height + ) as max_height + ), + spine as ( + select + row_number() over ( + order by + null + ) -1 + coalesce(block_height, 0)::integer as block_number, + min_height, + iff( + coalesce(to_latest, false), + latest_block_height, + min_height + ) as max_height, + latest_block_height + from + table(generator(ROWCOUNT => 500)), + heights + qualify block_number between min_height and max_height + ), + raw_receipts as ( + SELECT + latest_block_height, + block_number, + livequery_dev.live.udf_api( + 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', + livequery_Dev.utils.udf_json_rpc_call( + 'eth_getBlockReceipts', + [livequery_dev.utils.udf_int_to_hex(block_number)] + ) + ):data.result AS result, + v.value as DATA + from + spine, + lateral flatten(result) v + ), + raw_block_txs as ( + SELECT + block_number, + livequery_dev.live.udf_api( + 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', + livequery_Dev.utils.udf_json_rpc_call( + 'eth_getBlockByNumber', + [livequery_dev.utils.udf_int_to_hex(block_number), true] + ) + ):data.result AS DATA + from + spine + ), + raw_txs as ( + SELECT + block_number, + v.value as DATA + from + raw_block_txs r, + lateral flatten(r.data:transactions) v + ), + blocks as ( + select + block_number, + livequery_dev.utils.udf_hex_to_int(DATA :baseFeePerGas::STRING)::INT AS base_fee_per_gas, + livequery_dev.utils.udf_hex_to_int(DATA :difficulty::STRING)::INT AS difficulty, + DATA :extraData::STRING AS extra_data, + livequery_dev.utils.udf_hex_to_int(DATA :gasLimit::STRING)::INT AS gas_limit, + livequery_dev.utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS gas_used, + DATA :hash::STRING AS HASH, + DATA :logsBloom::STRING AS logs_bloom, + DATA :miner::STRING AS miner, + livequery_dev.utils.udf_hex_to_int(DATA :nonce::STRING)::INT AS nonce, + livequery_dev.utils.udf_hex_to_int(DATA :number::STRING)::INT AS NUMBER, + DATA :parentHash::STRING AS parent_hash, + DATA :receiptsRoot::STRING AS receipts_root, + DATA :sha3Uncles::STRING AS sha3_uncles, + livequery_dev.utils.udf_hex_to_int(DATA :size::STRING)::INT AS SIZE, + DATA :stateRoot::STRING AS state_root, + livequery_dev.utils.udf_hex_to_int(DATA :timestamp::STRING)::TIMESTAMP AS block_timestamp, + livequery_dev.utils.udf_hex_to_int(DATA :totalDifficulty::STRING)::INT AS total_difficulty, + ARRAY_SIZE(DATA :transactions) AS tx_count, + DATA :transactionsRoot::STRING AS transactions_root, + DATA :uncles AS uncles, + DATA :withdrawals AS withdrawals, + DATA :withdrawalsRoot::STRING AS withdrawals_root, + md5( + cast( + coalesce( + cast(block_number as TEXT), + '_dbt_utils_surrogate_key_null_' + ) as TEXT + ) + ) AS blocks_id, + livequery_dev.utils.udf_hex_to_int(DATA: blobGasUsed::STRING)::INT AS blob_gas_used, + livequery_dev.utils.udf_hex_to_int(DATA: excessBlobGas::STRING)::INT AS excess_blob_gas, + from + raw_block_txs + ), + receipts as ( + select + latest_block_height, + block_number, + DATA :blockHash::STRING AS block_hash, + livequery_dev.utils.udf_hex_to_int(DATA :blockNumber::STRING)::INT AS blockNumber, + livequery_dev.utils.udf_hex_to_int(DATA :cumulativeGasUsed::STRING)::INT AS cumulative_gas_used, + livequery_dev.utils.udf_hex_to_int(DATA :effectiveGasPrice::STRING)::INT / pow(10, 9) AS effective_gas_price, + DATA :from::STRING AS from_address, + livequery_dev.utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS gas_used, + DATA :logs AS logs, + DATA :logsBloom::STRING AS logs_bloom, + livequery_dev.utils.udf_hex_to_int(DATA :status::STRING)::INT AS status, + CASE + WHEN status = 1 THEN TRUE + ELSE FALSE + END AS tx_success, + CASE + WHEN status = 1 THEN 'SUCCESS' + ELSE 'FAIL' + END AS tx_status, + DATA :to::STRING AS to_address1, + CASE + WHEN to_address1 = '' THEN NULL + ELSE to_address1 + END AS to_address, + DATA :transactionHash::STRING AS tx_hash, + livequery_dev.utils.udf_hex_to_int(DATA :transactionIndex::STRING)::INT AS POSITION, + livequery_dev.utils.udf_hex_to_int(DATA :type::STRING)::INT AS TYPE, + livequery_dev.utils.udf_hex_to_int(DATA :effectiveGasPrice::STRING)::INT AS blob_gas_price, + livequery_dev.utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS blob_gas_used + from + raw_receipts + ), + txs as ( + select + A.block_number AS block_number, + A.data :blockHash::STRING AS block_hash, + livequery_dev.utils.udf_hex_to_int(A.data :blockNumber::STRING)::INT AS blockNumber, + livequery_dev.utils.udf_hex_to_int(A.data :chainId::STRING)::INT AS chain_id, + A.data :from::STRING AS from_address, + livequery_dev.utils.udf_hex_to_int(A.data :gas::STRING)::INT AS gas, + livequery_dev.utils.udf_hex_to_int(A.data :gasPrice::STRING)::INT / pow(10, 9) AS gas_price, + A.data :hash::STRING AS tx_hash, + A.data :input::STRING AS input_data, + SUBSTR(input_data, 1, 10) AS origin_function_signature, + livequery_dev.utils.udf_hex_to_int(A.data :maxFeePerGas::STRING)::INT / pow(10, 9) AS max_fee_per_gas, + livequery_dev.utils.udf_hex_to_int( + A.data :maxPriorityFeePerGas::STRING + )::INT / pow(10, 9) AS max_priority_fee_per_gas, + livequery_dev.utils.udf_hex_to_int(A.data :nonce::STRING)::INT AS nonce, + A.data :r::STRING AS r, + A.data :s::STRING AS s, + A.data :to::STRING AS to_address1, + livequery_dev.utils.udf_hex_to_int(A.data :transactionIndex::STRING)::INT AS POSITION, + A.data :type::STRING AS TYPE, + A.data :v::STRING AS v, + livequery_dev.utils.udf_hex_to_int(A.data :value::STRING) AS value_precise_raw, + value_precise_raw * power(10, -18) AS value_precise, + value_precise::FLOAT AS VALUE, + A.data :accessList AS access_list, + A.data, + A.data: blobVersionedHashes::ARRAY AS blob_versioned_hashes, + livequery_dev.utils.udf_hex_to_int(A.data: maxFeePerGas::STRING)::INT AS max_fee_per_blob_gas, + block_timestamp, + CASE + WHEN block_timestamp IS NULL + OR tx_status IS NULL THEN TRUE + ELSE FALSE + END AS is_pending, + r.gas_used, + tx_success, + tx_status, + cumulative_gas_used, + effective_gas_price, + livequery_dev.utils.udf_hex_to_int(A.data :gasPrice) * power(10, -18) * r.gas_used AS tx_fee_precise, + COALESCE(tx_fee_precise::FLOAT, 0) AS tx_fee, + r.type as tx_type, + r.blob_gas_used, + r.blob_gas_price, + from + raw_txs A + left join blocks b on b.block_number = A.block_number + left join receipts as r on r.tx_hash = A.data :hash::STRING + ), + raw_traces AS ( + SELECT + s.block_number, + v.index::INT AS tx_position, + v.value:result AS full_traces, + SYSDATE() AS _inserted_timestamp + FROM spine s, + LATERAL FLATTEN(input => PARSE_JSON( + livequery_dev.live.udf_api( + 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', + livequery_Dev.utils.udf_json_rpc_call( + 'debug_traceBlockByNumber', + [livequery_dev.utils.udf_int_to_hex(s.block_number), {'tracer': 'callTracer'}] + ) + ):data.result + )) v + ), + flatten_traces AS ( + SELECT + block_number, + tx_position, + IFF( + path IN ( + 'result', + 'result.value', + 'result.type', + 'result.to', + 'result.input', + 'result.gasUsed', + 'result.gas', + 'result.from', + 'result.output', + 'result.error', + 'result.revertReason', + 'gasUsed', + 'gas', + 'type', + 'to', + 'from', + 'value', + 'input', + 'error', + 'output', + 'revertReason' + ), + 'ORIGIN', + REGEXP_REPLACE(REGEXP_REPLACE(path, '[^0-9]+', '_'), '^_|_$', '') + ) AS trace_address, + _inserted_timestamp, + OBJECT_AGG( + key, + VALUE + ) AS trace_json, + CASE + WHEN trace_address = 'ORIGIN' THEN NULL + WHEN POSITION( + '_' IN trace_address + ) = 0 THEN 'ORIGIN' + ELSE REGEXP_REPLACE( + trace_address, + '_[0-9]+$', + '', + 1, + 1 + ) + END AS parent_trace_address, + SPLIT( + trace_address, + '_' + ) AS str_array + FROM + raw_traces, + TABLE( + FLATTEN( + input => PARSE_JSON(full_traces), + recursive => TRUE + ) + ) f + WHERE + f.index IS NULL + AND f.key != 'calls' + AND f.path != 'result' + GROUP BY + block_number, + tx_position, + trace_address, + _inserted_timestamp + ), + sub_traces AS ( + SELECT + block_number, + tx_position, + parent_trace_address, + COUNT(*) AS sub_traces + FROM + flatten_traces + GROUP BY + block_number, + tx_position, + parent_trace_address + ), + num_array AS ( + SELECT + block_number, + tx_position, + trace_address, + ARRAY_AGG(flat_value) AS num_array + FROM + ( + SELECT + block_number, + tx_position, + trace_address, + IFF( + VALUE :: STRING = 'ORIGIN', + -1, + VALUE :: INT + ) AS flat_value + FROM + flatten_traces, + LATERAL FLATTEN ( + input => str_array + ) + ) + GROUP BY + block_number, + tx_position, + trace_address + ), + cleaned_traces AS ( + SELECT + b.block_number, + b.tx_position, + b.trace_address, + IFNULL( + sub_traces, + 0 + ) AS sub_traces, + num_array, + ROW_NUMBER() over ( + PARTITION BY b.block_number, + b.tx_position + ORDER BY + num_array ASC + ) - 1 AS trace_index, + trace_json, + b._inserted_timestamp + FROM + flatten_traces b + LEFT JOIN sub_traces s + ON b.block_number = s.block_number + AND b.tx_position = s.tx_position + AND b.trace_address = s.parent_trace_address + JOIN num_array n + ON b.block_number = n.block_number + AND b.tx_position = n.tx_position + AND b.trace_address = n.trace_address + ), + final_traces AS ( + SELECT + tx_position, + trace_index, + block_number, + trace_address, + trace_json :error :: STRING AS error_reason, + trace_json :from :: STRING AS from_address, + trace_json :to :: STRING AS to_address, + IFNULL( + utils.udf_hex_to_int( + trace_json :value :: STRING + ), + '0' + ) AS eth_value_precise_raw, + ethereum.utils.udf_decimal_adjust( + eth_value_precise_raw, + 18 + ) AS eth_value_precise, + eth_value_precise :: FLOAT AS eth_value, + utils.udf_hex_to_int( + trace_json :gas :: STRING + ) :: INT AS gas, + utils.udf_hex_to_int( + trace_json :gasUsed :: STRING + ) :: INT AS gas_used, + trace_json :input :: STRING AS input, + trace_json :output :: STRING AS output, + trace_json :type :: STRING AS TYPE, + concat_ws( + '_', + TYPE, + trace_address + ) AS identifier, + concat_ws( + '-', + block_number, + tx_position, + identifier + ) AS _call_id, + _inserted_timestamp, + trace_json AS DATA, + sub_traces + FROM + cleaned_traces + ), + new_records AS ( + SELECT + f.block_number, + t.tx_hash, + t.block_timestamp, + t.tx_status, + f.tx_position, + f.trace_index, + f.from_address, + f.to_address, + f.eth_value_precise_raw, + f.eth_value_precise, + f.eth_value, + f.gas, + f.gas_used, + f.input, + f.output, + f.type, + f.identifier, + f.sub_traces, + f.error_reason, + IFF( + f.error_reason IS NULL, + 'SUCCESS', + 'FAIL' + ) AS trace_status, + f.data, + IFF( + t.tx_hash IS NULL + OR t.block_timestamp IS NULL + OR t.tx_status IS NULL, + TRUE, + FALSE + ) AS is_pending, + f._call_id, + f._inserted_timestamp + FROM + final_traces f + LEFT OUTER JOIN ethereum.silver.transactions t + ON f.tx_position = t.position + AND f.block_number = t.block_number + ), + traces_final AS ( + SELECT + block_number, + tx_hash, + block_timestamp, + tx_status, + tx_position, + trace_index, + from_address, + to_address, + eth_value_precise_raw, + eth_value_precise, + eth_value, + gas, + gas_used, + input, + output, + TYPE, + identifier, + sub_traces, + error_reason, + trace_status, + DATA, + is_pending, + _call_id, + _inserted_timestamp + FROM + new_records + ) + SELECT + tx_hash, + block_number, + block_timestamp, + from_address, + to_address, + eth_value AS VALUE, + eth_value_precise_raw AS value_precise_raw, + eth_value_precise AS value_precise, + gas, + gas_used, + input, + output, + TYPE, + identifier, + DATA, + tx_status, + sub_traces, + trace_status, + error_reason, + trace_index, + md5( + cast( + coalesce( + cast(tx_hash as TEXT), + '_dbt_utils_surrogate_key_null_' + ) || '-' || coalesce( + cast(trace_index as TEXT), + '_dbt_utils_surrogate_key_null_' + ) as TEXT + ) + ) as fact_traces_id, + COALESCE( + _inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + SYSDATE() AS modified_timestamp + FROM traces_final +{% endmacro %} + -- Get EVM chain ez data {% macro evm_live_view_ez_token_transfers(schema, blockchain, network) %} WITH fact_logs AS ( @@ -1155,4 +1669,3 @@ WITH heights AS ( ORDER BY _inserted_timestamp DESC )) = 1 {% endmacro %} - From b1dc0f81b01b4c608aef30f3cf3351170f676c60 Mon Sep 17 00:00:00 2001 From: shah Date: Tue, 15 Oct 2024 18:21:49 -0700 Subject: [PATCH 18/50] STREAM-1062 fix tf_fact_traces signature to match return column aliases --- macros/evm/evm.yaml.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index a930ff7..023c6a2 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -409,7 +409,7 @@ - [block_height, INTEGER, The start block height to get the transfers from] - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] return_type: - - "TABLE(tx_hash STRING, block_number NUMBER, block_timestamp TIMESTAMP_NTZ(9), from_address STRING, to_address STRING, eth_value FLOAT, eth_value_precise_raw STRING, eth_value_precise STRING, gas NUMBER, gas_used NUMBER, input STRING, output STRING, TYPE STRING, identifier STRING, DATA OBJECT, tx_status STRING, sub_traces NUMBER, trace_status STRING, error_reason STRING, trace_index NUMBER, fact_traces_id STRING, inserted_timestamp TIMESTAMP_NTZ(9), modified_timestamp TIMESTAMP_NTZ(9))" + - "TABLE(tx_hash STRING, block_number NUMBER, block_timestamp TIMESTAMP_NTZ(9), from_address STRING, to_address STRING, value FLOAT, value_precise_raw STRING, value_precise STRING, gas NUMBER, gas_used NUMBER, input STRING, output STRING, TYPE STRING, identifier STRING, DATA OBJECT, tx_status STRING, sub_traces NUMBER, trace_status STRING, error_reason STRING, trace_index NUMBER, fact_traces_id STRING, inserted_timestamp TIMESTAMP_NTZ(9), modified_timestamp TIMESTAMP_NTZ(9))" options: | NOT NULL RETURNS NULL ON NULL INPUT From 06e8fe8693d7d45a0c6cc4c0de7c96558a6ce7e7 Mon Sep 17 00:00:00 2001 From: shah Date: Tue, 15 Oct 2024 22:07:08 -0700 Subject: [PATCH 19/50] STREAM-1053 Add fact_transactions UDTF --- macros/evm/evm.yaml.sql | 14 ++ macros/evm/evm_live_views.sql | 252 +++++++++++++++++++++++++++++++++- 2 files changed, 264 insertions(+), 2 deletions(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index 023c6a2..385e2a4 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -418,6 +418,20 @@ sql: | {{ evm_live_view_fact_traces(schema, blockchain, network) | indent(4) -}} +- name: {{ schema -}}.tf_fact_transactions + signature: + - [block_height, INTEGER, The start block height to get the transfers from] + - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] + return_type: + - "TABLE(block_number NUMBER, block_timestamp TIMESTAMP_NTZ, block_hash STRING, tx_hash STRING, nonce NUMBER, POSITION NUMBER, origin_function_signature STRING, from_address STRING, to_address STRING, VALUE FLOAT, value_precise_raw STRING, value_precise STRING, tx_fee FLOAT, tx_fee_precise STRING, gas_price FLOAT, gas_limit NUMBER, gas_used NUMBER, cumulative_gas_used NUMBER, input_data STRING, status STRING, effective_gas_price FLOAT, max_fee_per_gas FLOAT, max_priority_fee_per_gas FLOAT, r STRING, s STRING, v STRING, tx_type NUMBER, chain_id NUMBER, blob_versioned_hashes ARRAY, max_fee_per_blob_gas NUMBER, blob_gas_used NUMBER, blob_gas_price NUMBER, fact_transactions_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + COMMENT = $$Returns the transactions for a given block height. If to_latest is true, it will continue fetching transactions until the latest block. Otherwise, it will fetch transactions until the block height is reached.$$ + sql: | + {{ evm_live_view_fact_transactions(schema, blockchain, network) | indent(4) -}} + - name: {{ schema -}}.tf_ez_native_transfers signature: - [block_height, INTEGER, The start block height to get the transfers from] diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index c871b81..a287f1e 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -491,6 +491,254 @@ LEFT JOIN {{ blockchain }}.core.dim_contracts AS D ON B.contract_address = D.address {% endmacro %} +{% macro evm_live_view_fact_transactions(schema, blockchain, network) %} + +WITH heights AS ( + SELECT + livequery_dev.live.udf_api( + 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', + livequery_Dev.utils.udf_json_rpc_call( + 'eth_blockNumber', + [] + ) + ):data AS result, + livequery_dev.utils.udf_hex_to_int(result:result)::integer as latest_block_height, + coalesce( + block_height, + latest_block_height + ) as min_height, + iff( + coalesce(to_latest, false), + latest_block_height, + min_height + ) as max_height + ), + spine as ( + select + row_number() over ( + order by + null + ) -1 + coalesce(block_height, 0)::integer as block_number, + min_height, + iff( + coalesce(to_latest, false), + latest_block_height, + min_height + ) as max_height, + latest_block_height + from + table(generator(ROWCOUNT => 1000)), + heights + qualify block_number between min_height and max_height + ), + raw_receipts as ( + SELECT + latest_block_height, + block_number, + livequery_dev.live.udf_api( + 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', + livequery_Dev.utils.udf_json_rpc_call( + 'eth_getBlockReceipts', + [livequery_dev.utils.udf_int_to_hex(block_number)] + ) + ):data.result AS result, + v.value as DATA + from + spine, + lateral flatten(result) v + ), + raw_block_txs as ( + SELECT + block_number, + livequery_dev.live.udf_api( + 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', + livequery_Dev.utils.udf_json_rpc_call( + 'eth_getBlockByNumber', + [livequery_dev.utils.udf_int_to_hex(block_number), true] + ) + ):data.result AS DATA + from + spine + ), + raw_txs as ( + SELECT + block_number, + v.value as DATA + from + raw_block_txs r, + lateral flatten(r.data:transactions) v + ), + blocks as ( + select + block_number, + livequery_dev.utils.udf_hex_to_int(DATA :baseFeePerGas::STRING)::INT AS base_fee_per_gas, + livequery_dev.utils.udf_hex_to_int(DATA :difficulty::STRING)::INT AS difficulty, + DATA :extraData::STRING AS extra_data, + livequery_dev.utils.udf_hex_to_int(DATA :gasLimit::STRING)::INT AS gas_limit, + livequery_dev.utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS gas_used, + DATA :hash::STRING AS HASH, + DATA :logsBloom::STRING AS logs_bloom, + DATA :miner::STRING AS miner, + livequery_dev.utils.udf_hex_to_int(DATA :nonce::STRING)::INT AS nonce, + livequery_dev.utils.udf_hex_to_int(DATA :number::STRING)::INT AS NUMBER, + DATA :parentHash::STRING AS parent_hash, + DATA :receiptsRoot::STRING AS receipts_root, + DATA :sha3Uncles::STRING AS sha3_uncles, + livequery_dev.utils.udf_hex_to_int(DATA :size::STRING)::INT AS SIZE, + DATA :stateRoot::STRING AS state_root, + livequery_dev.utils.udf_hex_to_int(DATA :timestamp::STRING)::TIMESTAMP AS block_timestamp, + livequery_dev.utils.udf_hex_to_int(DATA :totalDifficulty::STRING)::INT AS total_difficulty, + ARRAY_SIZE(DATA :transactions) AS tx_count, + DATA :transactionsRoot::STRING AS transactions_root, + DATA :uncles AS uncles, + DATA :withdrawals AS withdrawals, + DATA :withdrawalsRoot::STRING AS withdrawals_root, + md5( + cast( + coalesce( + cast(block_number as TEXT), + '_dbt_utils_surrogate_key_null_' + ) as TEXT + ) + ) AS blocks_id, + livequery_dev.utils.udf_hex_to_int(DATA: blobGasUsed::STRING)::INT AS blob_gas_used, + livequery_dev.utils.udf_hex_to_int(DATA: excessBlobGas::STRING)::INT AS excess_blob_gas, + from + raw_block_txs + ), + receipts as ( + select + latest_block_height, + block_number, + DATA :blockHash::STRING AS block_hash, + livequery_dev.utils.udf_hex_to_int(DATA :blockNumber::STRING)::INT AS blockNumber, + livequery_dev.utils.udf_hex_to_int(DATA :cumulativeGasUsed::STRING)::INT AS cumulative_gas_used, + livequery_dev.utils.udf_hex_to_int(DATA :effectiveGasPrice::STRING)::INT / pow(10, 9) AS effective_gas_price, + DATA :from::STRING AS from_address, + livequery_dev.utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS gas_used, + DATA :logs AS logs, + DATA :logsBloom::STRING AS logs_bloom, + livequery_dev.utils.udf_hex_to_int(DATA :status::STRING)::INT AS status, + CASE + WHEN status = 1 THEN TRUE + ELSE FALSE + END AS tx_success, + CASE + WHEN status = 1 THEN 'SUCCESS' + ELSE 'FAIL' + END AS tx_status, + DATA :to::STRING AS to_address1, + CASE + WHEN to_address1 = '' THEN NULL + ELSE to_address1 + END AS to_address, + DATA :transactionHash::STRING AS tx_hash, + livequery_dev.utils.udf_hex_to_int(DATA :transactionIndex::STRING)::INT AS POSITION, + livequery_dev.utils.udf_hex_to_int(DATA :type::STRING)::INT AS TYPE, + livequery_dev.utils.udf_hex_to_int(DATA :effectiveGasPrice::STRING)::INT AS blob_gas_price, + livequery_dev.utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS blob_gas_used + from + raw_receipts + ), + txs as ( + select + A.block_number AS block_number, + A.data :blockHash::STRING AS block_hash, + livequery_dev.utils.udf_hex_to_int(A.data :blockNumber::STRING)::INT AS blockNumber, + livequery_dev.utils.udf_hex_to_int(A.data :chainId::STRING)::INT AS chain_id, + A.data :from::STRING AS from_address, + livequery_dev.utils.udf_hex_to_int(A.data :gas::STRING)::INT AS gas, + livequery_dev.utils.udf_hex_to_int(A.data :gasPrice::STRING)::INT / pow(10, 9) AS gas_price, + A.data :hash::STRING AS tx_hash, + A.data :input::STRING AS input_data, + SUBSTR(input_data, 1, 10) AS origin_function_signature, + livequery_dev.utils.udf_hex_to_int(A.data :maxFeePerGas::STRING)::INT / pow(10, 9) AS max_fee_per_gas, + livequery_dev.utils.udf_hex_to_int( + A.data :maxPriorityFeePerGas::STRING + )::INT / pow(10, 9) AS max_priority_fee_per_gas, + livequery_dev.utils.udf_hex_to_int(A.data :nonce::STRING)::INT AS nonce, + A.data :r::STRING AS r, + A.data :s::STRING AS s, + A.data :to::STRING AS to_address, + livequery_dev.utils.udf_hex_to_int(A.data :transactionIndex::STRING)::INT AS POSITION, + A.data :type::STRING AS TYPE, + A.data :v::STRING AS v, + livequery_dev.utils.udf_hex_to_int(A.data :value::STRING) AS value_precise_raw, + value_precise_raw * power(10, -18) AS value_precise, + value_precise::FLOAT AS VALUE, + A.data :accessList AS access_list, + A.data, + A.data: blobVersionedHashes::ARRAY AS blob_versioned_hashes, + livequery_dev.utils.udf_hex_to_int(A.data: maxFeePerGas::STRING)::INT AS max_fee_per_blob_gas, + block_timestamp, + CASE + WHEN block_timestamp IS NULL + OR tx_status IS NULL THEN TRUE + ELSE FALSE + END AS is_pending, + r.gas_used, + tx_success, + tx_status, + cumulative_gas_used, + effective_gas_price, + livequery_dev.utils.udf_hex_to_int(A.data :gasPrice) * power(10, -18) * r.gas_used AS tx_fee_precise, + COALESCE(tx_fee_precise::FLOAT, 0) AS tx_fee, + r.type as tx_type, + r.blob_gas_used, + r.blob_gas_price, + from + raw_txs A + left join blocks b on b.block_number = A.block_number + left join receipts as r on r.tx_hash = A.data :hash::STRING + ) + SELECT + block_number, + block_timestamp, + block_hash, + tx_hash, + nonce, + POSITION, + origin_function_signature, + from_address, + to_address, + VALUE, + value_precise_raw, + value_precise::STRING as value_precise, + tx_fee, + tx_fee_precise::STRING as tx_fee_precise, + gas_price, + gas AS gas_limit, + gas_used, + cumulative_gas_used, + input_data, + tx_status AS status, + effective_gas_price, + max_fee_per_gas, + max_priority_fee_per_gas, + r, + s, + v, + tx_type, + chain_id, + blob_versioned_hashes, + max_fee_per_blob_gas, + blob_gas_used, + blob_gas_price, + md5( + cast( + coalesce( + cast(tx_hash as TEXT), + '_dbt_utils_surrogate_key_null_' + ) as TEXT + ) + ) AS fact_transactions_id, + SYSDATE() inserted_timestamp, + SYSDATE() AS modified_timestamp + FROM + txs + +{% endmacro %} + {% macro evm_live_view_fact_traces(schema, blockchain, network) %} WITH heights AS ( SELECT @@ -526,7 +774,7 @@ WITH heights AS ( ) as max_height, latest_block_height from - table(generator(ROWCOUNT => 500)), + table(generator(ROWCOUNT => 1000)), heights qualify block_number between min_height and max_height ), @@ -1108,7 +1356,7 @@ WITH heights AS ( ) as max_height, latest_block_height from - table(generator(ROWCOUNT => 500)), + table(generator(ROWCOUNT => 1000)), heights qualify block_number between min_height and max_height ), From baf6d7da55869ab33d853380cc51ed002dbd21e0 Mon Sep 17 00:00:00 2001 From: shah Date: Wed, 16 Oct 2024 15:16:17 -0700 Subject: [PATCH 20/50] STREAM-1049 Secrets interpolate fact_blocks macros --- macros/evm/evm_live_views.sql | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index a287f1e..fb47a92 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -1,12 +1,6 @@ -{% macro evm_live_view_latest_block_height(blockchain, network) %} +{% macro evm_live_view_latest_block_height(schema, blockchain, network) %} SELECT - live.udf_api( - '{service}/{Authentication}', - utils.udf_json_rpc_call( - 'eth_blockNumber', - [] - ) - ):data AS result, + {{ schema }}.udf_rpc('eth_blockNumber', []) as result, utils.udf_hex_to_int(result:result)::integer AS latest_block_height, COALESCE( block_height, @@ -19,9 +13,9 @@ ) AS max_height {% endmacro %} -{% macro evm_live_view_target_blocks(blockchain, network) %} +{% macro evm_live_view_target_blocks(schema, blockchain, network) %} WITH heights AS ( - {{ evm_live_view_latest_block_height(blockchain, network) | indent(4) -}} + {{ evm_live_view_latest_block_height(schema, blockchain, network) | indent(4) -}} ) SELECT ROW_NUMBER() OVER ( @@ -42,16 +36,12 @@ {% endmacro %} -- Get Raw EVM chain data -{% macro evm_live_view_bronze_blocks(table_name) %} +{% macro evm_live_view_bronze_blocks(schema, table_name) %} SELECT block_number, - live.udf_api( - '{service}/{Authentication}', - utils.udf_json_rpc_call( - 'eth_getBlockByNumber', - [utils.udf_int_to_hex(block_number), true] - ) - ):data.result AS DATA + {{ schema }}.udf_rpc( + 'eth_getBlockByNumber', + [utils.udf_int_to_hex(block_number), true]) AS DATA FROM {{ table_name }} {% endmacro %} @@ -247,10 +237,10 @@ FROM -- Get EVM chain fact data {% macro evm_live_view_fact_blocks(schema, blockchain, network) %} WITH spine AS ( - {{ evm_live_view_target_blocks(blockchain, network) | indent(4) -}} + {{ evm_live_view_target_blocks(schema, blockchain, network) | indent(4) -}} ), raw_block_txs AS ( - {{ evm_live_view_bronze_blocks('spine') | indent(4) -}} + {{ evm_live_view_bronze_blocks( schema, 'spine') | indent(4) -}} ), silver_blocks AS ( {{ evm_live_view_silver_blocks('raw_block_txs') | indent(4) -}} @@ -330,7 +320,7 @@ WITH spine AS ( {% macro evm_live_view_fact_logs(schema, blockchain, network) %} WITH spine AS ( - {{ evm_live_view_target_blocks(blockchain, network) | indent(4) -}} + {{ evm_live_view_target_blocks(schema, blockchain, network) | indent(4) -}} ), raw_block_txs AS ( {{ evm_live_view_bronze_blocks('spine') | indent(4) -}} From 678d9d267608012da2e7ee58d826abbc9f9a3a81 Mon Sep 17 00:00:00 2001 From: shah Date: Wed, 16 Oct 2024 17:04:03 -0700 Subject: [PATCH 21/50] STREAM-1049 Add secrets interpoation for fact_logs & fact_decoded_event_logs --- macros/evm/evm_live_views.sql | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index fb47a92..b1491bc 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -46,17 +46,13 @@ FROM {{ table_name }} {% endmacro %} -{% macro evm_live_view_bronze_receipts(table_name) %} +{% macro evm_live_view_bronze_receipts(schema, table_name) %} SELECT latest_block_height, block_number, - live.udf_api( - '{service}/{Authentication}', - utils.udf_json_rpc_call( - 'eth_getBlockReceipts', - [utils.udf_int_to_hex(block_number)] - ) - ):data.result AS result, + {{ schema }}.udf_rpc( + 'eth_getBlockReceipts', + [utils.udf_int_to_hex(block_number)]) AS result, v.value AS DATA FROM {{ table_name }}, @@ -323,10 +319,10 @@ WITH spine AS ( {{ evm_live_view_target_blocks(schema, blockchain, network) | indent(4) -}} ), raw_block_txs AS ( - {{ evm_live_view_bronze_blocks('spine') | indent(4) -}} + {{ evm_live_view_bronze_blocks(schema, 'spine') | indent(4) -}} ), raw_receipts AS ( - {{ evm_live_view_bronze_receipts('spine') | indent(4) -}} + {{ evm_live_view_bronze_receipts(schema, 'spine') | indent(4) -}} ), raw_logs AS ( {{ evm_live_view_bronze_logs('raw_receipts') | indent(4) -}} From 5196c3e135f0c97282ecba9ad09e8e93588dd656 Mon Sep 17 00:00:00 2001 From: shah Date: Wed, 16 Oct 2024 17:30:04 -0700 Subject: [PATCH 22/50] STREAM-1049 Added secrets interpolation to fact_traces, fact_transactions & ez_native_transfers --- macros/evm/evm_live_views.sql | 104 +++++++++------------------------- 1 file changed, 27 insertions(+), 77 deletions(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index b1491bc..602b627 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -481,13 +481,7 @@ LEFT JOIN {{ blockchain }}.core.dim_contracts AS D WITH heights AS ( SELECT - livequery_dev.live.udf_api( - 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', - livequery_Dev.utils.udf_json_rpc_call( - 'eth_blockNumber', - [] - ) - ):data AS result, + {{ schema }}.udf_rpc('eth_blockNumber', []) as result, livequery_dev.utils.udf_hex_to_int(result:result)::integer as latest_block_height, coalesce( block_height, @@ -521,13 +515,9 @@ WITH heights AS ( SELECT latest_block_height, block_number, - livequery_dev.live.udf_api( - 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', - livequery_Dev.utils.udf_json_rpc_call( - 'eth_getBlockReceipts', - [livequery_dev.utils.udf_int_to_hex(block_number)] - ) - ):data.result AS result, + {{ schema }}.udf_rpc( + 'eth_getBlockReceipts', + [utils.udf_int_to_hex(block_number)]) AS result, v.value as DATA from spine, @@ -536,13 +526,9 @@ WITH heights AS ( raw_block_txs as ( SELECT block_number, - livequery_dev.live.udf_api( - 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', - livequery_Dev.utils.udf_json_rpc_call( - 'eth_getBlockByNumber', - [livequery_dev.utils.udf_int_to_hex(block_number), true] - ) - ):data.result AS DATA + {{ schema }}.udf_rpc( + 'eth_getBlockByNumber', + [utils.udf_int_to_hex(block_number), true]) AS DATA from spine ), @@ -728,13 +714,7 @@ WITH heights AS ( {% macro evm_live_view_fact_traces(schema, blockchain, network) %} WITH heights AS ( SELECT - livequery_dev.live.udf_api( - 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', - livequery_Dev.utils.udf_json_rpc_call( - 'eth_blockNumber', - [] - ) - ):data AS result, + {{ schema }}.udf_rpc('eth_blockNumber', []) as result, livequery_dev.utils.udf_hex_to_int(result:result)::integer as latest_block_height, coalesce( block_height, @@ -768,13 +748,9 @@ WITH heights AS ( SELECT latest_block_height, block_number, - livequery_dev.live.udf_api( - 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', - livequery_Dev.utils.udf_json_rpc_call( - 'eth_getBlockReceipts', - [livequery_dev.utils.udf_int_to_hex(block_number)] - ) - ):data.result AS result, + {{ schema }}.udf_rpc( + 'eth_getBlockReceipts', + [utils.udf_int_to_hex(block_number)]) AS result, v.value as DATA from spine, @@ -783,13 +759,9 @@ WITH heights AS ( raw_block_txs as ( SELECT block_number, - livequery_dev.live.udf_api( - 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', - livequery_Dev.utils.udf_json_rpc_call( - 'eth_getBlockByNumber', - [livequery_dev.utils.udf_int_to_hex(block_number), true] - ) - ):data.result AS DATA + {{ schema }}.udf_rpc( + 'eth_getBlockByNumber', + [utils.udf_int_to_hex(block_number), true]) AS DATA from spine ), @@ -932,13 +904,9 @@ WITH heights AS ( SYSDATE() AS _inserted_timestamp FROM spine s, LATERAL FLATTEN(input => PARSE_JSON( - livequery_dev.live.udf_api( - 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', - livequery_Dev.utils.udf_json_rpc_call( - 'debug_traceBlockByNumber', - [livequery_dev.utils.udf_int_to_hex(s.block_number), {'tracer': 'callTracer'}] - ) - ):data.result + {{ schema }}.udf_rpc( + 'debug_traceBlockByNumber', + [utils.udf_int_to_hex(s.block_number), {'tracer': 'callTracer'}]) )) v ), flatten_traces AS ( @@ -1310,13 +1278,7 @@ AND from_address IS NOT NULL {% macro evm_live_view_ez_native_transfers(schema, blockchain, network) %} WITH heights AS ( SELECT - livequery_dev.live.udf_api( - 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', - livequery_Dev.utils.udf_json_rpc_call( - 'eth_blockNumber', - [] - ) - ):data AS result, + {{ schema }}.udf_rpc('eth_blockNumber', []) as result, livequery_dev.utils.udf_hex_to_int(result:result)::integer as latest_block_height, coalesce( block_height, @@ -1350,13 +1312,9 @@ WITH heights AS ( SELECT latest_block_height, block_number, - livequery_dev.live.udf_api( - 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', - livequery_Dev.utils.udf_json_rpc_call( - 'eth_getBlockReceipts', - [livequery_dev.utils.udf_int_to_hex(block_number)] - ) - ):data.result AS result, + {{ schema }}.udf_rpc( + 'eth_getBlockReceipts', + [utils.udf_int_to_hex(block_number)]) AS result, v.value as DATA from spine, @@ -1365,13 +1323,9 @@ WITH heights AS ( raw_block_txs as ( SELECT block_number, - livequery_dev.live.udf_api( - 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', - livequery_Dev.utils.udf_json_rpc_call( - 'eth_getBlockByNumber', - [livequery_dev.utils.udf_int_to_hex(block_number), true] - ) - ):data.result AS DATA + {{ schema }}.udf_rpc( + 'eth_getBlockByNumber', + [utils.udf_int_to_hex(block_number), true]) AS DATA from spine ), @@ -1514,13 +1468,9 @@ WITH heights AS ( SYSDATE() AS _inserted_timestamp FROM spine s, LATERAL FLATTEN(input => PARSE_JSON( - livequery_dev.live.udf_api( - 'https://indulgent-frosty-sanctuary.quiknode.pro/22555ab2563d38edce551aa3ab524e595d9ccba8/', - livequery_Dev.utils.udf_json_rpc_call( - 'debug_traceBlockByNumber', - [livequery_dev.utils.udf_int_to_hex(s.block_number), {'tracer': 'callTracer'}] - ) - ):data.result + {{ schema }}.udf_rpc( + 'debug_traceBlockByNumber', + [utils.udf_int_to_hex(s.block_number), {'tracer': 'callTracer'}]) )) v ), flatten_traces AS ( From ad8d9891c1bb60a3974df1080eaa8d3d4cd6e93a Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Thu, 17 Oct 2024 11:23:40 +0900 Subject: [PATCH 23/50] add discrepent columns between gold and livequery --- macros/evm/evm.yaml.sql | 8 +++---- macros/evm/evm_live_views.sql | 40 +++++++++++++++-------------------- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index 385e2a4..f4cabb9 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -396,7 +396,7 @@ - [block_height, INTEGER, The start block height to get the logs from] - [to_latest, BOOLEAN, Whether to continue fetching logs until the latest block or not] return_type: - - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, event_index INTEGER, contract_address STRING, contract_name STRING, event_name STRING, decoded_log OBJECT, full_decoded_log VARIANT, origin_function_signature STRING, origin_from_address STRING, origin_to_address STRING, topics VARIANT, data STRING, event_removed STRING, tx_status STRING, _log_id STRING, fact_event_logs_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, event_index INTEGER, contract_address STRING, contract_name STRING, event_name STRING, decoded_log OBJECT, full_decoded_log VARIANT, fact_decoded_event_logs_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" options: | NOT NULL RETURNS NULL ON NULL INPUT @@ -408,7 +408,7 @@ signature: - [block_height, INTEGER, The start block height to get the transfers from] - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] - return_type: + return_type: - "TABLE(tx_hash STRING, block_number NUMBER, block_timestamp TIMESTAMP_NTZ(9), from_address STRING, to_address STRING, value FLOAT, value_precise_raw STRING, value_precise STRING, gas NUMBER, gas_used NUMBER, input STRING, output STRING, TYPE STRING, identifier STRING, DATA OBJECT, tx_status STRING, sub_traces NUMBER, trace_status STRING, error_reason STRING, trace_index NUMBER, fact_traces_id STRING, inserted_timestamp TIMESTAMP_NTZ(9), modified_timestamp TIMESTAMP_NTZ(9))" options: | NOT NULL @@ -422,7 +422,7 @@ signature: - [block_height, INTEGER, The start block height to get the transfers from] - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] - return_type: + return_type: - "TABLE(block_number NUMBER, block_timestamp TIMESTAMP_NTZ, block_hash STRING, tx_hash STRING, nonce NUMBER, POSITION NUMBER, origin_function_signature STRING, from_address STRING, to_address STRING, VALUE FLOAT, value_precise_raw STRING, value_precise STRING, tx_fee FLOAT, tx_fee_precise STRING, gas_price FLOAT, gas_limit NUMBER, gas_used NUMBER, cumulative_gas_used NUMBER, input_data STRING, status STRING, effective_gas_price FLOAT, max_fee_per_gas FLOAT, max_priority_fee_per_gas FLOAT, r STRING, s STRING, v STRING, tx_type NUMBER, chain_id NUMBER, blob_versioned_hashes ARRAY, max_fee_per_blob_gas NUMBER, blob_gas_used NUMBER, blob_gas_price NUMBER, fact_transactions_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" options: | NOT NULL @@ -452,7 +452,7 @@ - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] - [ez_token_transfers_id, STRING, The topic of the token transfers to get] return_type: - - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, event_index INTEGER, origin_funcion_signature STRING, origin_from_address STRING, origin_to_address STRING, contract_address STRING, from_address STRING, to_address STRING, raw_amount_precise STRING, raw_amount FLOAT, amount_precise FLOAT, amount FLOAT, amount_usd FLOAT, decimals INTEGER, symbol STRING, token_price FLOAT, has_decimal STRING, has_price STRING, ez_token_transfers_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, event_index INTEGER, origin_function_signature STRING, origin_from_address STRING, origin_to_address STRING, contract_address STRING, from_address STRING, to_address STRING, raw_amount_precise STRING, raw_amount FLOAT, amount_precise FLOAT, amount FLOAT, amount_usd FLOAT, decimals INTEGER, symbol STRING, token_price FLOAT, has_decimal STRING, has_price STRING, _log_id STRING, ez_token_transfers_id STRING, _inserted_timestamp TIMESTAMP_NTZ, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" options: | NOT NULL RETURNS NULL ON NULL INPUT diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 602b627..59506fc 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -40,7 +40,7 @@ SELECT block_number, {{ schema }}.udf_rpc( - 'eth_getBlockByNumber', + 'eth_getBlockByNumber', [utils.udf_int_to_hex(block_number), true]) AS DATA FROM {{ table_name }} @@ -51,7 +51,7 @@ SELECT latest_block_height, block_number, {{ schema }}.udf_rpc( - 'eth_getBlockReceipts', + 'eth_getBlockReceipts', [utils.udf_int_to_hex(block_number)]) AS result, v.value AS DATA FROM @@ -460,15 +460,8 @@ SELECT B.event_name, B.decoded_flat AS decoded_log, B.decoded_data AS full_decoded_log, - C.origin_function_signature, - C.origin_from_address, - C.origin_to_address, - C.topics, - C.DATA, - C.event_removed :: STRING AS event_removed, - C.tx_status, - _log_id, - md5(_log_id) AS fact_event_logs_id, + md5(_log_id) AS fact_decoded_event_logs_id, + SYSDATE() AS _inserted_timestamp, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp FROM _flatten_logs AS B @@ -508,7 +501,7 @@ WITH heights AS ( latest_block_height from table(generator(ROWCOUNT => 1000)), - heights + heights qualify block_number between min_height and max_height ), raw_receipts as ( @@ -516,7 +509,7 @@ WITH heights AS ( latest_block_height, block_number, {{ schema }}.udf_rpc( - 'eth_getBlockReceipts', + 'eth_getBlockReceipts', [utils.udf_int_to_hex(block_number)]) AS result, v.value as DATA from @@ -527,7 +520,7 @@ WITH heights AS ( SELECT block_number, {{ schema }}.udf_rpc( - 'eth_getBlockByNumber', + 'eth_getBlockByNumber', [utils.udf_int_to_hex(block_number), true]) AS DATA from spine @@ -662,7 +655,7 @@ WITH heights AS ( raw_txs A left join blocks b on b.block_number = A.block_number left join receipts as r on r.tx_hash = A.data :hash::STRING - ) + ) SELECT block_number, block_timestamp, @@ -741,7 +734,7 @@ WITH heights AS ( latest_block_height from table(generator(ROWCOUNT => 1000)), - heights + heights qualify block_number between min_height and max_height ), raw_receipts as ( @@ -749,7 +742,7 @@ WITH heights AS ( latest_block_height, block_number, {{ schema }}.udf_rpc( - 'eth_getBlockReceipts', + 'eth_getBlockReceipts', [utils.udf_int_to_hex(block_number)]) AS result, v.value as DATA from @@ -760,7 +753,7 @@ WITH heights AS ( SELECT block_number, {{ schema }}.udf_rpc( - 'eth_getBlockByNumber', + 'eth_getBlockByNumber', [utils.udf_int_to_hex(block_number), true]) AS DATA from spine @@ -905,7 +898,7 @@ WITH heights AS ( FROM spine s, LATERAL FLATTEN(input => PARSE_JSON( {{ schema }}.udf_rpc( - 'debug_traceBlockByNumber', + 'debug_traceBlockByNumber', [utils.udf_int_to_hex(s.block_number), {'tracer': 'callTracer'}]) )) v ), @@ -1249,6 +1242,7 @@ SELECT WHEN price IS NULL THEN 'false' ELSE 'true' END AS has_price, + _log_id, md5( cast( coalesce( @@ -1305,7 +1299,7 @@ WITH heights AS ( latest_block_height from table(generator(ROWCOUNT => 1000)), - heights + heights qualify block_number between min_height and max_height ), raw_receipts as ( @@ -1313,7 +1307,7 @@ WITH heights AS ( latest_block_height, block_number, {{ schema }}.udf_rpc( - 'eth_getBlockReceipts', + 'eth_getBlockReceipts', [utils.udf_int_to_hex(block_number)]) AS result, v.value as DATA from @@ -1324,7 +1318,7 @@ WITH heights AS ( SELECT block_number, {{ schema }}.udf_rpc( - 'eth_getBlockByNumber', + 'eth_getBlockByNumber', [utils.udf_int_to_hex(block_number), true]) AS DATA from spine @@ -1469,7 +1463,7 @@ WITH heights AS ( FROM spine s, LATERAL FLATTEN(input => PARSE_JSON( {{ schema }}.udf_rpc( - 'debug_traceBlockByNumber', + 'debug_traceBlockByNumber', [utils.udf_int_to_hex(s.block_number), {'tracer': 'callTracer'}]) )) v ), From 914ab562f2f7b071edb7e97e40e0b64f2243c3c6 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Thu, 17 Oct 2024 11:42:41 +0900 Subject: [PATCH 24/50] fix cols --- macros/evm/evm_live_views.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 59506fc..591ebf2 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -461,7 +461,6 @@ SELECT B.decoded_flat AS decoded_log, B.decoded_data AS full_decoded_log, md5(_log_id) AS fact_decoded_event_logs_id, - SYSDATE() AS _inserted_timestamp, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp FROM _flatten_logs AS B @@ -1254,6 +1253,7 @@ SELECT ) as TEXT ) ) as ez_token_transfers_id, + SYSDATE() AS _inserted_timestamp, sysdate() as inserted_timestamp, sysdate() as modified_timestamp FROM From ffc0e502475b7ce30c3b22807d0404eb5bbb090a Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Thu, 17 Oct 2024 11:52:28 +0900 Subject: [PATCH 25/50] remove contract_name --- macros/evm/evm.yaml.sql | 2 +- macros/evm/evm_live_views.sql | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index f4cabb9..8ccd16c 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -396,7 +396,7 @@ - [block_height, INTEGER, The start block height to get the logs from] - [to_latest, BOOLEAN, Whether to continue fetching logs until the latest block or not] return_type: - - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, event_index INTEGER, contract_address STRING, contract_name STRING, event_name STRING, decoded_log OBJECT, full_decoded_log VARIANT, fact_decoded_event_logs_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, event_index INTEGER, contract_address STRING, event_name STRING, decoded_log OBJECT, full_decoded_log VARIANT, fact_decoded_event_logs_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" options: | NOT NULL RETURNS NULL ON NULL INPUT diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 591ebf2..9dd5b7b 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -456,7 +456,6 @@ SELECT B.tx_hash, B.event_index, B.contract_address, - D.name AS contract_name, B.event_name, B.decoded_flat AS decoded_log, B.decoded_data AS full_decoded_log, @@ -465,8 +464,6 @@ SELECT SYSDATE() AS modified_timestamp FROM _flatten_logs AS B LEFT JOIN _silver_decoded_logs AS C USING (block_number, _log_id) -LEFT JOIN {{ blockchain }}.core.dim_contracts AS D - ON B.contract_address = D.address {% endmacro %} {% macro evm_live_view_fact_transactions(schema, blockchain, network) %} From 166834a637c95b432517f9ffbe27f8f1556376fe Mon Sep 17 00:00:00 2001 From: shah Date: Wed, 16 Oct 2024 20:25:43 -0700 Subject: [PATCH 26/50] STREAM-1053 Refactor fact_transactions to use reusable LiveView CTE's --- macros/evm/evm_live_views.sql | 295 ++++++++-------------------------- 1 file changed, 66 insertions(+), 229 deletions(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 9dd5b7b..4095605 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -468,235 +468,72 @@ LEFT JOIN _silver_decoded_logs AS C USING (block_number, _log_id) {% macro evm_live_view_fact_transactions(schema, blockchain, network) %} -WITH heights AS ( - SELECT - {{ schema }}.udf_rpc('eth_blockNumber', []) as result, - livequery_dev.utils.udf_hex_to_int(result:result)::integer as latest_block_height, - coalesce( - block_height, - latest_block_height - ) as min_height, - iff( - coalesce(to_latest, false), - latest_block_height, - min_height - ) as max_height - ), - spine as ( - select - row_number() over ( - order by - null - ) -1 + coalesce(block_height, 0)::integer as block_number, - min_height, - iff( - coalesce(to_latest, false), - latest_block_height, - min_height - ) as max_height, - latest_block_height - from - table(generator(ROWCOUNT => 1000)), - heights - qualify block_number between min_height and max_height - ), - raw_receipts as ( - SELECT - latest_block_height, - block_number, - {{ schema }}.udf_rpc( - 'eth_getBlockReceipts', - [utils.udf_int_to_hex(block_number)]) AS result, - v.value as DATA - from - spine, - lateral flatten(result) v - ), - raw_block_txs as ( - SELECT - block_number, - {{ schema }}.udf_rpc( - 'eth_getBlockByNumber', - [utils.udf_int_to_hex(block_number), true]) AS DATA - from - spine - ), - raw_txs as ( - SELECT - block_number, - v.value as DATA - from - raw_block_txs r, - lateral flatten(r.data:transactions) v - ), - blocks as ( - select - block_number, - livequery_dev.utils.udf_hex_to_int(DATA :baseFeePerGas::STRING)::INT AS base_fee_per_gas, - livequery_dev.utils.udf_hex_to_int(DATA :difficulty::STRING)::INT AS difficulty, - DATA :extraData::STRING AS extra_data, - livequery_dev.utils.udf_hex_to_int(DATA :gasLimit::STRING)::INT AS gas_limit, - livequery_dev.utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS gas_used, - DATA :hash::STRING AS HASH, - DATA :logsBloom::STRING AS logs_bloom, - DATA :miner::STRING AS miner, - livequery_dev.utils.udf_hex_to_int(DATA :nonce::STRING)::INT AS nonce, - livequery_dev.utils.udf_hex_to_int(DATA :number::STRING)::INT AS NUMBER, - DATA :parentHash::STRING AS parent_hash, - DATA :receiptsRoot::STRING AS receipts_root, - DATA :sha3Uncles::STRING AS sha3_uncles, - livequery_dev.utils.udf_hex_to_int(DATA :size::STRING)::INT AS SIZE, - DATA :stateRoot::STRING AS state_root, - livequery_dev.utils.udf_hex_to_int(DATA :timestamp::STRING)::TIMESTAMP AS block_timestamp, - livequery_dev.utils.udf_hex_to_int(DATA :totalDifficulty::STRING)::INT AS total_difficulty, - ARRAY_SIZE(DATA :transactions) AS tx_count, - DATA :transactionsRoot::STRING AS transactions_root, - DATA :uncles AS uncles, - DATA :withdrawals AS withdrawals, - DATA :withdrawalsRoot::STRING AS withdrawals_root, - md5( - cast( - coalesce( - cast(block_number as TEXT), - '_dbt_utils_surrogate_key_null_' - ) as TEXT - ) - ) AS blocks_id, - livequery_dev.utils.udf_hex_to_int(DATA: blobGasUsed::STRING)::INT AS blob_gas_used, - livequery_dev.utils.udf_hex_to_int(DATA: excessBlobGas::STRING)::INT AS excess_blob_gas, - from - raw_block_txs - ), - receipts as ( - select - latest_block_height, - block_number, - DATA :blockHash::STRING AS block_hash, - livequery_dev.utils.udf_hex_to_int(DATA :blockNumber::STRING)::INT AS blockNumber, - livequery_dev.utils.udf_hex_to_int(DATA :cumulativeGasUsed::STRING)::INT AS cumulative_gas_used, - livequery_dev.utils.udf_hex_to_int(DATA :effectiveGasPrice::STRING)::INT / pow(10, 9) AS effective_gas_price, - DATA :from::STRING AS from_address, - livequery_dev.utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS gas_used, - DATA :logs AS logs, - DATA :logsBloom::STRING AS logs_bloom, - livequery_dev.utils.udf_hex_to_int(DATA :status::STRING)::INT AS status, - CASE - WHEN status = 1 THEN TRUE - ELSE FALSE - END AS tx_success, - CASE - WHEN status = 1 THEN 'SUCCESS' - ELSE 'FAIL' - END AS tx_status, - DATA :to::STRING AS to_address1, - CASE - WHEN to_address1 = '' THEN NULL - ELSE to_address1 - END AS to_address, - DATA :transactionHash::STRING AS tx_hash, - livequery_dev.utils.udf_hex_to_int(DATA :transactionIndex::STRING)::INT AS POSITION, - livequery_dev.utils.udf_hex_to_int(DATA :type::STRING)::INT AS TYPE, - livequery_dev.utils.udf_hex_to_int(DATA :effectiveGasPrice::STRING)::INT AS blob_gas_price, - livequery_dev.utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS blob_gas_used - from - raw_receipts - ), - txs as ( - select - A.block_number AS block_number, - A.data :blockHash::STRING AS block_hash, - livequery_dev.utils.udf_hex_to_int(A.data :blockNumber::STRING)::INT AS blockNumber, - livequery_dev.utils.udf_hex_to_int(A.data :chainId::STRING)::INT AS chain_id, - A.data :from::STRING AS from_address, - livequery_dev.utils.udf_hex_to_int(A.data :gas::STRING)::INT AS gas, - livequery_dev.utils.udf_hex_to_int(A.data :gasPrice::STRING)::INT / pow(10, 9) AS gas_price, - A.data :hash::STRING AS tx_hash, - A.data :input::STRING AS input_data, - SUBSTR(input_data, 1, 10) AS origin_function_signature, - livequery_dev.utils.udf_hex_to_int(A.data :maxFeePerGas::STRING)::INT / pow(10, 9) AS max_fee_per_gas, - livequery_dev.utils.udf_hex_to_int( - A.data :maxPriorityFeePerGas::STRING - )::INT / pow(10, 9) AS max_priority_fee_per_gas, - livequery_dev.utils.udf_hex_to_int(A.data :nonce::STRING)::INT AS nonce, - A.data :r::STRING AS r, - A.data :s::STRING AS s, - A.data :to::STRING AS to_address, - livequery_dev.utils.udf_hex_to_int(A.data :transactionIndex::STRING)::INT AS POSITION, - A.data :type::STRING AS TYPE, - A.data :v::STRING AS v, - livequery_dev.utils.udf_hex_to_int(A.data :value::STRING) AS value_precise_raw, - value_precise_raw * power(10, -18) AS value_precise, - value_precise::FLOAT AS VALUE, - A.data :accessList AS access_list, - A.data, - A.data: blobVersionedHashes::ARRAY AS blob_versioned_hashes, - livequery_dev.utils.udf_hex_to_int(A.data: maxFeePerGas::STRING)::INT AS max_fee_per_blob_gas, - block_timestamp, - CASE - WHEN block_timestamp IS NULL - OR tx_status IS NULL THEN TRUE - ELSE FALSE - END AS is_pending, - r.gas_used, - tx_success, - tx_status, - cumulative_gas_used, - effective_gas_price, - livequery_dev.utils.udf_hex_to_int(A.data :gasPrice) * power(10, -18) * r.gas_used AS tx_fee_precise, - COALESCE(tx_fee_precise::FLOAT, 0) AS tx_fee, - r.type as tx_type, - r.blob_gas_used, - r.blob_gas_price, - from - raw_txs A - left join blocks b on b.block_number = A.block_number - left join receipts as r on r.tx_hash = A.data :hash::STRING - ) - SELECT - block_number, - block_timestamp, - block_hash, - tx_hash, - nonce, - POSITION, - origin_function_signature, - from_address, - to_address, - VALUE, - value_precise_raw, - value_precise::STRING as value_precise, - tx_fee, - tx_fee_precise::STRING as tx_fee_precise, - gas_price, - gas AS gas_limit, - gas_used, - cumulative_gas_used, - input_data, - tx_status AS status, - effective_gas_price, - max_fee_per_gas, - max_priority_fee_per_gas, - r, - s, - v, - tx_type, - chain_id, - blob_versioned_hashes, - max_fee_per_blob_gas, - blob_gas_used, - blob_gas_price, - md5( - cast( - coalesce( - cast(tx_hash as TEXT), - '_dbt_utils_surrogate_key_null_' - ) as TEXT - ) - ) AS fact_transactions_id, - SYSDATE() inserted_timestamp, - SYSDATE() AS modified_timestamp - FROM - txs +WITH spine AS ( + {{ evm_live_view_target_blocks(schema, blockchain, network) | indent(4) -}} +), +raw_receipts AS ( + {{ evm_live_view_bronze_receipts(schema, 'spine') | indent(4) -}} +), +raw_block_txs AS ( +{{ evm_live_view_bronze_blocks(schema, 'spine') | indent(4) -}} +), +raw_transactions AS ( + {{ evm_live_view_bronze_transactions('raw_block_txs') | indent(4) -}} +), +blocks AS ( + {{ evm_live_view_silver_blocks('raw_block_txs') | indent(4) -}} +), +receipts AS ( + {{ evm_live_view_silver_receipts('raw_receipts') | indent(4) -}} +), +transactions AS ( + {{ evm_live_view_silver_transactions('raw_transactions', 'blocks', 'receipts') | indent(4) -}} +), +SELECT + block_number, + block_timestamp, + block_hash, + tx_hash, + nonce, + POSITION, + origin_function_signature, + from_address, + to_address, + VALUE, + value_precise_raw, + value_precise::STRING as value_precise, + tx_fee, + tx_fee_precise::STRING as tx_fee_precise, + gas_price, + gas AS gas_limit, + gas_used, + cumulative_gas_used, + input_data, + tx_status AS status, + effective_gas_price, + max_fee_per_gas, + max_priority_fee_per_gas, + r, + s, + v, + tx_type, + chain_id, + blob_versioned_hashes, + max_fee_per_blob_gas, + blob_gas_used, + blob_gas_price, + md5( + cast( + coalesce( + cast(tx_hash as TEXT), + '_dbt_utils_surrogate_key_null_' + ) as TEXT + ) + ) AS fact_transactions_id, + SYSDATE() inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + transactions {% endmacro %} From 212f49e7e5402818d1a7e2edfce523173e92d496 Mon Sep 17 00:00:00 2001 From: shah Date: Wed, 16 Oct 2024 20:49:11 -0700 Subject: [PATCH 27/50] STREAM-1053 Fix refactored reusable CTE column references --- macros/evm/evm_live_views.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 4095605..71ff2ad 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -475,7 +475,7 @@ raw_receipts AS ( {{ evm_live_view_bronze_receipts(schema, 'spine') | indent(4) -}} ), raw_block_txs AS ( -{{ evm_live_view_bronze_blocks(schema, 'spine') | indent(4) -}} + {{ evm_live_view_bronze_blocks(schema, 'spine') | indent(4) -}} ), raw_transactions AS ( {{ evm_live_view_bronze_transactions('raw_block_txs') | indent(4) -}} @@ -488,7 +488,7 @@ receipts AS ( ), transactions AS ( {{ evm_live_view_silver_transactions('raw_transactions', 'blocks', 'receipts') | indent(4) -}} -), +) SELECT block_number, block_timestamp, @@ -498,7 +498,7 @@ SELECT POSITION, origin_function_signature, from_address, - to_address, + to_address1 as to_address, VALUE, value_precise_raw, value_precise::STRING as value_precise, From 6f19eb0b9463e1bd1a1aa827d20c74ddb7d5b47c Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Thu, 17 Oct 2024 20:11:19 +0900 Subject: [PATCH 28/50] remove ez_token_transfers_id from signature --- macros/evm/evm.yaml.sql | 1 - macros/evm/evm_live_views.sql | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index 8ccd16c..ca5c4e7 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -450,7 +450,6 @@ signature: - [block_height, INTEGER, The start block height to get the transfers from] - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] - - [ez_token_transfers_id, STRING, The topic of the token transfers to get] return_type: - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, event_index INTEGER, origin_function_signature STRING, origin_from_address STRING, origin_to_address STRING, contract_address STRING, from_address STRING, to_address STRING, raw_amount_precise STRING, raw_amount FLOAT, amount_precise FLOAT, amount FLOAT, amount_usd FLOAT, decimals INTEGER, symbol STRING, token_price FLOAT, has_decimal STRING, has_price STRING, _log_id STRING, ez_token_transfers_id STRING, _inserted_timestamp TIMESTAMP_NTZ, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" options: | diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 71ff2ad..eabf54c 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -1096,7 +1096,7 @@ FROM AND DATE_TRUNC('hour', l.block_timestamp) = HOUR LEFT JOIN {{ blockchain }}.core.DIM_CONTRACTS C ON l.contract_address = C.address WHERE - topics [0]::STRING = ez_token_transfers_id + topics [0]::STRING = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' AND tx_status = 'SUCCESS' and raw_amount IS NOT NULL AND to_address IS NOT NULL From 2fb84f14d0fb62c694068462bee15d01abd06a08 Mon Sep 17 00:00:00 2001 From: shah Date: Thu, 17 Oct 2024 13:01:54 -0700 Subject: [PATCH 29/50] STREAM-1049 Pin view columns for mapping_tables --- models/mapping_tables/_eth__decoded_logs.sql | 18 +++++++++++++++++- models/mapping_tables/_eth__logs.sql | 15 ++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/models/mapping_tables/_eth__decoded_logs.sql b/models/mapping_tables/_eth__decoded_logs.sql index 6afcf20..f3090d8 100644 --- a/models/mapping_tables/_eth__decoded_logs.sql +++ b/models/mapping_tables/_eth__decoded_logs.sql @@ -3,7 +3,23 @@ grants = {'+select': fromyaml(var('ROLES'))} ) }} -SELECT * +SELECT + BLOCK_NUMBER, + BLOCK_TIMESTAMP, + TX_HASH, + EVENT_INDEX, + CONTRACT_ADDRESS, + CONTRACT_NAME, + EVENT_NAME, + DECODED_LOG, + FULL_DECODED_LOG, + ORIGIN_FUNCTION_SIGNATURE, + ORIGIN_FROM_ADDRESS, + ORIGIN_TO_ADDRESS, + TOPICS, + DATA, + EVENT_REMOVED, + TX_STATUS FROM {{ source( 'ethereum_core', diff --git a/models/mapping_tables/_eth__logs.sql b/models/mapping_tables/_eth__logs.sql index f430b00..7de3896 100644 --- a/models/mapping_tables/_eth__logs.sql +++ b/models/mapping_tables/_eth__logs.sql @@ -3,7 +3,20 @@ grants = {'+select': fromyaml(var('ROLES'))} ) }} -SELECT * +SELECT + BLOCK_NUMBER, + BLOCK_TIMESTAMP, + TX_HASH, + ORIGIN_FUNCTION_SIGNATURE, + ORIGIN_FROM_ADDRESS, + ORIGIN_TO_ADDRESS, + EVENT_INDEX, + CONTRACT_ADDRESS, + TOPICS, + DATA, + EVENT_REMOVED, + TX_STATUS, + _LOG_ID FROM {{ source( 'ethereum_core', From 87b8eda07ab3c747268940f33f5f8b5c203a9dba Mon Sep 17 00:00:00 2001 From: shah Date: Thu, 17 Oct 2024 15:53:13 -0700 Subject: [PATCH 30/50] STREAM-1062 Refactor fact_traces UDTF SQL | Add evm_live_view_bronze_traces CTE macro --- macros/evm/evm_live_views.sql | 833 ++++++++++++++-------------------- 1 file changed, 340 insertions(+), 493 deletions(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index eabf54c..810c9b7 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -77,6 +77,20 @@ FROM lateral flatten(r.data:transactions) v {% endmacro %} +{% macro evm_live_view_bronze_traces(schema, table_name)%} +SELECT + s.block_number, + v.index::INT AS tx_position, -- mimic's streamline's logic to add tx_position + v.value:result AS full_traces, + SYSDATE() AS _inserted_timestamp +FROM {{ table_name }} s, +LATERAL FLATTEN(input => PARSE_JSON( + {{ schema }}.udf_rpc( + 'debug_traceBlockByNumber', + [utils.udf_int_to_hex(s.block_number), {'tracer': 'callTracer'}]) +)) v +{% endmacro %} + -- Transformation macro for EVM chains {% macro evm_live_view_silver_blocks(table_name) %} SELECT @@ -230,6 +244,267 @@ FROM LEFT JOIN {{ silver_receipts }} AS r on r.tx_hash = A.data :hash::STRING {% endmacro %} +{% macro evm_live_view_silver_traces(raw_traces) %} +flatten_traces AS ( + SELECT + block_number, + tx_position, + IFF( + path IN ( + 'result', + 'result.value', + 'result.type', + 'result.to', + 'result.input', + 'result.gasUsed', + 'result.gas', + 'result.from', + 'result.output', + 'result.error', + 'result.revertReason', + 'gasUsed', + 'gas', + 'type', + 'to', + 'from', + 'value', + 'input', + 'error', + 'output', + 'revertReason' + ), + 'ORIGIN', + REGEXP_REPLACE(REGEXP_REPLACE(path, '[^0-9]+', '_'), '^_|_$', '') + ) AS trace_address, + _inserted_timestamp, + OBJECT_AGG( + key, + VALUE + ) AS trace_json, + CASE + WHEN trace_address = 'ORIGIN' THEN NULL + WHEN POSITION( + '_' IN trace_address + ) = 0 THEN 'ORIGIN' + ELSE REGEXP_REPLACE( + trace_address, + '_[0-9]+$', + '', + 1, + 1 + ) + END AS parent_trace_address, + SPLIT( + trace_address, + '_' + ) AS str_array + FROM + {{ raw_traces }}, + TABLE( + FLATTEN( + input => PARSE_JSON(full_traces), + recursive => TRUE + ) + ) f + WHERE + f.index IS NULL + AND f.key != 'calls' + AND f.path != 'result' + GROUP BY + block_number, + tx_position, + trace_address, + _inserted_timestamp +), +sub_traces AS ( + SELECT + block_number, + tx_position, + parent_trace_address, + COUNT(*) AS sub_traces + FROM + flatten_traces + GROUP BY + block_number, + tx_position, + parent_trace_address +), +num_array AS ( + SELECT + block_number, + tx_position, + trace_address, + ARRAY_AGG(flat_value) AS num_array + FROM + ( + SELECT + block_number, + tx_position, + trace_address, + IFF( + VALUE :: STRING = 'ORIGIN', + -1, + VALUE :: INT + ) AS flat_value + FROM + flatten_traces, + LATERAL FLATTEN ( + input => str_array + ) + ) + GROUP BY + block_number, + tx_position, + trace_address +), +cleaned_traces AS ( + SELECT + b.block_number, + b.tx_position, + b.trace_address, + IFNULL( + sub_traces, + 0 + ) AS sub_traces, + num_array, + ROW_NUMBER() over ( + PARTITION BY b.block_number, + b.tx_position + ORDER BY + num_array ASC + ) - 1 AS trace_index, + trace_json, + b._inserted_timestamp + FROM + flatten_traces b + LEFT JOIN sub_traces s + ON b.block_number = s.block_number + AND b.tx_position = s.tx_position + AND b.trace_address = s.parent_trace_address + JOIN num_array n + ON b.block_number = n.block_number + AND b.tx_position = n.tx_position + AND b.trace_address = n.trace_address +), +final_traces AS ( + SELECT + tx_position, + trace_index, + block_number, + trace_address, + trace_json :error :: STRING AS error_reason, + trace_json :from :: STRING AS from_address, + trace_json :to :: STRING AS to_address, + IFNULL( + utils.udf_hex_to_int( + trace_json :value :: STRING + ), + '0' + ) AS eth_value_precise_raw, + ethereum.utils.udf_decimal_adjust( + eth_value_precise_raw, + 18 + ) AS eth_value_precise, + eth_value_precise :: FLOAT AS eth_value, + utils.udf_hex_to_int( + trace_json :gas :: STRING + ) :: INT AS gas, + utils.udf_hex_to_int( + trace_json :gasUsed :: STRING + ) :: INT AS gas_used, + trace_json :input :: STRING AS input, + trace_json :output :: STRING AS output, + trace_json :type :: STRING AS TYPE, + concat_ws( + '_', + TYPE, + trace_address + ) AS identifier, + concat_ws( + '-', + block_number, + tx_position, + identifier + ) AS _call_id, + _inserted_timestamp, + trace_json AS DATA, + sub_traces + FROM + cleaned_traces +), +new_records AS ( + SELECT + f.block_number, + t.tx_hash, + t.block_timestamp, + t.tx_status, + f.tx_position, + f.trace_index, + f.from_address, + f.to_address, + f.eth_value_precise_raw, + f.eth_value_precise, + f.eth_value, + f.gas, + f.gas_used, + f.input, + f.output, + f.type, + f.identifier, + f.sub_traces, + f.error_reason, + IFF( + f.error_reason IS NULL, + 'SUCCESS', + 'FAIL' + ) AS trace_status, + f.data, + IFF( + t.tx_hash IS NULL + OR t.block_timestamp IS NULL + OR t.tx_status IS NULL, + TRUE, + FALSE + ) AS is_pending, + f._call_id, + f._inserted_timestamp + FROM + final_traces f + LEFT OUTER JOIN transactions t + ON f.tx_position = t.position + AND f.block_number = t.block_number +), +traces_final AS ( + SELECT + block_number, + tx_hash, + block_timestamp, + tx_status, + tx_position, + trace_index, + from_address, + to_address, + eth_value_precise_raw, + eth_value_precise, + eth_value, + gas, + gas_used, + input, + output, + TYPE, + identifier, + sub_traces, + error_reason, + trace_status, + DATA, + is_pending, + _call_id, + _inserted_timestamp + FROM + new_records +) +{% endmacro %} + -- Get EVM chain fact data {% macro evm_live_view_fact_blocks(schema, blockchain, network) %} WITH spine AS ( @@ -538,499 +813,71 @@ FROM {% endmacro %} {% macro evm_live_view_fact_traces(schema, blockchain, network) %} -WITH heights AS ( - SELECT - {{ schema }}.udf_rpc('eth_blockNumber', []) as result, - livequery_dev.utils.udf_hex_to_int(result:result)::integer as latest_block_height, - coalesce( - block_height, - latest_block_height - ) as min_height, - iff( - coalesce(to_latest, false), - latest_block_height, - min_height - ) as max_height - ), - spine as ( - select - row_number() over ( - order by - null - ) -1 + coalesce(block_height, 0)::integer as block_number, - min_height, - iff( - coalesce(to_latest, false), - latest_block_height, - min_height - ) as max_height, - latest_block_height - from - table(generator(ROWCOUNT => 1000)), - heights - qualify block_number between min_height and max_height - ), - raw_receipts as ( - SELECT - latest_block_height, - block_number, - {{ schema }}.udf_rpc( - 'eth_getBlockReceipts', - [utils.udf_int_to_hex(block_number)]) AS result, - v.value as DATA - from - spine, - lateral flatten(result) v - ), - raw_block_txs as ( - SELECT - block_number, - {{ schema }}.udf_rpc( - 'eth_getBlockByNumber', - [utils.udf_int_to_hex(block_number), true]) AS DATA - from - spine - ), - raw_txs as ( - SELECT - block_number, - v.value as DATA - from - raw_block_txs r, - lateral flatten(r.data:transactions) v - ), - blocks as ( - select - block_number, - livequery_dev.utils.udf_hex_to_int(DATA :baseFeePerGas::STRING)::INT AS base_fee_per_gas, - livequery_dev.utils.udf_hex_to_int(DATA :difficulty::STRING)::INT AS difficulty, - DATA :extraData::STRING AS extra_data, - livequery_dev.utils.udf_hex_to_int(DATA :gasLimit::STRING)::INT AS gas_limit, - livequery_dev.utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS gas_used, - DATA :hash::STRING AS HASH, - DATA :logsBloom::STRING AS logs_bloom, - DATA :miner::STRING AS miner, - livequery_dev.utils.udf_hex_to_int(DATA :nonce::STRING)::INT AS nonce, - livequery_dev.utils.udf_hex_to_int(DATA :number::STRING)::INT AS NUMBER, - DATA :parentHash::STRING AS parent_hash, - DATA :receiptsRoot::STRING AS receipts_root, - DATA :sha3Uncles::STRING AS sha3_uncles, - livequery_dev.utils.udf_hex_to_int(DATA :size::STRING)::INT AS SIZE, - DATA :stateRoot::STRING AS state_root, - livequery_dev.utils.udf_hex_to_int(DATA :timestamp::STRING)::TIMESTAMP AS block_timestamp, - livequery_dev.utils.udf_hex_to_int(DATA :totalDifficulty::STRING)::INT AS total_difficulty, - ARRAY_SIZE(DATA :transactions) AS tx_count, - DATA :transactionsRoot::STRING AS transactions_root, - DATA :uncles AS uncles, - DATA :withdrawals AS withdrawals, - DATA :withdrawalsRoot::STRING AS withdrawals_root, - md5( - cast( - coalesce( - cast(block_number as TEXT), - '_dbt_utils_surrogate_key_null_' - ) as TEXT - ) - ) AS blocks_id, - livequery_dev.utils.udf_hex_to_int(DATA: blobGasUsed::STRING)::INT AS blob_gas_used, - livequery_dev.utils.udf_hex_to_int(DATA: excessBlobGas::STRING)::INT AS excess_blob_gas, - from - raw_block_txs - ), - receipts as ( - select - latest_block_height, - block_number, - DATA :blockHash::STRING AS block_hash, - livequery_dev.utils.udf_hex_to_int(DATA :blockNumber::STRING)::INT AS blockNumber, - livequery_dev.utils.udf_hex_to_int(DATA :cumulativeGasUsed::STRING)::INT AS cumulative_gas_used, - livequery_dev.utils.udf_hex_to_int(DATA :effectiveGasPrice::STRING)::INT / pow(10, 9) AS effective_gas_price, - DATA :from::STRING AS from_address, - livequery_dev.utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS gas_used, - DATA :logs AS logs, - DATA :logsBloom::STRING AS logs_bloom, - livequery_dev.utils.udf_hex_to_int(DATA :status::STRING)::INT AS status, - CASE - WHEN status = 1 THEN TRUE - ELSE FALSE - END AS tx_success, - CASE - WHEN status = 1 THEN 'SUCCESS' - ELSE 'FAIL' - END AS tx_status, - DATA :to::STRING AS to_address1, - CASE - WHEN to_address1 = '' THEN NULL - ELSE to_address1 - END AS to_address, - DATA :transactionHash::STRING AS tx_hash, - livequery_dev.utils.udf_hex_to_int(DATA :transactionIndex::STRING)::INT AS POSITION, - livequery_dev.utils.udf_hex_to_int(DATA :type::STRING)::INT AS TYPE, - livequery_dev.utils.udf_hex_to_int(DATA :effectiveGasPrice::STRING)::INT AS blob_gas_price, - livequery_dev.utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS blob_gas_used - from - raw_receipts - ), - txs as ( - select - A.block_number AS block_number, - A.data :blockHash::STRING AS block_hash, - livequery_dev.utils.udf_hex_to_int(A.data :blockNumber::STRING)::INT AS blockNumber, - livequery_dev.utils.udf_hex_to_int(A.data :chainId::STRING)::INT AS chain_id, - A.data :from::STRING AS from_address, - livequery_dev.utils.udf_hex_to_int(A.data :gas::STRING)::INT AS gas, - livequery_dev.utils.udf_hex_to_int(A.data :gasPrice::STRING)::INT / pow(10, 9) AS gas_price, - A.data :hash::STRING AS tx_hash, - A.data :input::STRING AS input_data, - SUBSTR(input_data, 1, 10) AS origin_function_signature, - livequery_dev.utils.udf_hex_to_int(A.data :maxFeePerGas::STRING)::INT / pow(10, 9) AS max_fee_per_gas, - livequery_dev.utils.udf_hex_to_int( - A.data :maxPriorityFeePerGas::STRING - )::INT / pow(10, 9) AS max_priority_fee_per_gas, - livequery_dev.utils.udf_hex_to_int(A.data :nonce::STRING)::INT AS nonce, - A.data :r::STRING AS r, - A.data :s::STRING AS s, - A.data :to::STRING AS to_address1, - livequery_dev.utils.udf_hex_to_int(A.data :transactionIndex::STRING)::INT AS POSITION, - A.data :type::STRING AS TYPE, - A.data :v::STRING AS v, - livequery_dev.utils.udf_hex_to_int(A.data :value::STRING) AS value_precise_raw, - value_precise_raw * power(10, -18) AS value_precise, - value_precise::FLOAT AS VALUE, - A.data :accessList AS access_list, - A.data, - A.data: blobVersionedHashes::ARRAY AS blob_versioned_hashes, - livequery_dev.utils.udf_hex_to_int(A.data: maxFeePerGas::STRING)::INT AS max_fee_per_blob_gas, - block_timestamp, - CASE - WHEN block_timestamp IS NULL - OR tx_status IS NULL THEN TRUE - ELSE FALSE - END AS is_pending, - r.gas_used, - tx_success, - tx_status, - cumulative_gas_used, - effective_gas_price, - livequery_dev.utils.udf_hex_to_int(A.data :gasPrice) * power(10, -18) * r.gas_used AS tx_fee_precise, - COALESCE(tx_fee_precise::FLOAT, 0) AS tx_fee, - r.type as tx_type, - r.blob_gas_used, - r.blob_gas_price, - from - raw_txs A - left join blocks b on b.block_number = A.block_number - left join receipts as r on r.tx_hash = A.data :hash::STRING - ), - raw_traces AS ( - SELECT - s.block_number, - v.index::INT AS tx_position, - v.value:result AS full_traces, - SYSDATE() AS _inserted_timestamp - FROM spine s, - LATERAL FLATTEN(input => PARSE_JSON( - {{ schema }}.udf_rpc( - 'debug_traceBlockByNumber', - [utils.udf_int_to_hex(s.block_number), {'tracer': 'callTracer'}]) - )) v - ), - flatten_traces AS ( - SELECT - block_number, - tx_position, - IFF( - path IN ( - 'result', - 'result.value', - 'result.type', - 'result.to', - 'result.input', - 'result.gasUsed', - 'result.gas', - 'result.from', - 'result.output', - 'result.error', - 'result.revertReason', - 'gasUsed', - 'gas', - 'type', - 'to', - 'from', - 'value', - 'input', - 'error', - 'output', - 'revertReason' - ), - 'ORIGIN', - REGEXP_REPLACE(REGEXP_REPLACE(path, '[^0-9]+', '_'), '^_|_$', '') - ) AS trace_address, - _inserted_timestamp, - OBJECT_AGG( - key, - VALUE - ) AS trace_json, - CASE - WHEN trace_address = 'ORIGIN' THEN NULL - WHEN POSITION( - '_' IN trace_address - ) = 0 THEN 'ORIGIN' - ELSE REGEXP_REPLACE( - trace_address, - '_[0-9]+$', - '', - 1, - 1 - ) - END AS parent_trace_address, - SPLIT( - trace_address, - '_' - ) AS str_array - FROM - raw_traces, - TABLE( - FLATTEN( - input => PARSE_JSON(full_traces), - recursive => TRUE - ) - ) f - WHERE - f.index IS NULL - AND f.key != 'calls' - AND f.path != 'result' - GROUP BY - block_number, - tx_position, - trace_address, - _inserted_timestamp - ), - sub_traces AS ( - SELECT - block_number, - tx_position, - parent_trace_address, - COUNT(*) AS sub_traces - FROM - flatten_traces - GROUP BY - block_number, - tx_position, - parent_trace_address - ), - num_array AS ( - SELECT - block_number, - tx_position, - trace_address, - ARRAY_AGG(flat_value) AS num_array - FROM - ( - SELECT - block_number, - tx_position, - trace_address, - IFF( - VALUE :: STRING = 'ORIGIN', - -1, - VALUE :: INT - ) AS flat_value - FROM - flatten_traces, - LATERAL FLATTEN ( - input => str_array - ) - ) - GROUP BY - block_number, - tx_position, - trace_address - ), - cleaned_traces AS ( - SELECT - b.block_number, - b.tx_position, - b.trace_address, - IFNULL( - sub_traces, - 0 - ) AS sub_traces, - num_array, - ROW_NUMBER() over ( - PARTITION BY b.block_number, - b.tx_position - ORDER BY - num_array ASC - ) - 1 AS trace_index, - trace_json, - b._inserted_timestamp - FROM - flatten_traces b - LEFT JOIN sub_traces s - ON b.block_number = s.block_number - AND b.tx_position = s.tx_position - AND b.trace_address = s.parent_trace_address - JOIN num_array n - ON b.block_number = n.block_number - AND b.tx_position = n.tx_position - AND b.trace_address = n.trace_address - ), - final_traces AS ( - SELECT - tx_position, - trace_index, - block_number, - trace_address, - trace_json :error :: STRING AS error_reason, - trace_json :from :: STRING AS from_address, - trace_json :to :: STRING AS to_address, - IFNULL( - utils.udf_hex_to_int( - trace_json :value :: STRING - ), - '0' - ) AS eth_value_precise_raw, - ethereum.utils.udf_decimal_adjust( - eth_value_precise_raw, - 18 - ) AS eth_value_precise, - eth_value_precise :: FLOAT AS eth_value, - utils.udf_hex_to_int( - trace_json :gas :: STRING - ) :: INT AS gas, - utils.udf_hex_to_int( - trace_json :gasUsed :: STRING - ) :: INT AS gas_used, - trace_json :input :: STRING AS input, - trace_json :output :: STRING AS output, - trace_json :type :: STRING AS TYPE, - concat_ws( - '_', - TYPE, - trace_address - ) AS identifier, - concat_ws( - '-', - block_number, - tx_position, - identifier - ) AS _call_id, - _inserted_timestamp, - trace_json AS DATA, - sub_traces - FROM - cleaned_traces - ), - new_records AS ( - SELECT - f.block_number, - t.tx_hash, - t.block_timestamp, - t.tx_status, - f.tx_position, - f.trace_index, - f.from_address, - f.to_address, - f.eth_value_precise_raw, - f.eth_value_precise, - f.eth_value, - f.gas, - f.gas_used, - f.input, - f.output, - f.type, - f.identifier, - f.sub_traces, - f.error_reason, - IFF( - f.error_reason IS NULL, - 'SUCCESS', - 'FAIL' - ) AS trace_status, - f.data, - IFF( - t.tx_hash IS NULL - OR t.block_timestamp IS NULL - OR t.tx_status IS NULL, - TRUE, - FALSE - ) AS is_pending, - f._call_id, - f._inserted_timestamp - FROM - final_traces f - LEFT OUTER JOIN ethereum.silver.transactions t - ON f.tx_position = t.position - AND f.block_number = t.block_number - ), - traces_final AS ( - SELECT - block_number, - tx_hash, - block_timestamp, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - eth_value_precise_raw, - eth_value_precise, - eth_value, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - trace_status, - DATA, - is_pending, - _call_id, - _inserted_timestamp - FROM - new_records - ) - SELECT - tx_hash, - block_number, - block_timestamp, - from_address, - to_address, - eth_value AS VALUE, - eth_value_precise_raw AS value_precise_raw, - eth_value_precise AS value_precise, - gas, - gas_used, - input, - output, - TYPE, - identifier, - DATA, - tx_status, - sub_traces, - trace_status, - error_reason, - trace_index, - md5( - cast( - coalesce( - cast(tx_hash as TEXT), - '_dbt_utils_surrogate_key_null_' - ) || '-' || coalesce( - cast(trace_index as TEXT), - '_dbt_utils_surrogate_key_null_' - ) as TEXT - ) - ) as fact_traces_id, - COALESCE( - _inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - SYSDATE() AS modified_timestamp - FROM traces_final +WITH spine AS ( + {{ evm_live_view_target_blocks(schema, blockchain, network) | indent(4) -}} +), +raw_receipts AS ( + {{ evm_live_view_bronze_receipts(schema, 'spine') | indent(4) -}} +), +raw_block_txs AS ( + {{ evm_live_view_bronze_blocks(schema, 'spine') | indent(4) -}} +), +raw_transactions AS ( + {{ evm_live_view_bronze_transactions('raw_block_txs') | indent(4) -}} +), +blocks AS ( + {{ evm_live_view_silver_blocks('raw_block_txs') | indent(4) -}} +), +receipts AS ( + {{ evm_live_view_silver_receipts('raw_receipts') | indent(4) -}} +), +transactions AS ( + {{ evm_live_view_silver_transactions('raw_transactions', 'blocks', 'receipts') | indent(4) -}} +), +raw_traces AS ( + {{ evm_live_view_bronze_traces(schema, 'spine') | indent(4) -}} +), + +{{ evm_live_view_silver_traces('raw_traces') | indent(4) -}} + +SELECT + tx_hash, + block_number, + block_timestamp, + from_address, + to_address, + eth_value AS VALUE, + eth_value_precise_raw AS value_precise_raw, + eth_value_precise AS value_precise, + gas, + gas_used, + input, + output, + TYPE, + identifier, + DATA, + tx_status, + sub_traces, + trace_status, + error_reason, + trace_index, + md5( + cast( + coalesce( + cast(tx_hash as TEXT), + '_dbt_utils_surrogate_key_null_' + ) || '-' || coalesce( + cast(trace_index as TEXT), + '_dbt_utils_surrogate_key_null_' + ) as TEXT + ) + ) as fact_traces_id, + COALESCE( + _inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM traces_final {% endmacro %} -- Get EVM chain ez data From cf3134080a2c26354ef0bd57997a79611342afda Mon Sep 17 00:00:00 2001 From: shah Date: Thu, 17 Oct 2024 16:56:41 -0700 Subject: [PATCH 31/50] STREAM-1051 Refactor ez_native_transfers to use reusable CTE macros --- macros/evm/evm_live_views.sql | 711 +++++++--------------------------- 1 file changed, 141 insertions(+), 570 deletions(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 810c9b7..9dd77b7 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -951,580 +951,151 @@ AND from_address IS NOT NULL {% endmacro %} {% macro evm_live_view_ez_native_transfers(schema, blockchain, network) %} -WITH heights AS ( - SELECT - {{ schema }}.udf_rpc('eth_blockNumber', []) as result, - livequery_dev.utils.udf_hex_to_int(result:result)::integer as latest_block_height, - coalesce( - block_height, - latest_block_height - ) as min_height, - iff( - coalesce(to_latest, false), - latest_block_height, - min_height - ) as max_height - ), - spine as ( - select - row_number() over ( - order by - null - ) -1 + coalesce(block_height, 0)::integer as block_number, - min_height, - iff( - coalesce(to_latest, false), - latest_block_height, - min_height - ) as max_height, - latest_block_height - from - table(generator(ROWCOUNT => 1000)), - heights - qualify block_number between min_height and max_height - ), - raw_receipts as ( - SELECT - latest_block_height, - block_number, - {{ schema }}.udf_rpc( - 'eth_getBlockReceipts', - [utils.udf_int_to_hex(block_number)]) AS result, - v.value as DATA - from - spine, - lateral flatten(result) v - ), - raw_block_txs as ( - SELECT - block_number, - {{ schema }}.udf_rpc( - 'eth_getBlockByNumber', - [utils.udf_int_to_hex(block_number), true]) AS DATA - from - spine - ), - raw_txs as ( - SELECT - block_number, - v.value as DATA - from - raw_block_txs r, - lateral flatten(r.data:transactions) v - ), - blocks as ( - select - block_number, - livequery_dev.utils.udf_hex_to_int(DATA :baseFeePerGas::STRING)::INT AS base_fee_per_gas, - livequery_dev.utils.udf_hex_to_int(DATA :difficulty::STRING)::INT AS difficulty, - DATA :extraData::STRING AS extra_data, - livequery_dev.utils.udf_hex_to_int(DATA :gasLimit::STRING)::INT AS gas_limit, - livequery_dev.utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS gas_used, - DATA :hash::STRING AS HASH, - DATA :logsBloom::STRING AS logs_bloom, - DATA :miner::STRING AS miner, - livequery_dev.utils.udf_hex_to_int(DATA :nonce::STRING)::INT AS nonce, - livequery_dev.utils.udf_hex_to_int(DATA :number::STRING)::INT AS NUMBER, - DATA :parentHash::STRING AS parent_hash, - DATA :receiptsRoot::STRING AS receipts_root, - DATA :sha3Uncles::STRING AS sha3_uncles, - livequery_dev.utils.udf_hex_to_int(DATA :size::STRING)::INT AS SIZE, - DATA :stateRoot::STRING AS state_root, - livequery_dev.utils.udf_hex_to_int(DATA :timestamp::STRING)::TIMESTAMP AS block_timestamp, - livequery_dev.utils.udf_hex_to_int(DATA :totalDifficulty::STRING)::INT AS total_difficulty, - ARRAY_SIZE(DATA :transactions) AS tx_count, - DATA :transactionsRoot::STRING AS transactions_root, - DATA :uncles AS uncles, - DATA :withdrawals AS withdrawals, - DATA :withdrawalsRoot::STRING AS withdrawals_root, - md5( - cast( - coalesce( - cast(block_number as TEXT), - '_dbt_utils_surrogate_key_null_' - ) as TEXT - ) - ) AS blocks_id, - livequery_dev.utils.udf_hex_to_int(DATA: blobGasUsed::STRING)::INT AS blob_gas_used, - livequery_dev.utils.udf_hex_to_int(DATA: excessBlobGas::STRING)::INT AS excess_blob_gas, - from - raw_block_txs - ), - receipts as ( - select - latest_block_height, - block_number, - DATA :blockHash::STRING AS block_hash, - livequery_dev.utils.udf_hex_to_int(DATA :blockNumber::STRING)::INT AS blockNumber, - livequery_dev.utils.udf_hex_to_int(DATA :cumulativeGasUsed::STRING)::INT AS cumulative_gas_used, - livequery_dev.utils.udf_hex_to_int(DATA :effectiveGasPrice::STRING)::INT / pow(10, 9) AS effective_gas_price, - DATA :from::STRING AS from_address, - livequery_dev.utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS gas_used, - DATA :logs AS logs, - DATA :logsBloom::STRING AS logs_bloom, - livequery_dev.utils.udf_hex_to_int(DATA :status::STRING)::INT AS status, - CASE - WHEN status = 1 THEN TRUE - ELSE FALSE - END AS tx_success, - CASE - WHEN status = 1 THEN 'SUCCESS' - ELSE 'FAIL' - END AS tx_status, - DATA :to::STRING AS to_address1, - CASE - WHEN to_address1 = '' THEN NULL - ELSE to_address1 - END AS to_address, - DATA :transactionHash::STRING AS tx_hash, - livequery_dev.utils.udf_hex_to_int(DATA :transactionIndex::STRING)::INT AS POSITION, - livequery_dev.utils.udf_hex_to_int(DATA :type::STRING)::INT AS TYPE, - livequery_dev.utils.udf_hex_to_int(DATA :effectiveGasPrice::STRING)::INT AS blob_gas_price, - livequery_dev.utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS blob_gas_used - from - raw_receipts - ), - txs as ( - select - A.block_number AS block_number, - A.data :blockHash::STRING AS block_hash, - livequery_dev.utils.udf_hex_to_int(A.data :blockNumber::STRING)::INT AS blockNumber, - livequery_dev.utils.udf_hex_to_int(A.data :chainId::STRING)::INT AS chain_id, - A.data :from::STRING AS from_address, - livequery_dev.utils.udf_hex_to_int(A.data :gas::STRING)::INT AS gas, - livequery_dev.utils.udf_hex_to_int(A.data :gasPrice::STRING)::INT / pow(10, 9) AS gas_price, - A.data :hash::STRING AS tx_hash, - A.data :input::STRING AS input_data, - SUBSTR(input_data, 1, 10) AS origin_function_signature, - livequery_dev.utils.udf_hex_to_int(A.data :maxFeePerGas::STRING)::INT / pow(10, 9) AS max_fee_per_gas, - livequery_dev.utils.udf_hex_to_int( - A.data :maxPriorityFeePerGas::STRING - )::INT / pow(10, 9) AS max_priority_fee_per_gas, - livequery_dev.utils.udf_hex_to_int(A.data :nonce::STRING)::INT AS nonce, - A.data :r::STRING AS r, - A.data :s::STRING AS s, - A.data :to::STRING AS to_address1, - livequery_dev.utils.udf_hex_to_int(A.data :transactionIndex::STRING)::INT AS POSITION, - A.data :type::STRING AS TYPE, - A.data :v::STRING AS v, - livequery_dev.utils.udf_hex_to_int(A.data :value::STRING) AS value_precise_raw, - value_precise_raw * power(10, -18) AS value_precise, - value_precise::FLOAT AS VALUE, - A.data :accessList AS access_list, - A.data, - A.data: blobVersionedHashes::ARRAY AS blob_versioned_hashes, - livequery_dev.utils.udf_hex_to_int(A.data: maxFeePerGas::STRING)::INT AS max_fee_per_blob_gas, - block_timestamp, - CASE - WHEN block_timestamp IS NULL - OR tx_status IS NULL THEN TRUE - ELSE FALSE - END AS is_pending, - r.gas_used, - tx_success, - tx_status, - cumulative_gas_used, - effective_gas_price, - livequery_dev.utils.udf_hex_to_int(A.data :gasPrice) * power(10, -18) * r.gas_used AS tx_fee_precise, - COALESCE(tx_fee_precise::FLOAT, 0) AS tx_fee, - r.type as tx_type, - r.blob_gas_used, - r.blob_gas_price, - from - raw_txs A - left join blocks b on b.block_number = A.block_number - left join receipts as r on r.tx_hash = A.data :hash::STRING - ), - raw_traces AS ( - SELECT - s.block_number, - v.index::INT AS tx_position, - v.value:result AS full_traces, - SYSDATE() AS _inserted_timestamp - FROM spine s, - LATERAL FLATTEN(input => PARSE_JSON( - {{ schema }}.udf_rpc( - 'debug_traceBlockByNumber', - [utils.udf_int_to_hex(s.block_number), {'tracer': 'callTracer'}]) - )) v - ), - flatten_traces AS ( - SELECT - block_number, - tx_position, - IFF( - path IN ( - 'result', - 'result.value', - 'result.type', - 'result.to', - 'result.input', - 'result.gasUsed', - 'result.gas', - 'result.from', - 'result.output', - 'result.error', - 'result.revertReason', - 'gasUsed', - 'gas', - 'type', - 'to', - 'from', - 'value', - 'input', - 'error', - 'output', - 'revertReason' - ), - 'ORIGIN', - REGEXP_REPLACE(REGEXP_REPLACE(path, '[^0-9]+', '_'), '^_|_$', '') - ) AS trace_address, - _inserted_timestamp, - OBJECT_AGG( - key, - VALUE - ) AS trace_json, - CASE - WHEN trace_address = 'ORIGIN' THEN NULL - WHEN POSITION( - '_' IN trace_address - ) = 0 THEN 'ORIGIN' - ELSE REGEXP_REPLACE( - trace_address, - '_[0-9]+$', - '', - 1, - 1 - ) - END AS parent_trace_address, - SPLIT( - trace_address, - '_' - ) AS str_array - FROM - raw_traces, - TABLE( - FLATTEN( - input => PARSE_JSON(full_traces), - recursive => TRUE - ) - ) f - WHERE - f.index IS NULL - AND f.key != 'calls' - AND f.path != 'result' - GROUP BY - block_number, - tx_position, - trace_address, - _inserted_timestamp - ), - sub_traces AS ( - SELECT - block_number, - tx_position, - parent_trace_address, - COUNT(*) AS sub_traces - FROM - flatten_traces - GROUP BY - block_number, - tx_position, - parent_trace_address - ), - num_array AS ( - SELECT - block_number, - tx_position, - trace_address, - ARRAY_AGG(flat_value) AS num_array - FROM - ( - SELECT - block_number, - tx_position, - trace_address, - IFF( - VALUE :: STRING = 'ORIGIN', - -1, - VALUE :: INT - ) AS flat_value - FROM - flatten_traces, - LATERAL FLATTEN ( - input => str_array - ) - ) - GROUP BY - block_number, - tx_position, - trace_address - ), - cleaned_traces AS ( - SELECT - b.block_number, - b.tx_position, - b.trace_address, - IFNULL( - sub_traces, - 0 - ) AS sub_traces, - num_array, - ROW_NUMBER() over ( - PARTITION BY b.block_number, - b.tx_position - ORDER BY - num_array ASC - ) - 1 AS trace_index, - trace_json, - b._inserted_timestamp - FROM - flatten_traces b - LEFT JOIN sub_traces s - ON b.block_number = s.block_number - AND b.tx_position = s.tx_position - AND b.trace_address = s.parent_trace_address - JOIN num_array n - ON b.block_number = n.block_number - AND b.tx_position = n.tx_position - AND b.trace_address = n.trace_address - ), - final_traces AS ( - SELECT - tx_position, - trace_index, - block_number, - trace_address, - trace_json :error :: STRING AS error_reason, - trace_json :from :: STRING AS from_address, - trace_json :to :: STRING AS to_address, - IFNULL( - utils.udf_hex_to_int( - trace_json :value :: STRING - ), - '0' - ) AS eth_value_precise_raw, - ethereum.utils.udf_decimal_adjust( - eth_value_precise_raw, - 18 - ) AS eth_value_precise, - eth_value_precise :: FLOAT AS eth_value, - utils.udf_hex_to_int( - trace_json :gas :: STRING - ) :: INT AS gas, - utils.udf_hex_to_int( - trace_json :gasUsed :: STRING - ) :: INT AS gas_used, - trace_json :input :: STRING AS input, - trace_json :output :: STRING AS output, - trace_json :type :: STRING AS TYPE, - concat_ws( - '_', - TYPE, - trace_address - ) AS identifier, - concat_ws( - '-', - block_number, - tx_position, - identifier - ) AS _call_id, - _inserted_timestamp, - trace_json AS DATA, - sub_traces - FROM - cleaned_traces - ), - new_records AS ( - SELECT - f.block_number, - t.tx_hash, - t.block_timestamp, - t.tx_status, - f.tx_position, - f.trace_index, - f.from_address, - f.to_address, - f.eth_value_precise_raw, - f.eth_value_precise, - f.eth_value, - f.gas, - f.gas_used, - f.input, - f.output, - f.type, - f.identifier, - f.sub_traces, - f.error_reason, - IFF( - f.error_reason IS NULL, - 'SUCCESS', - 'FAIL' - ) AS trace_status, - f.data, - IFF( - t.tx_hash IS NULL - OR t.block_timestamp IS NULL - OR t.tx_status IS NULL, - TRUE, - FALSE - ) AS is_pending, - f._call_id, - f._inserted_timestamp - FROM - final_traces f - LEFT OUTER JOIN ethereum.silver.transactions t - ON f.tx_position = t.position - AND f.block_number = t.block_number - ), - traces_final AS ( - SELECT - block_number, - tx_hash, - block_timestamp, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - eth_value_precise_raw, - eth_value_precise, - eth_value, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - trace_status, - DATA, - is_pending, - _call_id, - _inserted_timestamp - FROM - new_records - ), - eth_base AS ( - SELECT - tx_hash, - block_number, - block_timestamp, - identifier, - from_address, - to_address, - eth_value AS amount, - _call_id, - _inserted_timestamp, - eth_value_precise_raw AS amount_precise_raw, - eth_value_precise AS amount_precise, - tx_position, - trace_index - FROM - traces_final - WHERE - eth_value > 0 - AND tx_status = 'SUCCESS' - AND trace_status = 'SUCCESS' - AND TYPE NOT IN ( - 'DELEGATECALL', - 'STATICCALL' - ) - ), - tx_table AS ( - SELECT - block_number, - tx_hash, - from_address AS origin_from_address, - to_address1 AS origin_to_address, - origin_function_signature - FROM - txs - WHERE - tx_hash IN ( - SELECT - DISTINCT tx_hash - FROM - eth_base - ) - ), - native_transfers AS ( - SELECT - e.tx_hash, - e.block_number, - e.block_timestamp, - e.identifier, - t.origin_from_address, - t.origin_to_address, - t.origin_function_signature, - e.from_address, - e.to_address, - e.amount, - e.amount_precise_raw, - e.amount_precise, - ROUND( - e.amount * p.price, - 2 - ) AS amount_usd, - e._call_id, - e._inserted_timestamp, - e.tx_position, - e.trace_index, - md5( - cast( - coalesce(cast(e.tx_hash as TEXT), '_dbt_utils_surrogate_key_null_') - || '-' || coalesce(cast(e.trace_index as TEXT), '_dbt_utils_surrogate_key_null_') - as TEXT - ) - ) as native_transfers_id, - SYSDATE() as inserted_timestamp, - SYSDATE() as modified_timestamp - FROM - eth_base e - JOIN tx_table t ON e.tx_hash = t.tx_hash AND e.block_number = t.block_number - LEFT JOIN ETHEREUM.price.EZ_PRICES_HOURLY p - ON DATE_TRUNC('hour', e.block_timestamp) = p.HOUR - AND p.token_address = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' -- WETH address - ) +WITH spine AS ( + {{ evm_live_view_target_blocks(schema, blockchain, network) | indent(4) -}} +), +raw_receipts AS ( + {{ evm_live_view_bronze_receipts(schema, 'spine') | indent(4) -}} +), +raw_block_txs AS ( + {{ evm_live_view_bronze_blocks(schema, 'spine') | indent(4) -}} +), +raw_transactions AS ( + {{ evm_live_view_bronze_transactions('raw_block_txs') | indent(4) -}} +), +blocks AS ( + {{ evm_live_view_silver_blocks('raw_block_txs') | indent(4) -}} +), +receipts AS ( + {{ evm_live_view_silver_receipts('raw_receipts') | indent(4) -}} +), +transactions AS ( + {{ evm_live_view_silver_transactions('raw_transactions', 'blocks', 'receipts') | indent(4) -}} +), +raw_traces AS ( + {{ evm_live_view_bronze_traces(schema, 'spine') | indent(4) -}} +), +{{ evm_live_view_silver_traces('raw_traces') | indent(4) -}} +, +eth_base AS ( SELECT tx_hash, block_number, block_timestamp, + identifier, + from_address, + to_address, + eth_value AS amount, + _call_id, + _inserted_timestamp, + eth_value_precise_raw AS amount_precise_raw, + eth_value_precise AS amount_precise, tx_position, - trace_index, - identifier AS trace_type, - origin_from_address, - origin_to_address, - origin_function_signature, - from_address AS trace_from_address, - to_address AS trace_to_address, - amount, - amount_precise_raw, - amount_precise, - amount_usd, - COALESCE( - native_transfers_id, - md5( - cast( - coalesce(cast(tx_hash as TEXT), '_dbt_utils_surrogate_key_null_') - || '-' || coalesce(cast(trace_index as TEXT), '_dbt_utils_surrogate_key_null_') - as TEXT - ) - ) - ) AS ez_native_transfers_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp + trace_index FROM - native_transfers - QUALIFY (ROW_NUMBER() OVER ( - PARTITION BY block_number, tx_position, trace_index - ORDER BY _inserted_timestamp DESC - )) = 1 + traces_final + WHERE + eth_value > 0 + AND tx_status = 'SUCCESS' + AND trace_status = 'SUCCESS' + AND TYPE NOT IN ( + 'DELEGATECALL', + 'STATICCALL' + ) +), +tx_table AS ( + SELECT + block_number, + tx_hash, + from_address AS origin_from_address, + to_address1 AS origin_to_address, + origin_function_signature + FROM + transactions + WHERE + tx_hash IN ( + SELECT + DISTINCT tx_hash + FROM + eth_base + ) +), +native_transfers AS ( + SELECT + e.tx_hash, + e.block_number, + e.block_timestamp, + e.identifier, + t.origin_from_address, + t.origin_to_address, + t.origin_function_signature, + e.from_address, + e.to_address, + e.amount, + e.amount_precise_raw, + e.amount_precise, + ROUND( + e.amount * p.price, + 2 + ) AS amount_usd, + e._call_id, + e._inserted_timestamp, + e.tx_position, + e.trace_index, + md5( + cast( + coalesce(cast(e.tx_hash as TEXT), '_dbt_utils_surrogate_key_null_') + || '-' || coalesce(cast(e.trace_index as TEXT), '_dbt_utils_surrogate_key_null_') + as TEXT + ) + ) as native_transfers_id, + SYSDATE() as inserted_timestamp, + SYSDATE() as modified_timestamp + FROM + eth_base e + JOIN tx_table t ON e.tx_hash = t.tx_hash AND e.block_number = t.block_number + LEFT JOIN {{ blockchain }}.PRICE.EZ_PRICES_HOURLY p + ON DATE_TRUNC('hour', e.block_timestamp) = p.HOUR + AND p.token_address = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' -- WETH address +) +SELECT + tx_hash, + block_number, + block_timestamp, + tx_position, + trace_index, + identifier AS trace_type, + origin_from_address, + origin_to_address, + origin_function_signature, + from_address AS trace_from_address, + to_address AS trace_to_address, + amount, + amount_precise_raw, + amount_precise, + amount_usd, + COALESCE( + native_transfers_id, + md5( + cast( + coalesce(cast(tx_hash as TEXT), '_dbt_utils_surrogate_key_null_') + || '-' || coalesce(cast(trace_index as TEXT), '_dbt_utils_surrogate_key_null_') + as TEXT + ) + ) + ) AS ez_native_transfers_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp +FROM + native_transfers +QUALIFY (ROW_NUMBER() OVER ( + PARTITION BY block_number, tx_position, trace_index + ORDER BY _inserted_timestamp DESC +)) = 1 {% endmacro %} From 75520af4f62bec773acc9b18433c608de8f8e85a Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Fri, 18 Oct 2024 11:35:40 +0900 Subject: [PATCH 32/50] rename fact logs to fact event logs --- macros/evm/evm.yaml.sql | 4 ++-- macros/evm/evm_live_views.sql | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index ca5c4e7..d42721c 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -377,7 +377,7 @@ sql: | {{ evm_live_view_fact_blocks(schema, blockchain, network) | indent(4) -}} -- name: {{ schema -}}.tf_fact_logs +- name: {{ schema -}}.tf_fact_event_logs signature: - [block_height, INTEGER, The start block height to get the logs from] - [to_latest, BOOLEAN, Whether to continue fetching logs until the latest block or not] @@ -389,7 +389,7 @@ VOLATILE COMMENT = $$Returns the logs for a given block height. If to_latest is true, it will continue fetching logs until the latest block. Otherwise, it will fetch logs until the block height is reached.$$ sql: | - {{ evm_live_view_fact_logs(schema, blockchain, network) | indent(4) -}} + {{ evm_live_view_fact_event_logs(schema, blockchain, network) | indent(4) -}} - name: {{ schema -}}.tf_fact_decoded_event_logs signature: diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 9dd77b7..1dd8055 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -589,7 +589,7 @@ WITH spine AS ( from silver_blocks {% endmacro %} -{% macro evm_live_view_fact_logs(schema, blockchain, network) %} +{% macro evm_live_view_fact_event_logs(schema, blockchain, network) %} WITH spine AS ( {{ evm_live_view_target_blocks(schema, blockchain, network) | indent(4) -}} ), @@ -653,7 +653,7 @@ FROM logs {% macro evm_live_view_fact_decoded_event_logs(schema, blockchain, network) %} WITH _fact_event_logs AS ( - {{ evm_live_view_fact_logs(schema, blockchain, network) | indent(4) -}} + {{ evm_live_view_fact_event_logs(schema, blockchain, network) | indent(4) -}} ), _silver_decoded_logs AS ( @@ -883,7 +883,7 @@ FROM traces_final -- Get EVM chain ez data {% macro evm_live_view_ez_token_transfers(schema, blockchain, network) %} WITH fact_logs AS ( - {{ evm_live_view_fact_logs(schema, blockchain, network) | indent(4) -}} + {{ evm_live_view_fact_event_logs(schema, blockchain, network) | indent(4) -}} ) SELECT From b37d7ba8e2a616eca859225ee93c859b9364e961 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Fri, 18 Oct 2024 15:38:32 +0900 Subject: [PATCH 33/50] [STREAM-1065] Add ez decoded event logs --- macros/evm/evm.yaml.sql | 16 +++ macros/evm/evm_live_views.sql | 196 ++++++++++++++++++++-------------- 2 files changed, 129 insertions(+), 83 deletions(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index d42721c..2786561 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -404,6 +404,7 @@ COMMENT = $$Returns the decoded event logs data for a given block height. If to_latest is true, it will continue fetching blocks until the latest block. Otherwise, it will fetch blocks until the block height is reached.$$ sql: | {{ evm_live_view_fact_decoded_event_logs(schema, blockchain, network) | indent(4) -}} + - name: {{ schema -}}.tf_fact_traces signature: - [block_height, INTEGER, The start block height to get the transfers from] @@ -432,6 +433,21 @@ sql: | {{ evm_live_view_fact_transactions(schema, blockchain, network) | indent(4) -}} +- name: {{ schema -}}.tf_ez_decoded_event_logs + signature: + - [block_height, INTEGER, The start block height to get the logs from] + - [to_latest, BOOLEAN, Whether to continue fetching logs until the latest block or not] + return_type: + - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, event_index INTEGER, contract_address STRING, contract_name STRING, event_name STRING, decoded_log OBJECT, full_decoded_log VARIANT, origin_function_signature STRING, origin_from_address STRING, origin_to_address STRING, topics VARIANT, data STRING, event_removed BOOLEAN, tx_status STRING, ez_decoded_event_logs_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + COMMENT = $$Returns the ez decoded event logs data for a given block height. If to_latest is true, it will continue fetching blocks until the latest block. Otherwise, it will fetch blocks until the block height is reached.$$ + sql: | + {{ evm_live_view_ez_decoded_event_logs(schema, blockchain, network) | indent(4) -}} + + - name: {{ schema -}}.tf_ez_native_transfers signature: - [block_height, INTEGER, The start block height to get the transfers from] diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 1dd8055..339f81e 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -652,93 +652,23 @@ FROM logs {% endmacro %} {% macro evm_live_view_fact_decoded_event_logs(schema, blockchain, network) %} -WITH _fact_event_logs AS ( - {{ evm_live_view_fact_event_logs(schema, blockchain, network) | indent(4) -}} -), - -_silver_decoded_logs AS ( - SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - event_index, - topics, - DATA, - contract_address, - OBJECT_CONSTRUCT('topics', topics, 'data', data, 'address', contract_address) AS event_data, - abi, - utils.udf_evm_decode_log(abi, event_data)[0] AS decoded_data, - event_removed, - decoded_data:name::string AS event_name, - {{ blockchain }}.utils.udf_transform_logs(decoded_data) AS transformed, - _log_id, - inserted_timestamp, - tx_status - FROM - _fact_event_logs - JOIN - {{ blockchain }}.core.dim_contract_abis - USING - (contract_address) - WHERE - tx_status = 'SUCCESS' -), - -_flatten_logs AS ( - SELECT - b.tx_hash, - b.block_number, - b.event_index, - b.event_name, - b.contract_address, - b.decoded_data, - b.transformed, - b._log_id, - b.inserted_timestamp, - OBJECT_AGG( - DISTINCT CASE - WHEN v.value :name = '' THEN CONCAT( - 'anonymous_', - v.index - ) - ELSE v.value :name - END, - v.value :value - ) AS decoded_flat - FROM - _silver_decoded_logs b, - LATERAL FLATTEN( - input => b.transformed :data - ) v - GROUP BY - b.tx_hash, - b.block_number, - b.event_index, - b.event_name, - b.contract_address, - b.decoded_data, - b.transformed, - b._log_id, - b.inserted_timestamp +WITH _ez_decoded_event_logs AS ( + {{ evm_live_view_ez_decoded_event_logs(schema, blockchain, network) | indent(4) -}} ) SELECT block_number, - C.block_timestamp, - B.tx_hash, - B.event_index, - B.contract_address, - B.event_name, - B.decoded_flat AS decoded_log, - B.decoded_data AS full_decoded_log, - md5(_log_id) AS fact_decoded_event_logs_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp -FROM _flatten_logs AS B -LEFT JOIN _silver_decoded_logs AS C USING (block_number, _log_id) + block_timestamp, + tx_hash, + event_index, + contract_address, + event_name, + decoded_log, + full_decoded_log, + fact_decoded_event_logs_id, + inserted_timestamp, + modified_timestamp +FROM _ez_decoded_event_logs {% endmacro %} {% macro evm_live_view_fact_transactions(schema, blockchain, network) %} @@ -881,6 +811,106 @@ FROM traces_final {% endmacro %} -- Get EVM chain ez data +{% macro evm_live_view_ez_decoded_event_logs(schema, blockchain, network) %} +WITH _fact_event_logs AS ( + {{ evm_live_view_fact_event_logs(schema, blockchain, network) | indent(4) -}} +), + +_silver_decoded_logs AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + event_index, + topics, + DATA, + contract_address, + OBJECT_CONSTRUCT('topics', topics, 'data', data, 'address', contract_address) AS event_data, + abi, + utils.udf_evm_decode_log(abi, event_data)[0] AS decoded_data, + event_removed, + decoded_data:name::string AS event_name, + {{ blockchain }}.utils.udf_transform_logs(decoded_data) AS transformed, + _log_id, + inserted_timestamp, + tx_status + FROM + _fact_event_logs + JOIN + {{ blockchain }}.core.dim_contract_abis + USING + (contract_address) + WHERE + tx_status = 'SUCCESS' +), + +_flatten_logs AS ( + SELECT + b.tx_hash, + b.block_number, + b.event_index, + b.event_name, + b.contract_address, + b.decoded_data, + b.transformed, + b._log_id, + b.inserted_timestamp, + OBJECT_AGG( + DISTINCT CASE + WHEN v.value :name = '' THEN CONCAT( + 'anonymous_', + v.index + ) + ELSE v.value :name + END, + v.value :value + ) AS decoded_flat + FROM + _silver_decoded_logs b, + LATERAL FLATTEN( + input => b.transformed :data + ) v + GROUP BY + b.tx_hash, + b.block_number, + b.event_index, + b.event_name, + b.contract_address, + b.decoded_data, + b.transformed, + b._log_id, + b.inserted_timestamp +) + +SELECT + block_number, + C.block_timestamp, + B.tx_hash, + B.event_index, + B.contract_address, + D.name AS contract_name, + B.event_name, + B.decoded_flat AS decoded_log, + B.decoded_data AS full_decoded_log, + C.origin_function_signature, + C.origin_from_address, + C.origin_to_address, + C.topics, + C.DATA, + C.event_removed, + C.tx_status, + md5(_log_id) AS fact_decoded_event_logs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM _flatten_logs AS B +LEFT JOIN _silver_decoded_logs AS C USING (block_number, _log_id) +LEFT JOIN {{ blockchain }}.core.dim_contracts AS D + ON B.contract_address = D.address +{% endmacro %} + {% macro evm_live_view_ez_token_transfers(schema, blockchain, network) %} WITH fact_logs AS ( {{ evm_live_view_fact_event_logs(schema, blockchain, network) | indent(4) -}} From 1dd35412c479b1c3f942a34bdf0ca8b1d1bceb2a Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Fri, 18 Oct 2024 16:01:51 +0900 Subject: [PATCH 34/50] remove wave apps folder --- apps/README.md | 45 -------- apps/__init__.py | 0 apps/app.py | 111 -------------------- apps/dashboards/__init__.py | 0 apps/dashboards/realtime_monitoring.py | 139 ------------------------- apps/env.sample | 7 -- apps/table_list.yaml | 35 ------- 7 files changed, 337 deletions(-) delete mode 100644 apps/README.md delete mode 100644 apps/__init__.py delete mode 100644 apps/app.py delete mode 100644 apps/dashboards/__init__.py delete mode 100644 apps/dashboards/realtime_monitoring.py delete mode 100644 apps/env.sample delete mode 100644 apps/table_list.yaml diff --git a/apps/README.md b/apps/README.md deleted file mode 100644 index e7b1599..0000000 --- a/apps/README.md +++ /dev/null @@ -1,45 +0,0 @@ -## Development - -1. cd your directory - - ```bash - cd apps - ``` - -2. Setup a virtual environment - - ```bash - conda create -n "livequery-models" python=3.9 - ``` - -3. Activate the virtual environment - - ```bash - conda activate livequery-models - ``` - -4. Create a `.env` file from the env.sample file and run, - - ```base - set -a; source .env; set +a - ``` - -5. Install - - ```bash - pip install -r ../requirements.txt - ``` - -6. Add PYTHONPATH - - ```bash - export PYTHONPATH=. - ``` - -7. Run the app - - ```bash - wave run app.py - ``` - -7. Visit the demo at [http://localhost:10101/demo](http://127.0.0.1:10101/demo) diff --git a/apps/__init__.py b/apps/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/apps/app.py b/apps/app.py deleted file mode 100644 index e717c3f..0000000 --- a/apps/app.py +++ /dev/null @@ -1,111 +0,0 @@ -import time -from h2o_wave import app, data, ui, Q, main - -from dashboards.realtime_monitoring import execute_query -import logging - - -logging.basicConfig(level=logging.INFO) - - -light_theme_colors = "$red $pink $purple $violet $indigo $blue $azure $cyan $teal $mint $green $amber $orange $tangerine".split() # noqa: E501 -dark_theme_colors = "$red $pink $blue $azure $cyan $teal $mint $green $lime $yellow $amber $orange $tangerine".split() - -_color_index = -1 -colors = dark_theme_colors - - -def next_color(): - global _color_index - _color_index += 1 - return colors[_color_index % len(colors)] - - -_curve_index = -1 -curves = "linear smooth step step-after step-before".split() - - -def next_curve(): - global _curve_index - _curve_index += 1 - return curves[_curve_index % len(curves)] - -async def update_realtime_dataset(q): - async for data_table in execute_query(): - try: - latest_block_number = max( - row["BLOCK_NUMBER"] for row in data_table.to_pylist() - ) - q.page["block_number_card"].value = str(latest_block_number) - except Exception as e: - print(e) - - # try: - # import pyarrow as pa - # import pyarrow.compute as pc - - # # Extract block_timestamp column - # block_timestamps = data_table.column("BLOCK_TIMESTAMP") - - # # Convert timestamps to minutes - # block_timestamps_minute = pc.strftime(block_timestamps, format="%Y-%m-%d %H:%M") - - # # Group by minute and count - # table = pa.table([block_timestamps_minute], names=["minute"]) - # grouped_data = table.group_by("minute").aggregate([("minute", "count")]) - - # # Convert grouped data to list and assign it back - # existing_data = list(q.page["line_plot"].data) # Convert Ref to list - # new_data = existing_data + [[row["minute"], row["minute_count"]] for row in grouped_data.to_pylist()] # Append new entries - # q.page["line_plot"].data = new_data # Assign back to Ref - # except Exception as e: - # print(e) - - # try: - # rows = [ - # ui.table_row( - # name=str(i), - # cells=[current_time, str(len(row))] - # ) - # for i, row in enumerate(data_table.to_pylist()) - # ] - # q.page["data_table"].rows = rows - # except Exception as e: - # print(e) - - await q.page.save() - -@app("/demo", mode='broadcast') -async def create_dashboard(q: Q): - q.page.drop() - line_plot = ui.plot_card( - box="1 3 4 4", - title="Token Transfers Data Size Over Time", - data=data("time length", -100), - plot=ui.plot( - [ui.mark(type="line", x="=time", y="=length", curve="smooth")] - ), - ) - - block_number_card = ui.small_stat_card(box="1 1 2 2", title="Latest Block Number", value="0") - - # data_table = ui.table( - # name='data_table', - # columns=[ - # ui.table_column(name='time', label='Time', sortable=True), - # ui.table_column(name='length', label='Data Length', sortable=True), - # ], - # rows=[], - # pagination=ui.table_pagination(total_rows=100, rows_per_page=100) - # ) - - # q.page["line_plot"] = line_plot - q.page["block_number_card"] = block_number_card - # q.page["data_table"] = data_table - - await q.page.save() - - try: - await update_realtime_dataset(q) - finally: - await q.close() diff --git a/apps/dashboards/__init__.py b/apps/dashboards/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/apps/dashboards/realtime_monitoring.py b/apps/dashboards/realtime_monitoring.py deleted file mode 100644 index 2aba2a5..0000000 --- a/apps/dashboards/realtime_monitoring.py +++ /dev/null @@ -1,139 +0,0 @@ -from h2o_wave import ui, data, Q -import pyarrow as pa -from pyarrow import compute as pc -import asyncio -import time -from concurrent.futures import ThreadPoolExecutor -import snowflake.connector -from snowflake.connector import ProgrammingError -import os -import yaml - -async def pull_data_from_snowflake(query: str): - with snowflake.connector.connect( - user=os.getenv("SNOWFLAKE_USER"), - password=os.getenv("SNOWFLAKE_PASSWORD"), - account=os.getenv("SNOWFLAKE_ACCOUNT"), - database=os.getenv("SNOWFLAKE_DATABASE"), - schema=os.getenv("SNOWFLAKE_SCHEMA"), - warehouse=os.getenv("SNOWFLAKE_WAREHOUSE"), - role=os.getenv("SNOWFLAKE_ROLE"), - session_parameters={ - "QUERY_TAG": "realtime_monitoring", - }, - ) as conn: - cur = conn.cursor() - try: - cur.execute_async(query) - query_id = cur.sfqid - - try: - while conn.is_still_running(conn.get_query_status_throw_if_error(query_id)): - time.sleep(1) - except ProgrammingError as err: - print('Programming Error: {0}'.format(err)) - - results = [] - cur.get_results_from_sfqid(query_id) - - #TODO: work with PyArrow and yield in batches to avoid OOM - for row in cur.fetch_arrow_batches(): - results.extend(row.to_pylist()) - - return results - finally: - cur.close() - - - -QUERY = """ - SELECT * - FROM TABLE({SCHEMA}.{TABLE_FUNCTION_NAME}( - {table_function_args} - )); -""" - -def load_table_list_from_yaml(yaml_file): - with open(yaml_file, 'r') as file: - data = yaml.safe_load(file) - return data - -# table_lists = load_table_list_from_yaml('apps/table_list.yaml') - -async def fetch_data_for_schema_table(schema, table, args): - latest_block_height = 0 - query = QUERY.format( - DATABASE=os.getenv("SNOWFLAKE_DATABASE"), - SCHEMA=schema, - TABLE_FUNCTION_NAME=table, - table_function_args=args, - ) - results = await pull_data_from_snowflake(query) - if results: - results_table = pa.Table.from_pylist(results) - latest_block_height = pc.max(results_table.column('BLOCK_NUMBER')).as_py() - - return (latest_block_height, results_table) - -# async def gather_tasks_and_parse_args(q: Q): -# """ -# Gather tasks and parse args for each table and network. -# """ -# table_lists = load_table_list_from_yaml('apps/table_list.yaml') -# tables = table_lists['tables'] - -# loop = asyncio.get_event_loop() -# with ThreadPoolExecutor() as executor: -# tasks = [] -# for table in tables: -# blockchain = table['blockchain'] -# for network in table['networks']: -# for network_name, network_data in network.items(): -# schema = f"{blockchain}_{network_name}" -# for table_data in network_data['tables']: -# table_name = table_data['name'] -# args = {**network_data['default_args'], **table_data.get('extra_args', {})} -# formatted_args = ', '.join(f"{key}={arg['value']}::{arg['type']}" for key, arg in args.items()) -# tasks.append( -# loop.run_in_executor( -# executor, fetch_data_for_schema_table, q, schema, table_name, formatted_args -# ) -# ) -# await asyncio.gather(*tasks) - -async def execute_query() -> pa.Table: - table_lists = { - "schema": "ethereum_mainnet", - "table": "tf_ez_token_transfers", - "args": [ - { - "block_height": "NULL::INTEGER", - "to_latest": "TRUE::BOOLEAN", - "ez_token_transfers_id": "'0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef'::STRING" - } - ] - } - - initial_args = ', '.join(f"{value}" for arg in table_lists['args'] for _, value in arg.items()) - latest_block_height, data_table = await fetch_data_for_schema_table(table_lists['schema'], table_lists['table'], initial_args) - - # Forever loop to fetch new data - while True: - table_lists['args'][0]['block_height'] = f"{latest_block_height}::INTEGER" - updated_args = ', '.join(f"{value}" for arg in table_lists['args'] for _, value in arg.items()) - latest_block_height, data_table = await fetch_data_for_schema_table(table_lists['schema'], table_lists['table'], updated_args) - - yield data_table - -if __name__ == "__main__": - # data = asyncio.run(execute_query()) - import logging - - logging.basicConfig(level=logging.INFO) - - async def log_data(): - async for data_table in execute_query(): - data_array = data_table.to_pylist() - logging.info(data_array) - - asyncio.run(log_data()) diff --git a/apps/env.sample b/apps/env.sample deleted file mode 100644 index c7ba6fd..0000000 --- a/apps/env.sample +++ /dev/null @@ -1,7 +0,0 @@ -SNOWFLAKE_ACCOUNT='vna27887.us-east-1' -SNOWFLAKE_ROLE='ACCOUNTADMIN' -SNOWFLAKE_USER='<>' -SNOWFLAKE_PASSWORD='<>' -SNOWFLAKE_REGION='us-east-1' -SNOWFLAKE_DATABASE='LIVEQUERY_DEV' -SNOWFLAKE_WAREHOUSE='INTERNAL_DEV' diff --git a/apps/table_list.yaml b/apps/table_list.yaml deleted file mode 100644 index 4adb06b..0000000 --- a/apps/table_list.yaml +++ /dev/null @@ -1,35 +0,0 @@ -tables: - - blockchain: ethereum - networks: - - mainnet: - tables: - - name: tf_fact_blocks - - name: tf_fact_logs - - name: tf_ez_token_transfers - extra_args: - ez_token_transfers_id: - value: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' - type: STRING - default_args: - block_height: - value: NULL - type: INTEGER - to_latest: - value: TRUE - type: BOOLEAN - - sepolia: - tables: - - name: tf_fact_blocks - - name: tf_fact_logs - - name: tf_ez_token_transfers - extra_args: - ez_token_transfers_id: - value: '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' - type: STRING - default_args: - block_height: - value: NULL - type: INTEGER - to_latest: - value: TRUE - type: BOOLEAN From 57ee368788bdfa115b7714a8f8a1f44d9d0528fc Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Thu, 24 Oct 2024 17:00:55 +0900 Subject: [PATCH 35/50] [STREAM-1068] Add fact token balances --- macros/evm/evm.yaml.sql | 14 ++++ macros/evm/evm_live_views.sql | 135 ++++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index 2786561..a523d4a 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -433,6 +433,20 @@ sql: | {{ evm_live_view_fact_transactions(schema, blockchain, network) | indent(4) -}} +- name: {{ schema -}}.tf_fact_token_balances + signature: + - [block_height, INTEGER, The start block height to get the transfers from] + - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] + return_type: + - "TABLE(block_number NUMBER, block_timestamp TIMESTAMP_NTZ, address STRING, contract_address STRING, balance INTEGER, fact_token_balances_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + COMMENT = $$Returns the token balances for a given block height. If to_latest is true, it will continue fetching transactions until the latest block. Otherwise, it will fetch transactions until the block height is reached.$$ + sql: | + {{ evm_live_view_fact_token_balances(schema, blockchain, network) | indent(4) -}} + - name: {{ schema -}}.tf_ez_decoded_event_logs signature: - [block_height, INTEGER, The start block height to get the logs from] diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 339f81e..78dca77 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -91,6 +91,33 @@ LATERAL FLATTEN(input => PARSE_JSON( )) v {% endmacro %} +{% macro evm_live_view_bronze_token_balances(schema, table_name) %} +SELECT + block_number, + block_timestamp, + address, + contract_address, + {{ schema }}.udf_rpc( + 'eth_call', + ARRAY_CONSTRUCT( + OBJECT_CONSTRUCT( + 'to', + contract_address, + 'data', + CONCAT( + '0x70a08231000000000000000000000000', + SUBSTR( + address, + 3 + ) + ) + ), + utils.udf_int_to_hex(block_number) + ) + ) AS DATA +FROM {{ table_name }} +{% endmacro %} + -- Transformation macro for EVM chains {% macro evm_live_view_silver_blocks(table_name) %} SELECT @@ -505,6 +532,96 @@ traces_final AS ( ) {% endmacro %} +{% macro evm_live_view_silver_token_balances(schema, blockchain, network) %} +WITH silver_logs AS ( + SELECT + CONCAT('0x', SUBSTR(l.topics [1] :: STRING, 27, 42)) AS address1, + CONCAT('0x', SUBSTR(l.topics [2] :: STRING, 27, 42)) AS address2, + l.contract_address, + l.block_timestamp, + l.block_number + FROM + ( + {{ evm_live_view_fact_event_logs(schema, blockchain, network) | indent(4) -}} + ) l + WHERE + ( + l.topics [0] :: STRING = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' + OR ( + l.topics [0] :: STRING = '0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65' + AND l.contract_address = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' + ) + OR ( + l.topics [0] :: STRING = '0xe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c' + AND l.contract_address = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' + ) + ) +), + +transfers AS ( + SELECT + DISTINCT block_number, + block_timestamp, + contract_address, + address1 AS address + FROM + silver_logs + WHERE + address1 IS NOT NULL + AND address1 <> '0x0000000000000000000000000000000000000000' + UNION + SELECT + DISTINCT block_number, + block_timestamp, + contract_address, + address2 AS address + FROM + silver_logs + WHERE + address2 IS NOT NULL + AND address2 <> '0x0000000000000000000000000000000000000000' +), + +balances AS ( + {{ evm_live_view_bronze_token_balances(schema, 'transfers') | indent(4) -}} +) + +SELECT + block_number, + block_timestamp, + address, + contract_address, + TRY_TO_NUMBER( + CASE + WHEN LENGTH( + DATA :result :: STRING + ) <= 4300 + AND DATA :result IS NOT NULL THEN utils.udf_hex_to_int(LEFT(DATA :result :: STRING, 66)) + ELSE NULL + END + ) AS balance, + SYSDATE() AS _inserted_timestamp, + cast( + coalesce( + cast(block_number as TEXT), + '_dbt_utils_surrogate_key_null_' + ) || '-' || + coalesce( + cast(address as TEXT), + '_dbt_utils_surrogate_key_null_' + ) || '-' || + coalesce( + cast(contract_address as TEXT), + '_dbt_utils_surrogate_key_null_' + ) as TEXT + ) AS id, + id AS token_balances_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + balances +{% endmacro %} + -- Get EVM chain fact data {% macro evm_live_view_fact_blocks(schema, blockchain, network) %} WITH spine AS ( @@ -810,6 +927,24 @@ SELECT FROM traces_final {% endmacro %} +{% macro evm_live_view_fact_token_balances(schema, blockchain, network) %} +WITH silver_token_balances AS ( + {{ evm_live_view_silver_token_balances(schema, blockchain, network) | indent(4) -}} +) + +SELECT + block_number, + block_timestamp, + address AS user_address, + contract_address, + balance, + token_balances_id AS fact_token_balances_id, + inserted_timestamp, + modified_timestamp +FROM + silver_token_balances +{% endmacro %} + -- Get EVM chain ez data {% macro evm_live_view_ez_decoded_event_logs(schema, blockchain, network) %} WITH _fact_event_logs AS ( From 468407778aa1a36b9e562ddcaddadb2eb9c8843e Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Fri, 25 Oct 2024 20:27:11 +0900 Subject: [PATCH 36/50] [STREAM-1068] update logic on balance --- macros/evm/evm.yaml.sql | 2 +- macros/evm/evm_live_views.sql | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index a523d4a..91e04cb 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -438,7 +438,7 @@ - [block_height, INTEGER, The start block height to get the transfers from] - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] return_type: - - "TABLE(block_number NUMBER, block_timestamp TIMESTAMP_NTZ, address STRING, contract_address STRING, balance INTEGER, fact_token_balances_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + - "TABLE(block_number NUMBER, block_timestamp TIMESTAMP_NTZ, address STRING, contract_address STRING, balance NUMBER(38,0), fact_token_balances_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" options: | NOT NULL RETURNS NULL ON NULL INPUT diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 78dca77..cf9353d 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -591,12 +591,22 @@ SELECT block_timestamp, address, contract_address, + IFF(DATA :: STRING = '{}', NULL, DATA :: STRING) AS casted_data, + CASE + WHEN + LENGTH( + casted_data + ) <= 4300 + AND casted_data IS NOT NULL THEN LEFT(casted_data, 66) + ELSE NULL + END + AS hex_balance, TRY_TO_NUMBER( CASE WHEN LENGTH( - DATA :result :: STRING + hex_balance ) <= 4300 - AND DATA :result IS NOT NULL THEN utils.udf_hex_to_int(LEFT(DATA :result :: STRING, 66)) + AND hex_balance IS NOT NULL THEN utils.udf_hex_to_int(hex_balance) ELSE NULL END ) AS balance, @@ -618,8 +628,7 @@ SELECT id AS token_balances_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp -FROM - balances +FROM balances {% endmacro %} -- Get EVM chain fact data From 95aa3a7b1138c35f1d737fc65d59e2f9cbb6e47c Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Mon, 28 Oct 2024 00:06:42 +0900 Subject: [PATCH 37/50] [STREAM-1069] Add fact eth balance livequery --- macros/evm/evm.yaml.sql | 16 +++++- macros/evm/evm_live_views.sql | 104 ++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 1 deletion(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index 91e04cb..2e0caf1 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -447,6 +447,21 @@ sql: | {{ evm_live_view_fact_token_balances(schema, blockchain, network) | indent(4) -}} +- name: {{ schema -}}.tf_fact_eth_balances + signature: + - [block_height, INTEGER, The start block height to get the transfers from] + - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] + return_type: + - "TABLE(block_number NUMBER, block_timestamp TIMESTAMP_NTZ, address STRING, balance NUMBER(38,0), fact_eth_balances_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + COMMENT = $$Returns the eth balances for a given block height. If to_latest is true, it will continue fetching transactions until the latest block. Otherwise, it will fetch transactions until the block height is reached.$$ + sql: | + {{ evm_live_view_fact_eth_balances(schema, blockchain, network) | indent(4) -}} + + - name: {{ schema -}}.tf_ez_decoded_event_logs signature: - [block_height, INTEGER, The start block height to get the logs from] @@ -461,7 +476,6 @@ sql: | {{ evm_live_view_ez_decoded_event_logs(schema, blockchain, network) | indent(4) -}} - - name: {{ schema -}}.tf_ez_native_transfers signature: - [block_height, INTEGER, The start block height to get the transfers from] diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index cf9353d..df0e3cd 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -118,6 +118,18 @@ SELECT FROM {{ table_name }} {% endmacro %} +{% macro evm_live_view_bronze_eth_balances(schema, table_name) %} +SELECT + block_number, + block_timestamp, + address, + {{ schema }}.udf_rpc( + 'eth_getBalance', + ARRAY_CONSTRUCT(address, utils.udf_int_to_hex(block_number)) + ) AS DATA +FROM {{ table_name }} +{% endmacro %} + -- Transformation macro for EVM chains {% macro evm_live_view_silver_blocks(table_name) %} SELECT @@ -631,6 +643,82 @@ SELECT FROM balances {% endmacro %} +{% macro evm_live_view_silver_eth_balances(schema, blockchain, network) %} +WITH silver_traces AS ( + SELECT + block_timestamp, + block_number, + from_address, + to_address + FROM + ( + {{ evm_live_view_fact_traces(schema, blockchain, network) | indent(4) -}} + ) l + WHERE + VALUE > 0 -- VALUE is the amount of ETH transferred + AND trace_status = 'SUCCESS' + AND tx_status = 'SUCCESS' +), + +stacked AS ( + SELECT + DISTINCT block_number, + block_timestamp, + from_address AS address + FROM + silver_traces + WHERE + from_address IS NOT NULL + AND from_address <> '0x0000000000000000000000000000000000000000' + UNION + SELECT + DISTINCT block_number, + block_timestamp, + to_address AS address + FROM + silver_traces + WHERE + to_address IS NOT NULL + AND to_address <> '0x0000000000000000000000000000000000000000' +), + +eth_balances AS ( + {{ evm_live_view_bronze_eth_balances(schema, 'stacked') | indent(4) -}} +) + +SELECT + block_number, + block_timestamp, + address, + IFF(DATA :: STRING = '{}', NULL, DATA :: STRING) AS casted_data, + CASE + WHEN casted_data IS NOT NULL THEN casted_data + ELSE NULL + END + AS hex_balance, + TRY_TO_NUMBER( + CASE + WHEN hex_balance IS NOT NULL THEN utils.udf_hex_to_int(hex_balance) + ELSE NULL + END + ) AS balance, + SYSDATE() AS _inserted_timestamp, + cast( + coalesce( + cast(block_number as TEXT), + '_dbt_utils_surrogate_key_null_' + ) || '-' || + coalesce( + cast(address as TEXT), + '_dbt_utils_surrogate_key_null_' + ) as TEXT + ) AS id, + id AS eth_balances_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM eth_balances +{% endmacro %} + -- Get EVM chain fact data {% macro evm_live_view_fact_blocks(schema, blockchain, network) %} WITH spine AS ( @@ -954,6 +1042,22 @@ FROM silver_token_balances {% endmacro %} +{% macro evm_live_view_fact_eth_balances(schema, blockchain, network) %} +WITH silver_eth_balances AS ( + {{ evm_live_view_silver_eth_balances(schema, blockchain, network) | indent(4) -}} +) + +SELECT + block_number, + block_timestamp, + address AS user_address, + balance, + eth_balances_id AS fact_eth_balances_id, + inserted_timestamp, + modified_timestamp +FROM silver_eth_balances +{% endmacro %} + -- Get EVM chain ez data {% macro evm_live_view_ez_decoded_event_logs(schema, blockchain, network) %} WITH _fact_event_logs AS ( From cc79aca6d7a758339235ba9af2839151bab0316f Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Mon, 4 Nov 2024 14:13:13 +0900 Subject: [PATCH 38/50] [STREAM-1070] add fact decode traces livequery --- macros/core/utils.yaml.sql | 14 +++- macros/evm/evm.yaml.sql | 17 +++++ macros/evm/evm_live_views.sql | 118 ++++++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 1 deletion(-) diff --git a/macros/core/utils.yaml.sql b/macros/core/utils.yaml.sql index a2a9774..52f3482 100644 --- a/macros/core/utils.yaml.sql +++ b/macros/core/utils.yaml.sql @@ -238,6 +238,18 @@ RETURNS NULL ON NULL INPUT sql: evm/decode/log +- name: {{ schema }}.udf_evm_decode_trace + signature: + - [abi, OBJECT] + - [data, OBJECT] + return_type: ARRAY + func_type: EXTERNAL + api_integration: '{{ var("API_INTEGRATION") }}' + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + sql: evm/decode/trace + - name: {{ schema }}.udf_base58_to_hex signature: - [base58, STRING] @@ -294,4 +306,4 @@ sql: | {{ create_udf_binary_to_int() | indent(4) }} -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index 2e0caf1..900e85c 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -405,6 +405,23 @@ sql: | {{ evm_live_view_fact_decoded_event_logs(schema, blockchain, network) | indent(4) -}} +- name: {{ schema -}}.tf_fact_decoded_traces + signature: + - [block_height, INTEGER, The start block height to get the logs from] + - [to_latest, BOOLEAN, Whether to continue fetching logs until the latest block or not] + return_type: + - "TABLE(block_number INTEGER, tx_hash STRING, block_timestamp TIMESTAMP_NTZ, tx_status STRING, tx_position INTEGER, trace_index INTEGER, from_address STRING, to_address STRING, VALUE FLOAT, value_precise_raw STRING, value_precise STRING, gas INTEGER, gas_used INTEGER, TYPE STRING, identifier STRING, sub_traces INTEGER, error_reason STRING, trace_status STRING, input STRING, output STRING, function_name STRING, decoded_input_data VARIANT, decoded_output_data VARIANT, fact_decoded_traces_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + COMMENT = $$Returns the decoded traces data for a given block height. If to_latest is true, + it will continue fetching blocks until the latest block. Otherwise, + it will fetch blocks until the block height is reached.$$ + sql: | + {{ evm_live_view_fact_decoded_traces(schema, + blockchain, network) | indent(4) -}} + - name: {{ schema -}}.tf_fact_traces signature: - [block_height, INTEGER, The start block height to get the transfers from] diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index df0e3cd..daa39d6 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -1024,6 +1024,124 @@ SELECT FROM traces_final {% endmacro %} +{% macro evm_live_view_fact_decoded_traces(schema, blockchain, network) %} +WITH spine AS ( + {{ evm_live_view_target_blocks(schema, blockchain, network) | indent(4) -}} +), +raw_receipts AS ( + {{ evm_live_view_bronze_receipts(schema, 'spine') | indent(4) -}} +), +raw_block_txs AS ( + {{ evm_live_view_bronze_blocks(schema, 'spine') | indent(4) -}} +), +raw_transactions AS ( + {{ evm_live_view_bronze_transactions('raw_block_txs') | indent(4) -}} +), +blocks AS ( + {{ evm_live_view_silver_blocks('raw_block_txs') | indent(4) -}} +), +receipts AS ( + {{ evm_live_view_silver_receipts('raw_receipts') | indent(4) -}} +), +transactions AS ( + {{ evm_live_view_silver_transactions('raw_transactions', 'blocks', 'receipts') | indent(4) -}} +), +raw_traces AS ( + {{ evm_live_view_bronze_traces(schema, 'spine') | indent(4) -}} +), + +{{ evm_live_view_silver_traces('raw_traces') | indent(4) -}} +, + +decoded_traces AS ( + SELECT + t.block_number, + t.tx_hash, + t.block_timestamp, + t.tx_status, + t.tx_position, + t.trace_index, + t.from_address, + t.to_address, + t.eth_value AS VALUE, + t.eth_value_precise_raw AS value_precise_raw, + t.eth_value_precise AS value_precise, + t.gas, + t.gas_used, + t.TYPE AS TYPE, + t.identifier, + t.sub_traces, + t.error_reason, + t.trace_status, + A.abi AS abi, + A.function_name AS function_name, + CASE + WHEN TYPE = 'DELEGATECALL' THEN from_address + ELSE to_address + END AS abi_address, + t.input AS input, + COALESCE( + t.output, + '0x' + ) AS output, + OBJECT_CONSTRUCT('input', input, 'output', output, 'function_name', function_name) AS function_data, + utils.udf_evm_decode_trace(abi, function_data)[0] AS decoded_data + FROM traces_final t + INNER JOIN {{ blockchain }}.SILVER.COMPLETE_FUNCTION_ABIS A + ON A.parent_contract_address = abi_address + AND LEFT( + t.input, + 10 + ) = LEFT( + A.function_signature, + 10 + ) + AND t.block_number BETWEEN A.start_block + AND A.end_block + AND t.block_number IS NOT NULL + +) + +SELECT + block_number, + tx_hash, + block_timestamp, + tx_status, + tx_position, + trace_index, + from_address, + to_address, + VALUE, + value_precise_raw, + value_precise, + gas, + gas_used, + TYPE, + identifier, + sub_traces, + error_reason, + trace_status, + input, + output, + decoded_data :function_name :: STRING AS function_name, + decoded_data :decoded_input_data AS decoded_input_data, + decoded_data :decoded_output_data AS decoded_output_data, + md5( + cast( + coalesce( + cast(tx_hash as TEXT), + '_dbt_utils_surrogate_key_null_' + ) || '-' || coalesce( + cast(trace_index as TEXT), + '_dbt_utils_surrogate_key_null_' + ) as TEXT + ) + ) AS fact_decoded_traces_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM decoded_traces +{% endmacro %} + {% macro evm_live_view_fact_token_balances(schema, blockchain, network) %} WITH silver_token_balances AS ( {{ evm_live_view_silver_token_balances(schema, blockchain, network) | indent(4) -}} From 30e9964b9c3b4737c99f9f8a50dd37c18a2ba280 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Mon, 4 Nov 2024 14:15:54 +0900 Subject: [PATCH 39/50] revert requirements --- requirements.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 29a3fa6..f32ca68 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1 @@ dbt-snowflake~=1.5.0 -snowflake-connector-python~=3.12.2 -pyarrow~=17.0.0 -h2o-wave~=1.5.1 From dfc5d61bd34530c7dc85557da71103ca22eea871 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Tue, 5 Nov 2024 08:59:37 -0800 Subject: [PATCH 40/50] Fix typo in README.md and update macro in evm_live_views.sql to fix `to_latest` --- README.md | 2 +- macros/evm/evm_live_views.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4e64cd9..eb6ccb0 100644 --- a/README.md +++ b/README.md @@ -469,7 +469,7 @@ When False, none of the on-run-start macros are executed on model run Default values are False * Usage: -`dbt run --var '{"UPDATE_UDFS_AND_SPS":True}' -m ...` +`dbt run --vars` '{"UPDATE_UDFS_AND_SPS":True}' -m ...` Dropping and creating udfs can also be done without running a model: diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index daa39d6..680fa2f 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -1,7 +1,7 @@ {% macro evm_live_view_latest_block_height(schema, blockchain, network) %} SELECT {{ schema }}.udf_rpc('eth_blockNumber', []) as result, - utils.udf_hex_to_int(result:result)::integer AS latest_block_height, + utils.udf_hex_to_int(result)::integer AS latest_block_height, COALESCE( block_height, latest_block_height From ea9b4cab9eff9e39b347a1e483a35ebfb57be16f Mon Sep 17 00:00:00 2001 From: shah Date: Tue, 5 Nov 2024 16:19:38 -0800 Subject: [PATCH 41/50] fix typo in readme.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index eb6ccb0..088e803 100644 --- a/README.md +++ b/README.md @@ -469,7 +469,7 @@ When False, none of the on-run-start macros are executed on model run Default values are False * Usage: -`dbt run --vars` '{"UPDATE_UDFS_AND_SPS":True}' -m ...` +`dbt run --vars '{"UPDATE_UDFS_AND_SPS":True}' -m ...` Dropping and creating udfs can also be done without running a model: From d381ce85d82c4b39cf98849b5525e7e9b6dcdabb Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Wed, 6 Nov 2024 18:41:59 +0900 Subject: [PATCH 42/50] add batch calls in bronze blocks and logs --- macros/evm/evm.yaml.sql | 6 +- macros/evm/evm_live_views.sql | 106 +++++++++++++++++++++++++--------- 2 files changed, 82 insertions(+), 30 deletions(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index 900e85c..df40296 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -365,8 +365,8 @@ - name: {{ schema -}}.tf_fact_blocks signature: - - [block_height, INTEGER, The start block height to get the transfers from] - - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] + - [block_height, INTEGER, The start block height to get the blocks from] + - [to_latest, BOOLEAN, Whether to continue fetching blocks until the latest block or not] return_type: - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, network STRING, blockchain STRING, tx_count INTEGER, difficulty INTEGER, total_difficulty INTEGER, extra_data STRING, gas_limit INTEGER, gas_used INTEGER, hash STRING, parent_hash STRING, miner STRING, nonce INTEGER, receipts_root STRING, sha3_uncles STRING, size INTEGER, uncle_blocks VARIANT, block_header_json OBJECT, excess_blob_gas INTEGER, blob_gas_used INTEGER, fact_blocks_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ, withdrawals VARIANT, withdrawals_root STRING)" options: | @@ -420,7 +420,7 @@ it will fetch blocks until the block height is reached.$$ sql: | {{ evm_live_view_fact_decoded_traces(schema, - blockchain, network) | indent(4) -}} + blockchain, network) | indent(4) -}} - name: {{ schema -}}.tf_fact_traces signature: diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 680fa2f..f55252c 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -16,47 +16,99 @@ {% macro evm_live_view_target_blocks(schema, blockchain, network) %} WITH heights AS ( {{ evm_live_view_latest_block_height(schema, blockchain, network) | indent(4) -}} + ), + + block_spine AS ( + SELECT + ROW_NUMBER() OVER ( + ORDER BY + NULL + ) - 1 + COALESCE(block_height, latest_block_height)::integer AS block_number, + min_height, + IFF( + COALESCE(to_latest, false), + block_height, + min_height + ) AS max_height, + latest_block_height + FROM + TABLE(generator(ROWCOUNT => 1000)), + heights qualify block_number BETWEEN min_height + AND max_height ) + SELECT - ROW_NUMBER() OVER ( - ORDER BY - NULL - ) - 1 + COALESCE(block_height, latest_block_height)::integer AS block_number, - min_height, - IFF( - COALESCE(to_latest, false), - block_height, - min_height - ) AS max_height, + CEIL(ROW_NUMBER() OVER (ORDER BY block_number) / {{ var('BATCH_SIZE') }}) AS batch_id, + block_number, latest_block_height - FROM - TABLE(generator(ROWCOUNT => 1000)), - heights qualify block_number BETWEEN min_height - AND max_height + FROM block_spine {% endmacro %} -- Get Raw EVM chain data {% macro evm_live_view_bronze_blocks(schema, table_name) %} +WITH blocks_agg AS ( + SELECT + batch_id, + ARRAY_AGG( + utils.udf_json_rpc_call( + 'eth_getBlockByNumber', + [utils.udf_int_to_hex(block_number), true] + ) + ) AS params + FROM + {{ table_name }} + GROUP BY 1 +), + +get_batch_result AS ( + SELECT + live.udf_api( + '{endpoint}', + params + ):data AS result, + COALESCE(value:result, {'error':value:error}) AS data + FROM blocks_agg, LATERAL FLATTEN(input => result) v +) + SELECT - block_number, - {{ schema }}.udf_rpc( - 'eth_getBlockByNumber', - [utils.udf_int_to_hex(block_number), true]) AS DATA -FROM - {{ table_name }} + utils.udf_hex_to_int(data:number::STRING)::INT AS block_number, + data +FROM get_batch_result {% endmacro %} {% macro evm_live_view_bronze_receipts(schema, table_name) %} +WITH blocks_agg AS ( + SELECT + batch_id, + latest_block_height, + ARRAY_AGG( + utils.udf_json_rpc_call( + 'eth_getBlockReceipts', + [utils.udf_int_to_hex(block_number)] + ) + ) AS params + FROM + {{ table_name }} + GROUP BY 1,2 +), + +get_batch_result AS ( + SELECT + latest_block_height, + live.udf_api( + '{endpoint}', + params + ):data AS result, + COALESCE(value:result, {'error':value:error}) AS data + FROM blocks_agg, LATERAL FLATTEN(input => result) v +) + SELECT latest_block_height, - block_number, - {{ schema }}.udf_rpc( - 'eth_getBlockReceipts', - [utils.udf_int_to_hex(block_number)]) AS result, + utils.udf_hex_to_int(v.value:blockNumber::STRING)::INT AS block_number, v.value AS DATA -FROM - {{ table_name }}, - LATERAL FLATTEN(result) v +FROM get_batch_result, + LATERAL FLATTEN(data) v {% endmacro %} {% macro evm_live_view_bronze_logs(table_name) %} From 14f52fce1f9a45bd6ef99269bb83a92de62b348c Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Wed, 6 Nov 2024 09:38:32 -0800 Subject: [PATCH 43/50] update evm_live_view_target_blocks macro to accept batch_size parameter --- macros/evm/evm_live_views.sql | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index f55252c..ac56965 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -13,11 +13,10 @@ ) AS max_height {% endmacro %} -{% macro evm_live_view_target_blocks(schema, blockchain, network) %} +{% macro evm_live_view_target_blocks(schema, blockchain, network, batch_size=10) %} WITH heights AS ( {{ evm_live_view_latest_block_height(schema, blockchain, network) | indent(4) -}} ), - block_spine AS ( SELECT ROW_NUMBER() OVER ( @@ -38,7 +37,7 @@ ) SELECT - CEIL(ROW_NUMBER() OVER (ORDER BY block_number) / {{ var('BATCH_SIZE') }}) AS batch_id, + CEIL(ROW_NUMBER() OVER (ORDER BY block_number) / {{ batch_size }}) AS batch_id, block_number, latest_block_height FROM block_spine From eb94d0ffa9536fbda2fa1ccb4dd504b7d4aa8b11 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:04:06 -0800 Subject: [PATCH 44/50] refactor evm_live_view_bronze_blocks macro to include blockchain and network parameters; add sql_live_rpc_batch_call helper macro for batch JSON RPC calls --- macros/evm/evm_live_views.sql | 22 ++++++++-------------- macros/livequery/utils.sql | 28 ++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index ac56965..e454824 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -44,7 +44,7 @@ {% endmacro %} -- Get Raw EVM chain data -{% macro evm_live_view_bronze_blocks(schema, table_name) %} +{% macro evm_live_view_bronze_blocks(schema, blockchain, network, table_name) %} WITH blocks_agg AS ( SELECT batch_id, @@ -60,13 +60,7 @@ WITH blocks_agg AS ( ), get_batch_result AS ( - SELECT - live.udf_api( - '{endpoint}', - params - ):data AS result, - COALESCE(value:result, {'error':value:error}) AS data - FROM blocks_agg, LATERAL FLATTEN(input => result) v + {{ sql_live_rpc_batch_call('blocks_agg', 'params', blockchain, network) | indent(4) }} ) SELECT @@ -776,7 +770,7 @@ WITH spine AS ( {{ evm_live_view_target_blocks(schema, blockchain, network) | indent(4) -}} ), raw_block_txs AS ( - {{ evm_live_view_bronze_blocks( schema, 'spine') | indent(4) -}} + {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} ), silver_blocks AS ( {{ evm_live_view_silver_blocks('raw_block_txs') | indent(4) -}} @@ -859,7 +853,7 @@ WITH spine AS ( {{ evm_live_view_target_blocks(schema, blockchain, network) | indent(4) -}} ), raw_block_txs AS ( - {{ evm_live_view_bronze_blocks(schema, 'spine') | indent(4) -}} + {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} ), raw_receipts AS ( {{ evm_live_view_bronze_receipts(schema, 'spine') | indent(4) -}} @@ -945,7 +939,7 @@ raw_receipts AS ( {{ evm_live_view_bronze_receipts(schema, 'spine') | indent(4) -}} ), raw_block_txs AS ( - {{ evm_live_view_bronze_blocks(schema, 'spine') | indent(4) -}} + {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} ), raw_transactions AS ( {{ evm_live_view_bronze_transactions('raw_block_txs') | indent(4) -}} @@ -1015,7 +1009,7 @@ raw_receipts AS ( {{ evm_live_view_bronze_receipts(schema, 'spine') | indent(4) -}} ), raw_block_txs AS ( - {{ evm_live_view_bronze_blocks(schema, 'spine') | indent(4) -}} + {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} ), raw_transactions AS ( {{ evm_live_view_bronze_transactions('raw_block_txs') | indent(4) -}} @@ -1083,7 +1077,7 @@ raw_receipts AS ( {{ evm_live_view_bronze_receipts(schema, 'spine') | indent(4) -}} ), raw_block_txs AS ( - {{ evm_live_view_bronze_blocks(schema, 'spine') | indent(4) -}} + {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} ), raw_transactions AS ( {{ evm_live_view_bronze_transactions('raw_block_txs') | indent(4) -}} @@ -1405,7 +1399,7 @@ raw_receipts AS ( {{ evm_live_view_bronze_receipts(schema, 'spine') | indent(4) -}} ), raw_block_txs AS ( - {{ evm_live_view_bronze_blocks(schema, 'spine') | indent(4) -}} + {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} ), raw_transactions AS ( {{ evm_live_view_bronze_transactions('raw_block_txs') | indent(4) -}} diff --git a/macros/livequery/utils.sql b/macros/livequery/utils.sql index 3da87bd..81f8bf9 100644 --- a/macros/livequery/utils.sql +++ b/macros/livequery/utils.sql @@ -4,7 +4,7 @@ Parameters: method (string): The JSON RPC method to call. - params (string): The JSON RPC parameters to pass to the method. + params (array): The JSON RPC parameters to pass to the method. blockchain (string): The blockchain to call the method on. network (string): The network to call the method on. Returns: @@ -15,10 +15,34 @@ live.udf_api( '{endpoint}' ,utils.udf_json_rpc_call({{ method }}, {{ params }}) - ,concat_ws('/', 'integration', _utils.udf_provider(), {{ blockchain }}, {{ network }}) + ,concat_ws('/', 'integration', _utils.udf_provider(), '{{ blockchain }}', '{{ network }}') )::VARIANT:data AS data ) SELECT COALESCE(data:result, {'error':data:error}) FROM result +{% endmacro -%} + +{% macro sql_live_rpc_batch_call(from, calls, blockchain, network) %} +{# + Helper macro to batch call a JSON RPC method on a live node. + + Parameters: + calls (array): The JSON RPC parameters to pass to the method. + blockchain (string): The blockchain to call the method on. + network (string): The network to call the method on. + Returns: + string: The SQL to call the method. + #} + WITH result as ( + SELECT + live.udf_api( + '{endpoint}' + ,{{ calls }} + ,concat_ws('/', 'integration', _utils.udf_provider(), {{ blockchain }}, {{ network }}) + )::VARIANT:data::ARRAY AS data + ) + SELECT + COALESCE(value:result, {'error':value:error}) + FROM result, LATERAL FLATTEN(input => result.data) v {% endmacro -%} \ No newline at end of file From 1d85053304315db2b2ae717789704b9652d13de7 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Wed, 6 Nov 2024 15:59:24 -0800 Subject: [PATCH 45/50] refactor evm_live_views.sql to improve batch processing and remove unused sql_live_rpc_batch_call macro --- macros/evm/evm_live_views.sql | 25 +++++++++++++++++-------- macros/livequery/utils.sql | 24 ------------------------ 2 files changed, 17 insertions(+), 32 deletions(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index e454824..8bfa3f9 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -56,17 +56,26 @@ WITH blocks_agg AS ( ) AS params FROM {{ table_name }} - GROUP BY 1 -), - -get_batch_result AS ( - {{ sql_live_rpc_batch_call('blocks_agg', 'params', blockchain, network) | indent(4) }} + GROUP BY batch_id +), result as ( + SELECT + live.udf_api( + '{endpoint}' + ,params + ,concat_ws('/', 'integration', _utils.udf_provider(), '{{ blockchain }}', '{{ network }}') + )::VARIANT:data::ARRAY AS data + FROM blocks_agg +) +, flattened as ( + SELECT + COALESCE(value:result, {'error':value:error}) AS result + FROM result, LATERAL FLATTEN(input => result.data) v ) SELECT - utils.udf_hex_to_int(data:number::STRING)::INT AS block_number, - data -FROM get_batch_result + utils.udf_hex_to_int(result:number::STRING)::INT AS block_number, + result as data +FROM flattened {% endmacro %} {% macro evm_live_view_bronze_receipts(schema, table_name) %} diff --git a/macros/livequery/utils.sql b/macros/livequery/utils.sql index 81f8bf9..8288d2f 100644 --- a/macros/livequery/utils.sql +++ b/macros/livequery/utils.sql @@ -22,27 +22,3 @@ COALESCE(data:result, {'error':data:error}) FROM result {% endmacro -%} - -{% macro sql_live_rpc_batch_call(from, calls, blockchain, network) %} -{# - Helper macro to batch call a JSON RPC method on a live node. - - Parameters: - calls (array): The JSON RPC parameters to pass to the method. - blockchain (string): The blockchain to call the method on. - network (string): The network to call the method on. - Returns: - string: The SQL to call the method. - #} - WITH result as ( - SELECT - live.udf_api( - '{endpoint}' - ,{{ calls }} - ,concat_ws('/', 'integration', _utils.udf_provider(), {{ blockchain }}, {{ network }}) - )::VARIANT:data::ARRAY AS data - ) - SELECT - COALESCE(value:result, {'error':value:error}) - FROM result, LATERAL FLATTEN(input => result.data) v -{% endmacro -%} \ No newline at end of file From 7e1ff90f2f1e7352b71f85aebc3ce5f0d16b8e40 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Thu, 7 Nov 2024 10:32:07 +0900 Subject: [PATCH 46/50] add batch call to debug trace call --- macros/evm/evm_live_views.sql | 36 +++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 8bfa3f9..3764213 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -132,17 +132,41 @@ FROM {% endmacro %} {% macro evm_live_view_bronze_traces(schema, table_name)%} +WITH blocks_agg AS ( + SELECT + batch_id, + ARRAY_AGG( + utils.udf_json_rpc_call( + 'debug_traceBlockByNumber', + [utils.udf_int_to_hex(s.block_number), {'tracer': 'callTracer'}], + s.block_number -- to put block_number in the id to retrieve the block numberlater + ) + ) AS params + FROM + {{ table_name }} + GROUP BY batch_id +), result as ( + SELECT + live.udf_api( + '{endpoint}' + ,params + ,concat_ws('/', 'integration', _utils.udf_provider(), '{{ blockchain }}', '{{ network }}') + )::VARIANT:data::ARRAY AS data + FROM blocks_agg +), flattened as ( + SELECT + value:id::INT AS block_number, + COALESCE(value:result, {'error':value:error}) AS result + FROM result, LATERAL FLATTEN(input => result.data) v +) + SELECT s.block_number, v.index::INT AS tx_position, -- mimic's streamline's logic to add tx_position v.value:result AS full_traces, SYSDATE() AS _inserted_timestamp -FROM {{ table_name }} s, -LATERAL FLATTEN(input => PARSE_JSON( - {{ schema }}.udf_rpc( - 'debug_traceBlockByNumber', - [utils.udf_int_to_hex(s.block_number), {'tracer': 'callTracer'}]) -)) v +FROM flattened s, +LATERAL FLATTEN(input => result) v {% endmacro %} {% macro evm_live_view_bronze_token_balances(schema, table_name) %} From 8580a6712a43a10bdf4958d8150f610cbcd42e3c Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Thu, 7 Nov 2024 10:41:58 +0900 Subject: [PATCH 47/50] fix missing alias --- macros/evm/evm_live_views.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 3764213..4b6c42f 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -143,7 +143,7 @@ WITH blocks_agg AS ( ) ) AS params FROM - {{ table_name }} + {{ table_name }} s GROUP BY batch_id ), result as ( SELECT From f417a2df75c1345d2133e5038628f15393647db2 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Thu, 7 Nov 2024 22:49:18 +0900 Subject: [PATCH 48/50] add batch trace, eth balance & token balance --- macros/evm/evm_live_views.sql | 197 ++++++++++++++++++++++------------ 1 file changed, 130 insertions(+), 67 deletions(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 4b6c42f..1a5ea22 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -43,6 +43,14 @@ FROM block_spine {% endmacro %} +{% macro evm_batch_udf_api(blockchain, network) %} + live.udf_api( + '{endpoint}' + ,params + ,concat_ws('/', 'integration', _utils.udf_provider(), '{{ blockchain }}', '{{ network }}') + )::VARIANT:data::ARRAY AS data +{% endmacro %} + -- Get Raw EVM chain data {% macro evm_live_view_bronze_blocks(schema, blockchain, network, table_name) %} WITH blocks_agg AS ( @@ -59,11 +67,7 @@ WITH blocks_agg AS ( GROUP BY batch_id ), result as ( SELECT - live.udf_api( - '{endpoint}' - ,params - ,concat_ws('/', 'integration', _utils.udf_provider(), '{{ blockchain }}', '{{ network }}') - )::VARIANT:data::ARRAY AS data + {{ evm_batch_udf_api(blockchain, network) }} FROM blocks_agg ) , flattened as ( @@ -78,7 +82,7 @@ SELECT FROM flattened {% endmacro %} -{% macro evm_live_view_bronze_receipts(schema, table_name) %} +{% macro evm_live_view_bronze_receipts(schema, blockchain, network, table_name) %} WITH blocks_agg AS ( SELECT batch_id, @@ -97,20 +101,20 @@ WITH blocks_agg AS ( get_batch_result AS ( SELECT latest_block_height, - live.udf_api( - '{endpoint}', - params - ):data AS result, - COALESCE(value:result, {'error':value:error}) AS data - FROM blocks_agg, LATERAL FLATTEN(input => result) v + {{ evm_batch_udf_api(blockchain, network) }} + FROM blocks_agg, LATERAL FLATTEN(input => data) v ) SELECT latest_block_height, - utils.udf_hex_to_int(v.value:blockNumber::STRING)::INT AS block_number, - v.value AS DATA -FROM get_batch_result, - LATERAL FLATTEN(data) v + utils.udf_hex_to_int(w.value:blockNumber::STRING)::INT AS block_number, + w.value AS DATA +FROM + (SELECT + latest_block_height, + v.value:result AS DATA + FROM get_batch_result, + LATERAL FLATTEN(data) v), LATERAL FLATTEN(data) w {% endmacro %} {% macro evm_live_view_bronze_logs(table_name) %} @@ -131,7 +135,7 @@ FROM lateral flatten(r.data:transactions) v {% endmacro %} -{% macro evm_live_view_bronze_traces(schema, table_name)%} +{% macro evm_live_view_bronze_traces(schema, blockchain, network, table_name)%} WITH blocks_agg AS ( SELECT batch_id, @@ -147,11 +151,7 @@ WITH blocks_agg AS ( GROUP BY batch_id ), result as ( SELECT - live.udf_api( - '{endpoint}' - ,params - ,concat_ws('/', 'integration', _utils.udf_provider(), '{{ blockchain }}', '{{ network }}') - )::VARIANT:data::ARRAY AS data + {{ evm_batch_udf_api(blockchain, network) }} FROM blocks_agg ), flattened as ( SELECT @@ -169,43 +169,100 @@ FROM flattened s, LATERAL FLATTEN(input => result) v {% endmacro %} -{% macro evm_live_view_bronze_token_balances(schema, table_name) %} -SELECT - block_number, - block_timestamp, - address, - contract_address, - {{ schema }}.udf_rpc( - 'eth_call', - ARRAY_CONSTRUCT( - OBJECT_CONSTRUCT( - 'to', - contract_address, - 'data', +{% macro evm_live_view_bronze_token_balances(schema, blockchain, network, table_name) %} +WITH block_spine AS ( + SELECT + CEIL(ROW_NUMBER() OVER (ORDER BY block_number, address, contract_address) / 10) AS batch_id, + block_number, + address, + contract_address + FROM + {{ table_name }} +), +blocks_agg AS ( + SELECT + batch_id, + ARRAY_AGG( + utils.udf_json_rpc_call( + 'eth_call', + ARRAY_CONSTRUCT( + OBJECT_CONSTRUCT( + 'to', + contract_address, + 'data', + CONCAT( + '0x70a08231000000000000000000000000', + SUBSTR( + address, + 3 + ) + ) + ), + utils.udf_int_to_hex(block_number) + ), CONCAT( - '0x70a08231000000000000000000000000', - SUBSTR( - address, - 3 - ) + block_number, + '-', + address, + '-', + contract_address ) - ), - utils.udf_int_to_hex(block_number) - ) - ) AS DATA -FROM {{ table_name }} + ) + ) AS params + FROM + block_spine + GROUP BY batch_id +), result as ( + SELECT + {{ evm_batch_udf_api(blockchain, network) }} + FROM blocks_agg +) + +SELECT + SPLIT(value:id::STRING, '-')[0]::INT AS block_number, + SPLIT(value:id::STRING, '-')[1]::STRING AS address, + SPLIT(value:id::STRING, '-')[2]::STRING AS contract_address, + COALESCE(value:result, {'error':value:error}) AS DATA +FROM result, LATERAL FLATTEN(input => result.data) v {% endmacro %} -{% macro evm_live_view_bronze_eth_balances(schema, table_name) %} +{% macro evm_live_view_bronze_eth_balances(schema, blockchain, network, table_name) %} +WITH block_spine AS ( + SELECT + CEIL(ROW_NUMBER() OVER (ORDER BY block_number, address) / 10) AS batch_id, + block_number, + address + FROM + {{ table_name }} +), +blocks_agg AS ( + SELECT + batch_id, + ARRAY_AGG( + utils.udf_json_rpc_call( + 'eth_getBalance', + ARRAY_CONSTRUCT(address, utils.udf_int_to_hex(block_number)), + CONCAT( + block_number, + '-', + address + ) + ) + ) AS params + FROM + block_spine + GROUP BY batch_id +), result as ( + SELECT + {{ evm_batch_udf_api(blockchain, network) }} + FROM blocks_agg +) + SELECT - block_number, - block_timestamp, - address, - {{ schema }}.udf_rpc( - 'eth_getBalance', - ARRAY_CONSTRUCT(address, utils.udf_int_to_hex(block_number)) - ) AS DATA -FROM {{ table_name }} + SPLIT(value:id::STRING, '-')[0]::INT AS block_number, + SPLIT(value:id::STRING, '-')[1]::STRING AS address, + COALESCE(value:result, {'error':value:error}) AS DATA +FROM result, LATERAL FLATTEN(input => result.data) v {% endmacro %} -- Transformation macro for EVM chains @@ -673,11 +730,11 @@ transfers AS ( ), balances AS ( - {{ evm_live_view_bronze_token_balances(schema, 'transfers') | indent(4) -}} + {{ evm_live_view_bronze_token_balances(schema, blockchain, network, 'transfers') | indent(4) -}} ) SELECT - block_number, + b.block_number, block_timestamp, address, contract_address, @@ -718,7 +775,10 @@ SELECT id AS token_balances_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp -FROM balances +FROM balances b +LEFT JOIN ( + SELECT DISTINCT block_number, block_timestamp FROM transfers +) USING (block_number) {% endmacro %} {% macro evm_live_view_silver_eth_balances(schema, blockchain, network) %} @@ -761,7 +821,7 @@ stacked AS ( ), eth_balances AS ( - {{ evm_live_view_bronze_eth_balances(schema, 'stacked') | indent(4) -}} + {{ evm_live_view_bronze_eth_balances(schema, blockchain, network, 'stacked') | indent(4) -}} ) SELECT @@ -795,6 +855,9 @@ SELECT SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp FROM eth_balances +LEFT JOIN ( + SELECT DISTINCT block_number, block_timestamp FROM stacked +) USING (block_number) {% endmacro %} -- Get EVM chain fact data @@ -889,7 +952,7 @@ raw_block_txs AS ( {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} ), raw_receipts AS ( - {{ evm_live_view_bronze_receipts(schema, 'spine') | indent(4) -}} + {{ evm_live_view_bronze_receipts(schema, blockchain, network, 'spine') | indent(4) -}} ), raw_logs AS ( {{ evm_live_view_bronze_logs('raw_receipts') | indent(4) -}} @@ -969,7 +1032,7 @@ WITH spine AS ( {{ evm_live_view_target_blocks(schema, blockchain, network) | indent(4) -}} ), raw_receipts AS ( - {{ evm_live_view_bronze_receipts(schema, 'spine') | indent(4) -}} + {{ evm_live_view_bronze_receipts(schema, blockchain, network, 'spine') | indent(4) -}} ), raw_block_txs AS ( {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} @@ -1039,7 +1102,7 @@ WITH spine AS ( {{ evm_live_view_target_blocks(schema, blockchain, network) | indent(4) -}} ), raw_receipts AS ( - {{ evm_live_view_bronze_receipts(schema, 'spine') | indent(4) -}} + {{ evm_live_view_bronze_receipts(schema, blockchain, network, 'spine') | indent(4) -}} ), raw_block_txs AS ( {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} @@ -1057,7 +1120,7 @@ transactions AS ( {{ evm_live_view_silver_transactions('raw_transactions', 'blocks', 'receipts') | indent(4) -}} ), raw_traces AS ( - {{ evm_live_view_bronze_traces(schema, 'spine') | indent(4) -}} + {{ evm_live_view_bronze_traces(schema, blockchain, network, 'spine') | indent(4) -}} ), {{ evm_live_view_silver_traces('raw_traces') | indent(4) -}} @@ -1104,10 +1167,10 @@ FROM traces_final {% macro evm_live_view_fact_decoded_traces(schema, blockchain, network) %} WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network) | indent(4) -}} + {{ evm_live_view_target_blocks(schema, blockchain, network, 5) | indent(4) -}} ), raw_receipts AS ( - {{ evm_live_view_bronze_receipts(schema, 'spine') | indent(4) -}} + {{ evm_live_view_bronze_receipts(schema, blockchain, network, 'spine') | indent(4) -}} ), raw_block_txs AS ( {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} @@ -1125,7 +1188,7 @@ transactions AS ( {{ evm_live_view_silver_transactions('raw_transactions', 'blocks', 'receipts') | indent(4) -}} ), raw_traces AS ( - {{ evm_live_view_bronze_traces(schema, 'spine') | indent(4) -}} + {{ evm_live_view_bronze_traces(schema, blockchain, network, 'spine') | indent(4) -}} ), {{ evm_live_view_silver_traces('raw_traces') | indent(4) -}} @@ -1429,7 +1492,7 @@ WITH spine AS ( {{ evm_live_view_target_blocks(schema, blockchain, network) | indent(4) -}} ), raw_receipts AS ( - {{ evm_live_view_bronze_receipts(schema, 'spine') | indent(4) -}} + {{ evm_live_view_bronze_receipts(schema, blockchain, network, 'spine') | indent(4) -}} ), raw_block_txs AS ( {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} @@ -1447,7 +1510,7 @@ transactions AS ( {{ evm_live_view_silver_transactions('raw_transactions', 'blocks', 'receipts') | indent(4) -}} ), raw_traces AS ( - {{ evm_live_view_bronze_traces(schema, 'spine') | indent(4) -}} + {{ evm_live_view_bronze_traces(schema, blockchain, network, 'spine') | indent(4) -}} ), {{ evm_live_view_silver_traces('raw_traces') | indent(4) -}} , From da8e648d742556e14183eeb27daf7ee9ca8cc0ff Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Tue, 12 Nov 2024 19:35:27 +0900 Subject: [PATCH 49/50] fix bug in duplicate rows --- macros/evm/evm_live_views.sql | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 1a5ea22..adbdf52 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -102,7 +102,7 @@ get_batch_result AS ( SELECT latest_block_height, {{ evm_batch_udf_api(blockchain, network) }} - FROM blocks_agg, LATERAL FLATTEN(input => data) v + FROM blocks_agg ) SELECT @@ -946,7 +946,7 @@ WITH spine AS ( {% macro evm_live_view_fact_event_logs(schema, blockchain, network) %} WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network) | indent(4) -}} + {{ evm_live_view_target_blocks(schema, blockchain, network, 5) | indent(4) -}} ), raw_block_txs AS ( {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} @@ -1029,7 +1029,7 @@ FROM _ez_decoded_event_logs {% macro evm_live_view_fact_transactions(schema, blockchain, network) %} WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network) | indent(4) -}} + {{ evm_live_view_target_blocks(schema, blockchain, network, 5) | indent(4) -}} ), raw_receipts AS ( {{ evm_live_view_bronze_receipts(schema, blockchain, network, 'spine') | indent(4) -}} @@ -1413,7 +1413,11 @@ SELECT SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp FROM _flatten_logs AS B -LEFT JOIN _silver_decoded_logs AS C USING (block_number, _log_id) +LEFT JOIN ( + SELECT + DISTINCT block_number, _log_id + FROM _silver_decoded_logs + ) AS C USING (block_number, _log_id) LEFT JOIN {{ blockchain }}.core.dim_contracts AS D ON B.contract_address = D.address {% endmacro %} From c65d739d9bb5aa95e760ae29ca62749f62b51f44 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Tue, 12 Nov 2024 20:29:19 +0900 Subject: [PATCH 50/50] revert changes in join logic --- macros/evm/evm_live_views.sql | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index adbdf52..542410e 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -1413,11 +1413,7 @@ SELECT SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp FROM _flatten_logs AS B -LEFT JOIN ( - SELECT - DISTINCT block_number, _log_id - FROM _silver_decoded_logs - ) AS C USING (block_number, _log_id) +LEFT JOIN _silver_decoded_logs AS C USING (block_number, _log_id) LEFT JOIN {{ blockchain }}.core.dim_contracts AS D ON B.contract_address = D.address {% endmacro %}