From 747ca2f8039d0d2a0f19d1c9eb9ca9de1bc190b5 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 5 Feb 2025 14:52:48 +0800 Subject: [PATCH] core changes --- .../gold/core/core__ez_native_transfers.sql | 4 -- models/gold/defi/defi__ez_dex_swaps.sql | 4 +- models/gold/nft/nft__ez_nft_sales.sql | 16 +++++--- models/gold/nft/nft__ez_nft_transfers.sql | 34 +++++++++++++---- models/silver/abis/silver__proxies.sql | 4 +- .../abis/silver__user_verified_abis.sql | 2 +- .../silver/core/silver__created_contracts.sql | 4 +- models/silver/core/silver__logs.sql | 6 +++ .../silver/core/silver__native_transfers.sql | 37 ++++++++----------- .../core/silver__relevant_contracts.sql | 10 ++--- models/silver/core/silver__transfers.sql | 12 ++++-- models/silver/core/silver__tx_count.sql | 6 +-- .../silver_olas__create_service_multisigs.sql | 20 ++++++---- .../olas/silver_olas__service_event_logs.sql | 22 ++++++----- .../silver_olas__service_registrations.sql | 23 +++++++----- 15 files changed, 120 insertions(+), 84 deletions(-) diff --git a/models/gold/core/core__ez_native_transfers.sql b/models/gold/core/core__ez_native_transfers.sql index c918992..165eaab 100644 --- a/models/gold/core/core__ez_native_transfers.sql +++ b/models/gold/core/core__ez_native_transfers.sql @@ -10,10 +10,6 @@ SELECT block_timestamp, tx_position, trace_index, - trace_address, - -- new column - TYPE, - -- new column origin_from_address, origin_to_address, origin_function_signature, diff --git a/models/gold/defi/defi__ez_dex_swaps.sql b/models/gold/defi/defi__ez_dex_swaps.sql index e029219..68ed523 100644 --- a/models/gold/defi/defi__ez_dex_swaps.sql +++ b/models/gold/defi/defi__ez_dex_swaps.sql @@ -53,7 +53,6 @@ SELECT token_out, symbol_in, symbol_out, - _log_id, COALESCE ( complete_dex_swaps_id, {{ dbt_utils.generate_surrogate_key( @@ -67,6 +66,7 @@ SELECT COALESCE( modified_timestamp, '2000-01-01' - ) AS modified_timestamp + ) AS modified_timestamp, + _log_id --deprecate FROM {{ ref('silver_dex__complete_dex_swaps') }} diff --git a/models/gold/nft/nft__ez_nft_sales.sql b/models/gold/nft/nft__ez_nft_sales.sql index aa25263..fb5b7e4 100644 --- a/models/gold/nft/nft__ez_nft_sales.sql +++ b/models/gold/nft/nft__ez_nft_sales.sql @@ -17,10 +17,11 @@ SELECT aggregator_name, seller_address, buyer_address, - nft_address, - project_name, - tokenId, - erc1155_value, + nft_address as contract_address, -- new + project_name as name, -- new + tokenId as token_id, -- new + coalesce(erc1155_value, '1')::STRING AS quantity, + case when erc1155_value is null then 'erc721' else 'erc1155' end as token_standard, currency_symbol, currency_address, price, @@ -49,6 +50,11 @@ SELECT COALESCE( modified_timestamp, '2000-01-01' - ) AS modified_timestamp + ) AS modified_timestamp, + tokenId, --deprecate + erc1155_value, --deprecate + project_name, --deprecate + nft_address --deprecate + FROM {{ ref('silver__complete_nft_sales') }} diff --git a/models/gold/nft/nft__ez_nft_transfers.sql b/models/gold/nft/nft__ez_nft_transfers.sql index 486b18a..0ebea7f 100644 --- a/models/gold/nft/nft__ez_nft_transfers.sql +++ b/models/gold/nft/nft__ez_nft_transfers.sql @@ -9,15 +9,26 @@ SELECT block_timestamp, block_number, tx_hash, + {#tx_position, -- new #} event_index, intra_event_index, - event_type, - contract_address AS nft_address, - project_name, - from_address AS nft_from_address, - to_address AS nft_to_address, - tokenId, - erc1155_value, + token_transfer_type, + iff(from_address = '0x0000000000000000000000000000000000000000', TRUE, FALSE) as is_mint, -- new + from_address, -- new + to_address, -- new + contract_address, -- new + tokenId as token_id, -- new + coalesce(erc1155_value, '1')::STRING AS quantity, + case + when token_transfer_type = 'erc721_Transfer' then 'erc721' + when token_transfer_type = 'erc1155_TransferSingle' then 'erc1155' + when token_transfer_type = 'erc1155_TransferBatch' then 'erc1155' + end as token_standard, + project_name as name, -- new + {#origin_function_signature, -- new + origin_from_address, -- new + origin_to_address -- new + #} COALESCE ( nft_transfers_id, {{ dbt_utils.generate_surrogate_key( @@ -31,6 +42,13 @@ SELECT COALESCE( modified_timestamp, '2000-01-01' - ) AS modified_timestamp + ) AS modified_timestamp, + event_type, --deprecate + from_address as nft_from_address, --deprecate + to_address as nft_to_address, --deprecate + contract_address as nft_address, --deprecate + tokenId, --deprecate + erc1155_value, --deprecate + project_name --deprecate FROM {{ ref('silver__nft_transfers') }} diff --git a/models/silver/abis/silver__proxies.sql b/models/silver/abis/silver__proxies.sql index 3cceeaf..a328b24 100644 --- a/models/silver/abis/silver__proxies.sql +++ b/models/silver/abis/silver__proxies.sql @@ -16,8 +16,8 @@ WITH base AS ( {{ ref('core__fact_traces') }} WHERE TYPE = 'DELEGATECALL' - AND trace_status = 'SUCCESS' - AND tx_status = 'SUCCESS' + AND trace_succeeded + AND tx_succeeded AND from_address != to_address -- exclude self-calls {% if is_incremental() %} diff --git a/models/silver/abis/silver__user_verified_abis.sql b/models/silver/abis/silver__user_verified_abis.sql index fcab5b0..4de20b9 100644 --- a/models/silver/abis/silver__user_verified_abis.sql +++ b/models/silver/abis/silver__user_verified_abis.sql @@ -506,7 +506,7 @@ valid_traces AS ( SELECT base_address FROM - {{ ref('silver__logs') }} + {{ ref('core__fact_event_logs') }} l JOIN event_mapping ia ON ia.contract_address = l.contract_address diff --git a/models/silver/core/silver__created_contracts.sql b/models/silver/core/silver__created_contracts.sql index bbc1f30..23b956e 100644 --- a/models/silver/core/silver__created_contracts.sql +++ b/models/silver/core/silver__created_contracts.sql @@ -27,8 +27,8 @@ WHERE AND to_address IS NOT NULL AND input IS NOT NULL AND input != '0x' - AND tx_status = 'SUCCESS' - AND trace_status = 'SUCCESS' + AND tx_succeeded + AND trace_succeeded {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/core/silver__logs.sql b/models/silver/core/silver__logs.sql index 8de38ef..781a667 100644 --- a/models/silver/core/silver__logs.sql +++ b/models/silver/core/silver__logs.sql @@ -16,6 +16,7 @@ WITH base AS ( from_address AS origin_from_address, to_address AS origin_to_address, tx_status, + {#position as tx_position,#} -- new logs, _inserted_timestamp FROM @@ -39,6 +40,7 @@ flat_logs AS ( origin_from_address, origin_to_address, tx_status, + {#position as tx_position,#} -- new VALUE :address :: STRING AS contract_address, VALUE :blockHash :: STRING AS block_hash, VALUE :data :: STRING AS DATA, @@ -64,6 +66,7 @@ new_records AS ( txs.origin_function_signature, l.tx_status, l.contract_address, + {#l.tx_position,#} -- new l.block_hash, l.data, l.event_index, @@ -107,6 +110,7 @@ missing_data AS ( t.origin_to_address, txs.origin_function_signature, t.tx_status, + {#t.tx_position,#} -- new t.contract_address, t.block_hash, t.data, @@ -140,6 +144,7 @@ FINAL AS ( origin_to_address, origin_function_signature, tx_status, + {#tx_position,#} -- new contract_address, block_hash, DATA, @@ -162,6 +167,7 @@ SELECT origin_to_address, origin_function_signature, tx_status, + {#tx_position,#} -- new contract_address, block_hash, DATA, diff --git a/models/silver/core/silver__native_transfers.sql b/models/silver/core/silver__native_transfers.sql index 1b86ab5..17672d4 100644 --- a/models/silver/core/silver__native_transfers.sql +++ b/models/silver/core/silver__native_transfers.sql @@ -14,14 +14,9 @@ WITH eth_base AS ( block_number, block_timestamp, identifier, - --deprecate - trace_address, - -- new - TYPE, - -- new from_address, to_address, - VALUE, + VALUE AS eth_value, concat_ws( '-', block_number, @@ -33,8 +28,8 @@ WITH eth_base AS ( ) ) AS _call_id, modified_timestamp AS _inserted_timestamp, - value_precise_raw, - value_precise, + value_precise_raw AS eth_value_precise_raw, + value_precise AS eth_value_precise, tx_position, trace_index, origin_from_address, @@ -44,8 +39,8 @@ WITH eth_base AS ( {{ ref('core__fact_traces') }} WHERE VALUE > 0 - AND tx_status = 'SUCCESS' - AND trace_status = 'SUCCESS' + AND tx_succeeded + AND trace_succeeded AND TYPE NOT IN ( 'DELEGATECALL', 'STATICCALL' @@ -61,28 +56,26 @@ AND _inserted_timestamp >= ( {% endif %} ) SELECT + tx_hash, block_number, block_timestamp, - tx_hash, - tx_position, - trace_index, identifier, - trace_address, - TYPE, + origin_from_address, + origin_to_address, + origin_function_signature, from_address, to_address, - VALUE AS amount, - value_precise_raw AS amount_precise_raw, - value_precise AS amount_precise, + eth_value AS amount, + eth_value_precise_raw AS amount_precise_raw, + eth_value_precise AS amount_precise, ROUND( - VALUE * price, + eth_value * price, 2 ) AS amount_usd, _call_id, A._inserted_timestamp, - origin_from_address, - origin_to_address, - origin_function_signature, + tx_position, + trace_index, {{ dbt_utils.generate_surrogate_key( ['tx_hash', 'trace_index'] ) }} AS native_transfers_id, diff --git a/models/silver/core/silver__relevant_contracts.sql b/models/silver/core/silver__relevant_contracts.sql index 23aefb7..14ba6cf 100644 --- a/models/silver/core/silver__relevant_contracts.sql +++ b/models/silver/core/silver__relevant_contracts.sql @@ -10,14 +10,14 @@ WITH emitted_events AS ( SELECT contract_address, COUNT(*) AS event_count, - MAX(_inserted_timestamp) AS max_inserted_timestamp_logs, + MAX(modified_timestamp) AS max_inserted_timestamp_logs, MAX(block_number) AS latest_event_block FROM - {{ ref('silver__logs') }} + {{ ref('core__fact_event_logs') }} {% if is_incremental() %} WHERE - _inserted_timestamp > ( + modified_timestamp > ( SELECT MAX(max_inserted_timestamp_logs) FROM @@ -36,8 +36,8 @@ function_calls AS ( FROM {{ ref('core__fact_traces') }} WHERE - tx_status = 'SUCCESS' - AND trace_status = 'SUCCESS' + tx_succeeded + AND trace_succeeded AND to_address IS NOT NULL AND input IS NOT NULL AND input <> '0x' diff --git a/models/silver/core/silver__transfers.sql b/models/silver/core/silver__transfers.sql index dc688df..d98f573 100644 --- a/models/silver/core/silver__transfers.sql +++ b/models/silver/core/silver__transfers.sql @@ -11,7 +11,11 @@ WITH logs AS ( SELECT - _log_id, + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, block_number, tx_hash, block_timestamp, @@ -24,12 +28,12 @@ WITH logs AS ( utils.udf_hex_to_int(SUBSTR(DATA, 3, 64)) AS raw_amount_precise, raw_amount_precise :: FLOAT AS raw_amount, event_index, - _inserted_timestamp + modified_timestamp AS _inserted_timestamp FROM - {{ ref('silver__logs') }} + {{ ref('core__fact_event_logs') }} WHERE topics [0] :: STRING = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' - AND tx_status = 'SUCCESS' + AND tx_succeeded {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/core/silver__tx_count.sql b/models/silver/core/silver__tx_count.sql index 0635342..02fa05d 100644 --- a/models/silver/core/silver__tx_count.sql +++ b/models/silver/core/silver__tx_count.sql @@ -9,14 +9,14 @@ WITH base AS ( SELECT block_number, - MIN(_inserted_timestamp) AS _inserted_timestamp, + MIN(modified_timestamp) AS _inserted_timestamp, COUNT(*) AS tx_count FROM - {{ ref('silver__transactions') }} + {{ ref('core__fact_transactions') }} {% if is_incremental() %} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT MAX(_inserted_timestamp) _inserted_timestamp FROM diff --git a/models/silver/protocols/olas/silver_olas__create_service_multisigs.sql b/models/silver/protocols/olas/silver_olas__create_service_multisigs.sql index dc66076..146fb2e 100644 --- a/models/silver/protocols/olas/silver_olas__create_service_multisigs.sql +++ b/models/silver/protocols/olas/silver_olas__create_service_multisigs.sql @@ -15,10 +15,10 @@ SELECT origin_to_address, contract_address, event_index, - topics [0] :: STRING AS topic_0, - topics [1] :: STRING AS topic_1, - topics [2] :: STRING AS topic_2, - topics [3] :: STRING AS topic_3, + topic_0, + topic_1, + topic_2, + topic_3, 'CreateMultisigWithAgents' AS event_name, DATA, regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, @@ -28,8 +28,12 @@ SELECT ) ) AS id, CONCAT('0x', SUBSTR(topic_2, 27, 40)) AS multisig_address, - _log_id, - _inserted_timestamp, + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp AS _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( ['tx_hash','event_index'] ) }} AS create_service_multisigs_id, @@ -37,11 +41,11 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - {{ ref('silver__logs') }} + {{ ref('core__fact_event_logs') }} WHERE contract_address = '0x3c1ff68f5aa342d296d4dee4bb1cacca912d95fe' --Service Registry (AUTONOLAS-SERVICE-V1) AND topic_0 = '0x2d53f895cd5faf3cddba94a25c2ced2105885b5b37450ff430ffa3cbdf332c74' --CreateMultisigWithAgents - AND tx_status = 'SUCCESS' + AND tx_succeeded {% if is_incremental() %} AND _inserted_timestamp >= ( diff --git a/models/silver/protocols/olas/silver_olas__service_event_logs.sql b/models/silver/protocols/olas/silver_olas__service_event_logs.sql index 23102e7..24c2751 100644 --- a/models/silver/protocols/olas/silver_olas__service_event_logs.sql +++ b/models/silver/protocols/olas/silver_olas__service_event_logs.sql @@ -25,14 +25,18 @@ SELECT d.event_index, s.multisig_address, s.id AS service_id, - d.topics [0] :: STRING AS topic_0, - d.topics [1] :: STRING AS topic_1, - d.topics [2] :: STRING AS topic_2, - d.topics [3] :: STRING AS topic_3, + d.topic_0, + d.topic_1, + d.topic_2, + d.topic_3, d.data, regexp_substr_all(SUBSTR(d.data, 3, len(d.data)), '.{64}') AS segmented_data, - d._log_id, - d._inserted_timestamp, + CONCAT( + d.tx_hash :: STRING, + '-', + d.event_index :: STRING + ) AS _log_id, + d.modified_timestamp AS _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( ['d.tx_hash','d.event_index'] ) }} AS service_event_logs_id, @@ -40,15 +44,15 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - {{ ref('silver__logs') }} + {{ ref('core__fact_event_logs') }} d INNER JOIN service_multisigs s ON d.origin_to_address = s.multisig_address WHERE - d.tx_status = 'SUCCESS' + d.tx_succeeded {% if is_incremental() %} -AND d._inserted_timestamp >= ( +AND d.modified_timestamp >= ( SELECT MAX(_inserted_timestamp) - INTERVAL '12 hours' FROM diff --git a/models/silver/protocols/olas/silver_olas__service_registrations.sql b/models/silver/protocols/olas/silver_olas__service_registrations.sql index bd01f75..74f285e 100644 --- a/models/silver/protocols/olas/silver_olas__service_registrations.sql +++ b/models/silver/protocols/olas/silver_olas__service_registrations.sql @@ -17,20 +17,24 @@ WITH registry_evt AS ( origin_to_address, contract_address, event_index, - topics [0] :: STRING AS topic_0, - topics [1] :: STRING AS topic_1, - topics [2] :: STRING AS topic_2, - topics [3] :: STRING AS topic_3, + topic_0, + topic_1, + topic_2, + topic_3, CASE WHEN topic_0 = '0xb34c1e02384201736eb4693b9b173306cb41bff12f15894dea5773088e9a3b1c' THEN 'CreateService' WHEN topic_0 = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' THEN 'Transfer' END AS event_name, DATA, regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp AS _inserted_timestamp FROM - {{ ref('silver__logs') }} + {{ ref('core__fact_event_logs') }} WHERE contract_address = '0x3c1ff68f5aa342d296d4dee4bb1cacca912d95fe' --Service Registry (AUTONOLAS-SERVICE-V1) AND topic_0 IN ( @@ -38,7 +42,7 @@ WITH registry_evt AS ( --CreateService (for services) '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' --Transfer ) - AND tx_status = 'SUCCESS' + AND tx_succeeded {% if is_incremental() %} AND _inserted_timestamp >= ( @@ -142,7 +146,8 @@ heal_model AS ( t0.service_id, t0.config_hash, t0.owner_address, - m.multisig_address, --fill late-arriving or replace with current multisig + m.multisig_address, + --fill late-arriving or replace with current multisig t0._log_id, t0._inserted_timestamp FROM