diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index 16457ad..7f7c42a 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -58,6 +58,16 @@ There is more information on how to use dbt docs in the last section of this doc - [ez_core_metrics_hourly](#!/model/model.flow_models.stats__ez_core_metrics_hourly) +### EVM Tables (`flow.core_evm`) + +- [core_evm__dim_contracts](#!/model/model.flow_models.core_evm__dim_contracts) +- [core_evm__fact_event_logs](#!/model/model.flow_models.core_evm__fact_event_logs) +- [core_evm__fact_traces](#!/model/model.flow_models.core_evm__fact_traces) +- [core_evm__fact_blocks](#!/model/model.flow_models.core_evm__fact_blocks) +- [core_evm__fact_transactions](#!/model/model.flow_models.core_evm__fact_transactions) +- [core_evm__ez_native_transfers](#!/model/model.flow_models.core_evm__ez_native_transfers) +- [core_evm__ez_token_transfers](#!/model/model.flow_models.core_evm__ez_token_transfers) + ## **Data Model Overview** The FLOW diff --git a/models/evm/gold/core/core_evm__ez_native_transfers.sql b/models/evm/gold/core/core_evm__ez_native_transfers.sql new file mode 100644 index 0000000..b2616c1 --- /dev/null +++ b/models/evm/gold/core/core_evm__ez_native_transfers.sql @@ -0,0 +1,122 @@ +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = ['block_timestamp::DATE'], + tags = ['evm'] +) }} + +WITH base AS ( + + SELECT + block_number, + block_timestamp, + tx_hash, + TYPE, + trace_address, + origin_from_address, + origin_to_address, + origin_function_signature, + from_address, + to_address, + VALUE AS amount, + value_precise_raw AS amount_precise_raw, + value_precise AS amount_precise, + ROUND( + VALUE * price, + 2 + ) AS amount_usd, + tx_position, + trace_index, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'trace_index'] + ) }} AS ez_native_transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp + FROM + {{ ref('core_evm__fact_traces') }} + tr + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + ON DATE_TRUNC( + 'hour', + block_timestamp + ) = hour + AND is_native + WHERE + tr.value > 0 + AND tr.tx_succeeded + AND tr.trace_succeeded + AND tr.type NOT IN ( + 'DELEGATECALL', + 'STATICCALL' + ) + +{% if is_incremental() %} +AND tr.modified_timestamp > ( + SELECT + COALESCE(MAX(modified_timestamp), '1970-01-01' :: TIMESTAMP) AS modified_timestamp + FROM + {{ this }} +) +{% endif %} +) +SELECT + block_number, + block_timestamp, + tx_hash, + tx_position, + trace_index, + trace_address, + TYPE, + from_address, + to_address, + amount, + amount_precise_raw, + amount_precise, + amount_usd, + origin_from_address, + origin_to_address, + origin_function_signature, + ez_native_transfers_id, + inserted_timestamp, + modified_timestamp +FROM + base + +{% if is_incremental() %} +UNION ALL +SELECT + t.block_number, + t.block_timestamp, + t.tx_hash, + t.tx_position, + t.trace_index, + t.trace_address, + t.type, + t.from_address, + t.to_address, + t.amount, + t.amount_precise_raw, + t.amount_precise, + t.amount * p.price AS amount_usd_heal, + t.origin_from_address, + t.origin_to_address, + t.origin_function_signature, + t.ez_native_transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ this }} + t + INNER JOIN {{ ref('price__ez_prices_hourly') }} p + ON DATE_TRUNC( + 'hour', + block_timestamp + ) = hour + AND p.is_native + LEFT JOIN base b USING (ez_native_transfers_id) +WHERE + t.amount_usd IS NULL + AND t.block_timestamp :: DATE >= '2024-01-01' + AND b.ez_native_transfers_id IS NULL +{% endif %} diff --git a/models/evm/gold/core/core_evm__ez_native_transfers.yml b/models/evm/gold/core/core_evm__ez_native_transfers.yml new file mode 100644 index 0000000..1d5713f --- /dev/null +++ b/models/evm/gold/core/core_evm__ez_native_transfers.yml @@ -0,0 +1,44 @@ +version: 2 +models: + - name: core_evm__ez_native_transfers + description: '{{ doc("evm_ez_native_transfers_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("evm_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("evm_block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("evm_tx_hash") }}' + - name: TX_POSITION + description: '{{ doc("evm_tx_position") }}' + - name: TRACE_INDEX + description: '{{ doc("evm_trace_index") }}' + - name: TRACE_ADDRESS + description: '{{ doc("evm_trace_address") }}' + - name: TYPE + description: '{{ doc("evm_traces_type") }}' + - name: FROM_ADDRESS + description: '{{ doc("evm_transfer_from_address") }}' + - name: TO_ADDRESS + description: '{{ doc("evm_transfer_to_address") }}' + - name: AMOUNT + description: '{{ doc("evm_transfer_amount") }}' + - name: AMOUNT_PRECISE_RAW + description: '{{ doc("evm_transfer_raw_amount_precise") }}' + - name: AMOUNT_PRECISE + description: '{{ doc("evm_transfer_amount_precise") }}' + - name: AMOUNT_USD + description: '{{ doc("evm_amount_usd") }}' + - name: ORIGIN_FROM_ADDRESS + description: '{{ doc("evm_origin_from") }}' + - name: ORIGIN_TO_ADDRESS + description: '{{ doc("evm_origin_to") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("evm_tx_origin_sig") }}' + - name: EZ_NATIVE_TRANSFERS_ID + description: '{{ doc("evm_pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("evm_inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("evm_modified_timestamp") }}' \ No newline at end of file diff --git a/models/evm/gold/core/core_evm__ez_token_transfers.sql b/models/evm/gold/core/core_evm__ez_token_transfers.sql new file mode 100644 index 0000000..950fa35 --- /dev/null +++ b/models/evm/gold/core/core_evm__ez_token_transfers.sql @@ -0,0 +1,221 @@ +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = ['block_timestamp::DATE'], + tags = ['evm'] +) }} + +WITH base AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + tx_position, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + contract_address, + CONCAT('0x', SUBSTR(topic_1, 27, 40)) :: STRING AS from_address, + CONCAT('0x', SUBSTR(topic_2, 27, 40)) :: STRING AS to_address, + utils.udf_hex_to_int(SUBSTR(DATA, 3, 64)) AS raw_amount_precise, + raw_amount_precise :: FLOAT AS raw_amount, + IFF( + C.decimals IS NULL, + NULL, + utils.udf_decimal_adjust( + raw_amount_precise, + C.decimals + ) + ) AS amount_precise, + amount_precise :: FLOAT AS amount, + C.decimals, + C.symbol, + C.name, + IFF( + topic_0 = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', + 'erc20', + NULL + ) AS token_standard, + fact_event_logs_id AS ez_token_transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp + FROM + {{ ref('core_evm__fact_event_logs') }} + f + LEFT JOIN {{ ref('core_evm__dim_contracts') }} C + ON contract_address = C.address + AND ( + C.decimals IS NOT NULL + OR C.symbol IS NOT NULL + OR C.name IS NOT NULL + ) + WHERE + topic_0 = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' + AND tx_succeeded + AND NOT event_removed + AND topic_1 IS NOT NULL + AND topic_2 IS NOT NULL + AND DATA IS NOT NULL + AND raw_amount IS NOT NULL + +{% if is_incremental() %} +AND f.modified_timestamp > ( + SELECT + COALESCE(MAX(modified_timestamp), '1970-01-01' :: TIMESTAMP) AS modified_timestamp + FROM + {{ this }} +) +{% endif %} +) +SELECT + block_number, + block_timestamp, + tx_hash, + tx_position, + event_index, + from_address, + to_address, + contract_address, + token_standard, + NAME, + symbol, + decimals, + raw_amount_precise, + raw_amount, + amount_precise, + amount, + origin_function_signature, + origin_from_address, + origin_to_address, + ez_token_transfers_id, + inserted_timestamp, + modified_timestamp +FROM + base + +{% if is_incremental() %} +UNION ALL +SELECT + t0.block_number, + t0.block_timestamp, + t0.tx_hash, + t0.tx_position, + t0.event_index, + t0.from_address, + t0.to_address, + t0.contract_address, + t0.token_standard, + c0.name, + c0.symbol, + c0.decimals, + t0.raw_amount_precise, + t0.raw_amount, + IFF( + c0.decimals IS NULL, + NULL, + utils.udf_decimal_adjust( + t0.raw_amount_precise, + c0.decimals + ) + ) AS amount_precise, + amount_precise :: FLOAT AS amount, + t0.origin_function_signature, + t0.origin_from_address, + t0.origin_to_address, + t0.ez_token_transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ this }} + t0 + LEFT JOIN {{ ref('core_evm__dim_contracts') }} + c0 + ON t0.contract_address = c0.address + AND ( + c0.decimals IS NOT NULL + OR c0.symbol IS NOT NULL + OR c0.name IS NOT NULL + ) + LEFT JOIN base b USING (ez_token_transfers_id) +WHERE + b.ez_token_transfers_id IS NULL + AND ( + t0.block_number IN ( + SELECT + DISTINCT t1.block_number + FROM + {{ this }} + t1 + WHERE + t1.decimals IS NULL + AND t1.modified_timestamp <= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('core_evm__dim_contracts') }} + c1 + WHERE + c1.modified_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND c1.decimals IS NOT NULL + AND t1.contract_address = c1.address) + ) -- Only heal decimals if new data exists + OR t0.block_number IN ( + SELECT + DISTINCT t2.block_number + FROM + {{ this }} + t2 + WHERE + t2.symbol IS NULL + AND t2.modified_timestamp <= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('core_evm__dim_contracts') }} + c2 + WHERE + c2.modified_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND c2.symbol IS NOT NULL + AND t2.contract_address = c2.address) + ) -- Only heal symbol if new data exists + OR t0.block_number IN ( + SELECT + DISTINCT t3.block_number + FROM + {{ this }} + t3 + WHERE + t3.name IS NULL + AND t3.modified_timestamp <= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + AND EXISTS ( + SELECT + 1 + FROM + {{ ref('core_evm__dim_contracts') }} + c3 + WHERE + c3.modified_timestamp > DATEADD('DAY', -14, SYSDATE()) + AND c3.name IS NOT NULL + AND t3.contract_address = c3.address) + ) -- Only heal name if new data exists + ) +{% endif %} diff --git a/models/evm/gold/core/core_evm__ez_token_transfers.yml b/models/evm/gold/core/core_evm__ez_token_transfers.yml new file mode 100644 index 0000000..9590a46 --- /dev/null +++ b/models/evm/gold/core/core_evm__ez_token_transfers.yml @@ -0,0 +1,52 @@ +version: 2 +models: + - name: core_evm__ez_token_transfers + description: '{{ doc("evm_ez_token_transfers_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("evm_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("evm_block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("evm_tx_hash") }}' + - name: TX_POSITION + description: '{{ doc("evm_tx_position") }}' + - name: EVENT_INDEX + description: '{{ doc("evm_event_index") }}' + - name: FROM_ADDRESS + description: '{{ doc("evm_transfer_from_address") }}' + - name: TO_ADDRESS + description: '{{ doc("evm_transfer_to_address") }}' + - name: CONTRACT_ADDRESS + description: '{{ doc("evm_contract_address") }}' + - name: TOKEN_STANDARD + description: '{{ doc("evm_token_standard") }}' + - name: NAME + description: '{{ doc("evm_contracts_name") }}' + - name: SYMBOL + description: '{{ doc("evm_contracts_symbol") }}' + - name: DECIMALS + description: '{{ doc("evm_decimals") }}' + - name: RAW_AMOUNT_PRECISE + description: '{{ doc("evm_transfer_raw_amount_precise") }}' + - name: RAW_AMOUNT + description: '{{ doc("evm_transfer_raw_amount") }}' + - name: AMOUNT_PRECISE + description: '{{ doc("evm_transfer_amount_precise") }}' + - name: AMOUNT + description: '{{ doc("evm_transfer_amount") }}' + - name: AMOUNT_USD + description: '{{ doc("evm_transfer_amount_usd") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("evm_tx_origin_sig") }}' + - name: ORIGIN_FROM_ADDRESS + description: '{{ doc("evm_origin_from") }}' + - name: ORIGIN_TO_ADDRESS + description: '{{ doc("evm_origin_to") }}' + - name: EZ_TOKEN_TRANSFERS_ID + description: '{{ doc("evm_pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("evm_inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("evm_modified_timestamp") }}' \ No newline at end of file