From f34f80e87a7854d85e3168765d91e40814f97c83 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Thu, 19 Sep 2024 09:27:27 -0600 Subject: [PATCH] first commit, cloned from m1 with updated resources --- README.md | 75 +++ analyses/.gitkeep | 0 data/.gitkeep | 0 data/github_actions__workflows.csv | 6 + dbt_project.yml | 103 ++++ docs/.gitkeep | 0 macros/.gitkeep | 0 macros/create_sps.sql | 6 + macros/create_udfs.sql | 10 + macros/custom_naming_macros.sql | 23 + macros/custom_query_tag.sql | 11 + macros/dbt/get_merge_sql.sql | 16 + macros/dbt/get_tmp_relation_type.sql | 8 + macros/run_sp_create_prod_clone.sql | 10 + macros/sp_create_prod_clone.sql | 44 ++ macros/streamline/api_integrations.sql | 21 + macros/streamline/models.sql | 70 +++ macros/streamline/streamline_udfs.sql | 10 + macros/tags/add_database_or_schema_tags.sql | 6 + macros/tags/snowflake_tagging.sql | 127 ++++ macros/tests/compare_model_subset.sql | 29 + macros/tests/sequence_gaps.sql | 37 ++ macros/tests/tx_gaps.sql | 33 ++ macros/utils.sql | 42 ++ models/bronze/core/bronze__blocks_tx.sql | 7 + models/bronze/core/bronze__blocks_tx_FR.sql | 7 + models/bronze/core/bronze__transactions.sql | 7 + .../bronze/core/bronze__transactions_FR.sql | 7 + .../prices/bronze__complete_native_prices.sql | 29 + models/descriptions/__overview__.md | 70 +++ models/descriptions/accumulator_root_hash.md | 5 + models/descriptions/address.md | 5 + models/descriptions/address_change.md | 5 + models/descriptions/address_event.md | 5 + models/descriptions/amount.md | 5 + models/descriptions/block_hash.md | 5 + models/descriptions/block_number.md | 5 + models/descriptions/block_timestamp.md | 5 + models/descriptions/blockchain.md | 5 + models/descriptions/change_address.md | 5 + models/descriptions/change_data.md | 5 + models/descriptions/change_index.md | 5 + models/descriptions/change_module.md | 5 + models/descriptions/change_resource.md | 5 + models/descriptions/change_type.md | 5 + models/descriptions/changes.md | 5 + models/descriptions/creation_number.md | 5 + models/descriptions/creator.md | 5 + .../defi__fact_bridge_activity.md | 55 ++ models/descriptions/epoch.md | 5 + models/descriptions/event_address.md | 5 + models/descriptions/event_data.md | 5 + models/descriptions/event_index.md | 5 + models/descriptions/event_module.md | 5 + models/descriptions/event_resource.md | 5 + models/descriptions/event_root_hash.md | 5 + models/descriptions/event_type.md | 5 + models/descriptions/events.md | 5 + .../descriptions/expiration_timestamp_secs.md | 5 + models/descriptions/first_version.md | 5 + models/descriptions/from_address_transfer.md | 5 + models/descriptions/gas_unit_price.md | 5 + models/descriptions/gas_used.md | 5 + models/descriptions/handle_change.md | 5 + models/descriptions/inner_change_type.md | 5 + models/descriptions/inserted_timestamp.md | 5 + models/descriptions/key_change.md | 5 + models/descriptions/label.md | 5 + models/descriptions/label_subtype.md | 5 + models/descriptions/label_type.md | 5 + models/descriptions/last_version.md | 5 + models/descriptions/max_gas_amount.md | 5 + models/descriptions/modified_timestamp.md | 5 + models/descriptions/payload.md | 5 + models/descriptions/payload_function.md | 5 + models/descriptions/pk.md | 5 + models/descriptions/proposer.md | 5 + models/descriptions/round.md | 5 + models/descriptions/sender.md | 5 + models/descriptions/sequence_number.md | 5 + models/descriptions/signature.md | 5 + models/descriptions/stats/stats_core.md | 71 +++ models/descriptions/success.md | 5 + .../descriptions/tables/core__fact_blocks.md | 8 + .../descriptions/tables/core__fact_changes.md | 5 + .../descriptions/tables/core__fact_events.md | 5 + .../tables/core__fact_transactions.md | 6 + .../core__fact_transactions_block_metadata.md | 5 + ...ore__fact_transactions_state_checkpoint.md | 5 + models/descriptions/to_address_transfer.md | 5 + models/descriptions/token_address.md | 5 + models/descriptions/transfer_event.md | 5 + models/descriptions/tx_count.md | 5 + models/descriptions/tx_hash.md | 5 + models/descriptions/tx_type.md | 5 + models/descriptions/value_change.md | 5 + models/descriptions/version.md | 5 + models/descriptions/vm_status.md | 5 + .../api_udf/bronze_evm_api__contract_abis.sql | 53 ++ .../api_udf/bronze_evm_api__contract_abis.yml | 22 + .../api_udf/bronze_evm_api__token_reads.sql | 126 ++++ .../api_udf/bronze_evm_api__token_reads.yml | 18 + .../silver_evm__complete_event_abis.sql | 216 +++++++ .../silver_evm__complete_event_abis.yml | 9 + .../silver_evm__flat_event_abis.sql | 112 ++++ models/evm/silver/abis/silver_evm__abis.sql | 181 ++++++ models/evm/silver/abis/silver_evm__abis.yml | 8 + .../silver/abis/silver_evm__bytecode_abis.sql | 76 +++ .../silver/abis/silver_evm__bytecode_abis.yml | 8 + .../silver/abis/silver_evm__override_abis.sql | 8 + .../evm/silver/abis/silver_evm__proxies.sql | 104 ++++ .../evm/silver/abis/silver_evm__proxies.yml | 7 + .../abis/silver_evm__user_verified_abis.sql | 550 ++++++++++++++++++ .../abis/silver_evm__user_verified_abis.yml | 7 + .../silver/abis/silver_evm__verified_abis.sql | 108 ++++ .../silver/abis/silver_evm__verified_abis.yml | 7 + models/evm/silver/core/silver_evm__blocks.sql | 81 +++ .../core/silver_evm__confirmed_blocks.sql | 55 ++ .../evm/silver/core/silver_evm__contracts.sql | 103 ++++ .../evm/silver/core/silver_evm__contracts.yml | 24 + .../core/silver_evm__created_contracts.sql | 44 ++ .../core/silver_evm__created_contracts.yml | 29 + models/evm/silver/core/silver_evm__logs.sql | 188 ++++++ .../core/silver_evm__native_transfers.sql | 110 ++++ .../evm/silver/core/silver_evm__receipts.sql | 105 ++++ .../core/silver_evm__relevant_contracts.sql | 145 +++++ .../core/silver_evm__relevant_contracts.yml | 7 + models/evm/silver/core/silver_evm__traces.sql | 429 ++++++++++++++ .../silver/core/silver_evm__transactions.sql | 395 +++++++++++++ .../evm/silver/core/silver_evm__transfers.sql | 110 ++++ .../blocks/test_silver_evm__blocks_full.sql | 11 + .../blocks/test_silver_evm__blocks_full.yml | 95 +++ .../blocks/test_silver_evm__blocks_recent.sql | 24 + .../blocks/test_silver_evm__blocks_recent.yml | 27 + ...test_silver_evm__confirmed_blocks_full.sql | 9 + ...test_silver_evm__confirmed_blocks_full.yml | 34 ++ ...st_silver_evm__confirmed_blocks_recent.sql | 23 + ...st_silver_evm__confirmed_blocks_recent.yml | 34 ++ .../event_logs/test_silver_evm__logs_full.sql | 11 + .../event_logs/test_silver_evm__logs_full.yml | 77 +++ .../test_silver_evm__logs_recent.sql | 24 + .../test_silver_evm__logs_recent.yml | 33 ++ .../test_silver_evm__receipts_full.sql | 9 + .../test_silver_evm__receipts_full.yml | 82 +++ .../test_silver_evm__receipts_recent.sql | 23 + .../test_silver_evm__receipts_recent.yml | 28 + .../traces/test_silver_evm__traces_full.sql | 11 + .../traces/test_silver_evm__traces_full.yml | 59 ++ .../traces/test_silver_evm__traces_recent.sql | 24 + .../traces/test_silver_evm__traces_recent.yml | 35 ++ .../test_silver_evm__transactions_full.sql | 11 + .../test_silver_evm__transactions_full.yml | 114 ++++ .../test_silver_evm__transactions_recent.sql | 24 + .../test_silver_evm__transactions_recent.yml | 18 + models/evm/streamline/_evm_block_lookback.sql | 11 + .../evm/streamline/_max_evm_block_by_date.sql | 27 + .../evm/streamline/_max_evm_block_by_hour.sql | 37 ++ .../core/bronze_evm__streamline_FR_blocks.sql | 7 + ...onze_evm__streamline_FR_confirm_blocks.sql | 7 + .../bronze_evm__streamline_FR_receipts.sql | 7 + .../core/bronze_evm__streamline_FR_traces.sql | 7 + ...bronze_evm__streamline_FR_transactions.sql | 7 + .../core/bronze_evm__streamline_blocks.sql | 7 + .../bronze_evm__streamline_confirm_blocks.sql | 7 + .../core/bronze_evm__streamline_receipts.sql | 7 + .../core/bronze_evm__streamline_traces.sql | 7 + .../bronze_evm__streamline_transactions.sql | 7 + .../decoder/bronze_evm__decoded_logs.sql | 41 ++ .../decoder/bronze_evm__fr_decoded_logs.sql | 40 ++ .../streamline__complete_evm_blocks.sql | 34 ++ ...eamline__complete_evm_confirmed_blocks.sql | 34 ++ .../streamline__complete_evm_receipts.sql | 34 ++ .../streamline__complete_evm_traces.sql | 34 ++ .../streamline__complete_evm_transactions.sql | 34 ++ .../streamline__evm_blocks_history.sql | 85 +++ ...streamline__evm_confirm_blocks_history.sql | 94 +++ .../streamline__evm_receipts_history.sql | 86 +++ .../streamline__evm_traces_history.sql | 87 +++ .../streamline__evm_transactions_history.sql | 86 +++ .../streamline__evm_blocks_realtime.sql | 64 ++ ...treamline__evm_confirm_blocks_realtime.sql | 64 ++ .../streamline__evm_receipts_realtime.sql | 65 +++ .../streamline__evm_traces_realtime.sql | 66 +++ .../streamline__evm_transactions_realtime.sql | 65 +++ .../silver/core/retry/_missing_receipts.sql | 34 ++ .../silver/core/retry/_missing_traces.sql | 34 ++ .../silver/core/retry/_unconfirmed_blocks.sql | 34 ++ .../streamline__complete_decode_logs.sql | 31 + .../silver/decoder/streamline__evm_blocks.sql | 26 + .../github_actions__current_task_status.sql | 6 + .../github_actions__current_task_status.yml | 17 + .../github_actions__task_history.sql | 5 + .../github_actions__task_performance.sql | 5 + .../github_actions__task_schedule.sql | 5 + .../github_actions/github_actions__tasks.sql | 5 + models/gold/core/core__fact_blocks.sql | 33 ++ models/gold/core/core__fact_blocks.yml | 26 + models/gold/core/core__fact_changes.sql | 48 ++ models/gold/core/core__fact_changes.yml | 49 ++ models/gold/core/core__fact_events.sql | 45 ++ models/gold/core/core__fact_events.yml | 44 ++ models/gold/core/core__fact_transactions.sql | 61 ++ models/gold/core/core__fact_transactions.yml | 51 ++ ...core__fact_transactions_block_metadata.sql | 63 ++ ...core__fact_transactions_block_metadata.yml | 44 ++ ...re__fact_transactions_state_checkpoint.sql | 43 ++ ...re__fact_transactions_state_checkpoint.yml | 31 + .../tests/blocks/test_core__blocks_full.sql | 11 + .../tests/blocks/test_core__blocks_full.yml | 66 +++ .../tests/blocks/test_core__blocks_recent.sql | 27 + .../tests/blocks/test_core__blocks_recent.yml | 21 + .../test_core__transactions_full.sql | 11 + .../test_core__transactions_full.yml | 186 ++++++ .../test_core__transactions_recent.sql | 27 + .../test_core__transactions_recent.yml | 29 + .../stats/stats__ez_core_metrics_hourly.sql | 27 + .../stats/stats__ez_core_metrics_hourly.yml | 34 ++ ...ver_observability__blocks_completeness.sql | 168 ++++++ ...ver_observability__blocks_completeness.yml | 67 +++ ...servability__transactions_completeness.sql | 137 +++++ ...servability__transactions_completeness.yml | 69 +++ models/silver/core/silver__blocks.sql | 54 ++ models/silver/core/silver__blocks.yml | 39 ++ models/silver/core/silver__changes.sql | 43 ++ models/silver/core/silver__changes.yml | 26 + models/silver/core/silver__events.sql | 55 ++ models/silver/core/silver__events.yml | 23 + models/silver/core/silver__transactions.sql | 112 ++++ models/silver/core/silver__transactions.yml | 37 ++ .../tests/blocks/test_silver__blocks_full.sql | 11 + .../tests/blocks/test_silver__blocks_full.yml | 84 +++ .../blocks/test_silver__blocks_recent.sql | 27 + .../blocks/test_silver__blocks_recent.yml | 21 + .../test_silver__transactions_full.sql | 11 + .../test_silver__transactions_full.yml | 58 ++ .../test_silver__transactions_recent.sql | 27 + .../test_silver__transactions_recent.yml | 26 + .../price/silver__complete_native_prices.sql | 40 ++ .../price/silver__complete_native_prices.yml | 37 ++ .../stats/silver__core_metrics_hourly.sql | 83 +++ .../stats/silver__core_metrics_hourly.yml | 70 +++ models/sources.yml | 45 ++ .../streamline__blocks_tx_complete.sql | 41 ++ .../streamline__transactions_complete.sql | 46 ++ .../streamline/silver/_max_block_by_date.sql | 27 + .../streamline__blocks_tx_realtime.sql | 45 ++ .../streamline__transactions_realtime.sql | 94 +++ .../streamline/silver/streamline__blocks.sql | 22 + .../silver/streamline__chainhead.sql | 20 + package-lock.yml | 16 + packages.yml | 11 + profiles.yml | 15 + requirements.txt | 2 + snapshots/.gitkeep | 0 254 files changed, 9605 insertions(+) create mode 100644 README.md create mode 100644 analyses/.gitkeep create mode 100644 data/.gitkeep create mode 100644 data/github_actions__workflows.csv create mode 100644 dbt_project.yml create mode 100644 docs/.gitkeep create mode 100644 macros/.gitkeep create mode 100644 macros/create_sps.sql create mode 100644 macros/create_udfs.sql create mode 100644 macros/custom_naming_macros.sql create mode 100644 macros/custom_query_tag.sql create mode 100644 macros/dbt/get_merge_sql.sql create mode 100644 macros/dbt/get_tmp_relation_type.sql create mode 100644 macros/run_sp_create_prod_clone.sql create mode 100644 macros/sp_create_prod_clone.sql create mode 100644 macros/streamline/api_integrations.sql create mode 100644 macros/streamline/models.sql create mode 100644 macros/streamline/streamline_udfs.sql create mode 100644 macros/tags/add_database_or_schema_tags.sql create mode 100644 macros/tags/snowflake_tagging.sql create mode 100644 macros/tests/compare_model_subset.sql create mode 100644 macros/tests/sequence_gaps.sql create mode 100644 macros/tests/tx_gaps.sql create mode 100644 macros/utils.sql create mode 100644 models/bronze/core/bronze__blocks_tx.sql create mode 100644 models/bronze/core/bronze__blocks_tx_FR.sql create mode 100644 models/bronze/core/bronze__transactions.sql create mode 100644 models/bronze/core/bronze__transactions_FR.sql create mode 100644 models/bronze/prices/bronze__complete_native_prices.sql create mode 100644 models/descriptions/__overview__.md create mode 100644 models/descriptions/accumulator_root_hash.md create mode 100644 models/descriptions/address.md create mode 100644 models/descriptions/address_change.md create mode 100644 models/descriptions/address_event.md create mode 100644 models/descriptions/amount.md create mode 100644 models/descriptions/block_hash.md create mode 100644 models/descriptions/block_number.md create mode 100644 models/descriptions/block_timestamp.md create mode 100644 models/descriptions/blockchain.md create mode 100644 models/descriptions/change_address.md create mode 100644 models/descriptions/change_data.md create mode 100644 models/descriptions/change_index.md create mode 100644 models/descriptions/change_module.md create mode 100644 models/descriptions/change_resource.md create mode 100644 models/descriptions/change_type.md create mode 100644 models/descriptions/changes.md create mode 100644 models/descriptions/creation_number.md create mode 100644 models/descriptions/creator.md create mode 100644 models/descriptions/defi__fact_bridge_activity.md create mode 100644 models/descriptions/epoch.md create mode 100644 models/descriptions/event_address.md create mode 100644 models/descriptions/event_data.md create mode 100644 models/descriptions/event_index.md create mode 100644 models/descriptions/event_module.md create mode 100644 models/descriptions/event_resource.md create mode 100644 models/descriptions/event_root_hash.md create mode 100644 models/descriptions/event_type.md create mode 100644 models/descriptions/events.md create mode 100644 models/descriptions/expiration_timestamp_secs.md create mode 100644 models/descriptions/first_version.md create mode 100644 models/descriptions/from_address_transfer.md create mode 100644 models/descriptions/gas_unit_price.md create mode 100644 models/descriptions/gas_used.md create mode 100644 models/descriptions/handle_change.md create mode 100644 models/descriptions/inner_change_type.md create mode 100644 models/descriptions/inserted_timestamp.md create mode 100644 models/descriptions/key_change.md create mode 100644 models/descriptions/label.md create mode 100644 models/descriptions/label_subtype.md create mode 100644 models/descriptions/label_type.md create mode 100644 models/descriptions/last_version.md create mode 100644 models/descriptions/max_gas_amount.md create mode 100644 models/descriptions/modified_timestamp.md create mode 100644 models/descriptions/payload.md create mode 100644 models/descriptions/payload_function.md create mode 100644 models/descriptions/pk.md create mode 100644 models/descriptions/proposer.md create mode 100644 models/descriptions/round.md create mode 100644 models/descriptions/sender.md create mode 100644 models/descriptions/sequence_number.md create mode 100644 models/descriptions/signature.md create mode 100644 models/descriptions/stats/stats_core.md create mode 100644 models/descriptions/success.md create mode 100644 models/descriptions/tables/core__fact_blocks.md create mode 100644 models/descriptions/tables/core__fact_changes.md create mode 100644 models/descriptions/tables/core__fact_events.md create mode 100644 models/descriptions/tables/core__fact_transactions.md create mode 100644 models/descriptions/tables/core__fact_transactions_block_metadata.md create mode 100644 models/descriptions/tables/core__fact_transactions_state_checkpoint.md create mode 100644 models/descriptions/to_address_transfer.md create mode 100644 models/descriptions/token_address.md create mode 100644 models/descriptions/transfer_event.md create mode 100644 models/descriptions/tx_count.md create mode 100644 models/descriptions/tx_hash.md create mode 100644 models/descriptions/tx_type.md create mode 100644 models/descriptions/value_change.md create mode 100644 models/descriptions/version.md create mode 100644 models/descriptions/vm_status.md create mode 100644 models/evm/bronze/api_udf/bronze_evm_api__contract_abis.sql create mode 100644 models/evm/bronze/api_udf/bronze_evm_api__contract_abis.yml create mode 100644 models/evm/bronze/api_udf/bronze_evm_api__token_reads.sql create mode 100644 models/evm/bronze/api_udf/bronze_evm_api__token_reads.yml create mode 100644 models/evm/silver/abis/event_logs/silver_evm__complete_event_abis.sql create mode 100644 models/evm/silver/abis/event_logs/silver_evm__complete_event_abis.yml create mode 100644 models/evm/silver/abis/event_logs/silver_evm__flat_event_abis.sql create mode 100644 models/evm/silver/abis/silver_evm__abis.sql create mode 100644 models/evm/silver/abis/silver_evm__abis.yml create mode 100644 models/evm/silver/abis/silver_evm__bytecode_abis.sql create mode 100644 models/evm/silver/abis/silver_evm__bytecode_abis.yml create mode 100644 models/evm/silver/abis/silver_evm__override_abis.sql create mode 100644 models/evm/silver/abis/silver_evm__proxies.sql create mode 100644 models/evm/silver/abis/silver_evm__proxies.yml create mode 100644 models/evm/silver/abis/silver_evm__user_verified_abis.sql create mode 100644 models/evm/silver/abis/silver_evm__user_verified_abis.yml create mode 100644 models/evm/silver/abis/silver_evm__verified_abis.sql create mode 100644 models/evm/silver/abis/silver_evm__verified_abis.yml create mode 100644 models/evm/silver/core/silver_evm__blocks.sql create mode 100644 models/evm/silver/core/silver_evm__confirmed_blocks.sql create mode 100644 models/evm/silver/core/silver_evm__contracts.sql create mode 100644 models/evm/silver/core/silver_evm__contracts.yml create mode 100644 models/evm/silver/core/silver_evm__created_contracts.sql create mode 100644 models/evm/silver/core/silver_evm__created_contracts.yml create mode 100644 models/evm/silver/core/silver_evm__logs.sql create mode 100644 models/evm/silver/core/silver_evm__native_transfers.sql create mode 100644 models/evm/silver/core/silver_evm__receipts.sql create mode 100644 models/evm/silver/core/silver_evm__relevant_contracts.sql create mode 100644 models/evm/silver/core/silver_evm__relevant_contracts.yml create mode 100644 models/evm/silver/core/silver_evm__traces.sql create mode 100644 models/evm/silver/core/silver_evm__transactions.sql create mode 100644 models/evm/silver/core/silver_evm__transfers.sql create mode 100644 models/evm/silver/core/tests/blocks/test_silver_evm__blocks_full.sql create mode 100644 models/evm/silver/core/tests/blocks/test_silver_evm__blocks_full.yml create mode 100644 models/evm/silver/core/tests/blocks/test_silver_evm__blocks_recent.sql create mode 100644 models/evm/silver/core/tests/blocks/test_silver_evm__blocks_recent.yml create mode 100644 models/evm/silver/core/tests/confirmed_blocks/test_silver_evm__confirmed_blocks_full.sql create mode 100644 models/evm/silver/core/tests/confirmed_blocks/test_silver_evm__confirmed_blocks_full.yml create mode 100644 models/evm/silver/core/tests/confirmed_blocks/test_silver_evm__confirmed_blocks_recent.sql create mode 100644 models/evm/silver/core/tests/confirmed_blocks/test_silver_evm__confirmed_blocks_recent.yml create mode 100644 models/evm/silver/core/tests/event_logs/test_silver_evm__logs_full.sql create mode 100644 models/evm/silver/core/tests/event_logs/test_silver_evm__logs_full.yml create mode 100644 models/evm/silver/core/tests/event_logs/test_silver_evm__logs_recent.sql create mode 100644 models/evm/silver/core/tests/event_logs/test_silver_evm__logs_recent.yml create mode 100644 models/evm/silver/core/tests/receipts/test_silver_evm__receipts_full.sql create mode 100644 models/evm/silver/core/tests/receipts/test_silver_evm__receipts_full.yml create mode 100644 models/evm/silver/core/tests/receipts/test_silver_evm__receipts_recent.sql create mode 100644 models/evm/silver/core/tests/receipts/test_silver_evm__receipts_recent.yml create mode 100644 models/evm/silver/core/tests/traces/test_silver_evm__traces_full.sql create mode 100644 models/evm/silver/core/tests/traces/test_silver_evm__traces_full.yml create mode 100644 models/evm/silver/core/tests/traces/test_silver_evm__traces_recent.sql create mode 100644 models/evm/silver/core/tests/traces/test_silver_evm__traces_recent.yml create mode 100644 models/evm/silver/core/tests/transactions/test_silver_evm__transactions_full.sql create mode 100644 models/evm/silver/core/tests/transactions/test_silver_evm__transactions_full.yml create mode 100644 models/evm/silver/core/tests/transactions/test_silver_evm__transactions_recent.sql create mode 100644 models/evm/silver/core/tests/transactions/test_silver_evm__transactions_recent.yml create mode 100644 models/evm/streamline/_evm_block_lookback.sql create mode 100644 models/evm/streamline/_max_evm_block_by_date.sql create mode 100644 models/evm/streamline/_max_evm_block_by_hour.sql create mode 100644 models/evm/streamline/bronze/core/bronze_evm__streamline_FR_blocks.sql create mode 100644 models/evm/streamline/bronze/core/bronze_evm__streamline_FR_confirm_blocks.sql create mode 100644 models/evm/streamline/bronze/core/bronze_evm__streamline_FR_receipts.sql create mode 100644 models/evm/streamline/bronze/core/bronze_evm__streamline_FR_traces.sql create mode 100644 models/evm/streamline/bronze/core/bronze_evm__streamline_FR_transactions.sql create mode 100644 models/evm/streamline/bronze/core/bronze_evm__streamline_blocks.sql create mode 100644 models/evm/streamline/bronze/core/bronze_evm__streamline_confirm_blocks.sql create mode 100644 models/evm/streamline/bronze/core/bronze_evm__streamline_receipts.sql create mode 100644 models/evm/streamline/bronze/core/bronze_evm__streamline_traces.sql create mode 100644 models/evm/streamline/bronze/core/bronze_evm__streamline_transactions.sql create mode 100644 models/evm/streamline/bronze/decoder/bronze_evm__decoded_logs.sql create mode 100644 models/evm/streamline/bronze/decoder/bronze_evm__fr_decoded_logs.sql create mode 100644 models/evm/streamline/silver/core/complete/streamline__complete_evm_blocks.sql create mode 100644 models/evm/streamline/silver/core/complete/streamline__complete_evm_confirmed_blocks.sql create mode 100644 models/evm/streamline/silver/core/complete/streamline__complete_evm_receipts.sql create mode 100644 models/evm/streamline/silver/core/complete/streamline__complete_evm_traces.sql create mode 100644 models/evm/streamline/silver/core/complete/streamline__complete_evm_transactions.sql create mode 100644 models/evm/streamline/silver/core/history/streamline__evm_blocks_history.sql create mode 100644 models/evm/streamline/silver/core/history/streamline__evm_confirm_blocks_history.sql create mode 100644 models/evm/streamline/silver/core/history/streamline__evm_receipts_history.sql create mode 100644 models/evm/streamline/silver/core/history/streamline__evm_traces_history.sql create mode 100644 models/evm/streamline/silver/core/history/streamline__evm_transactions_history.sql create mode 100644 models/evm/streamline/silver/core/realtime/streamline__evm_blocks_realtime.sql create mode 100644 models/evm/streamline/silver/core/realtime/streamline__evm_confirm_blocks_realtime.sql create mode 100644 models/evm/streamline/silver/core/realtime/streamline__evm_receipts_realtime.sql create mode 100644 models/evm/streamline/silver/core/realtime/streamline__evm_traces_realtime.sql create mode 100644 models/evm/streamline/silver/core/realtime/streamline__evm_transactions_realtime.sql create mode 100644 models/evm/streamline/silver/core/retry/_missing_receipts.sql create mode 100644 models/evm/streamline/silver/core/retry/_missing_traces.sql create mode 100644 models/evm/streamline/silver/core/retry/_unconfirmed_blocks.sql create mode 100644 models/evm/streamline/silver/decoder/complete/streamline__complete_decode_logs.sql create mode 100644 models/evm/streamline/silver/decoder/streamline__evm_blocks.sql create mode 100644 models/github_actions/github_actions__current_task_status.sql create mode 100644 models/github_actions/github_actions__current_task_status.yml create mode 100644 models/github_actions/github_actions__task_history.sql create mode 100644 models/github_actions/github_actions__task_performance.sql create mode 100644 models/github_actions/github_actions__task_schedule.sql create mode 100644 models/github_actions/github_actions__tasks.sql create mode 100644 models/gold/core/core__fact_blocks.sql create mode 100644 models/gold/core/core__fact_blocks.yml create mode 100644 models/gold/core/core__fact_changes.sql create mode 100644 models/gold/core/core__fact_changes.yml create mode 100644 models/gold/core/core__fact_events.sql create mode 100644 models/gold/core/core__fact_events.yml create mode 100644 models/gold/core/core__fact_transactions.sql create mode 100644 models/gold/core/core__fact_transactions.yml create mode 100644 models/gold/core/core__fact_transactions_block_metadata.sql create mode 100644 models/gold/core/core__fact_transactions_block_metadata.yml create mode 100644 models/gold/core/core__fact_transactions_state_checkpoint.sql create mode 100644 models/gold/core/core__fact_transactions_state_checkpoint.yml create mode 100644 models/gold/core/tests/blocks/test_core__blocks_full.sql create mode 100644 models/gold/core/tests/blocks/test_core__blocks_full.yml create mode 100644 models/gold/core/tests/blocks/test_core__blocks_recent.sql create mode 100644 models/gold/core/tests/blocks/test_core__blocks_recent.yml create mode 100644 models/gold/core/tests/transactions/test_core__transactions_full.sql create mode 100644 models/gold/core/tests/transactions/test_core__transactions_full.yml create mode 100644 models/gold/core/tests/transactions/test_core__transactions_recent.sql create mode 100644 models/gold/core/tests/transactions/test_core__transactions_recent.yml create mode 100644 models/gold/stats/stats__ez_core_metrics_hourly.sql create mode 100644 models/gold/stats/stats__ez_core_metrics_hourly.yml create mode 100644 models/silver/_observability/silver_observability__blocks_completeness.sql create mode 100644 models/silver/_observability/silver_observability__blocks_completeness.yml create mode 100644 models/silver/_observability/silver_observability__transactions_completeness.sql create mode 100644 models/silver/_observability/silver_observability__transactions_completeness.yml create mode 100644 models/silver/core/silver__blocks.sql create mode 100644 models/silver/core/silver__blocks.yml create mode 100644 models/silver/core/silver__changes.sql create mode 100644 models/silver/core/silver__changes.yml create mode 100644 models/silver/core/silver__events.sql create mode 100644 models/silver/core/silver__events.yml create mode 100644 models/silver/core/silver__transactions.sql create mode 100644 models/silver/core/silver__transactions.yml create mode 100644 models/silver/core/tests/blocks/test_silver__blocks_full.sql create mode 100644 models/silver/core/tests/blocks/test_silver__blocks_full.yml create mode 100644 models/silver/core/tests/blocks/test_silver__blocks_recent.sql create mode 100644 models/silver/core/tests/blocks/test_silver__blocks_recent.yml create mode 100644 models/silver/core/tests/transactions/test_silver__transactions_full.sql create mode 100644 models/silver/core/tests/transactions/test_silver__transactions_full.yml create mode 100644 models/silver/core/tests/transactions/test_silver__transactions_recent.sql create mode 100644 models/silver/core/tests/transactions/test_silver__transactions_recent.yml create mode 100644 models/silver/price/silver__complete_native_prices.sql create mode 100644 models/silver/price/silver__complete_native_prices.yml create mode 100644 models/silver/stats/silver__core_metrics_hourly.sql create mode 100644 models/silver/stats/silver__core_metrics_hourly.yml create mode 100644 models/sources.yml create mode 100644 models/streamline/complete/streamline__blocks_tx_complete.sql create mode 100644 models/streamline/complete/streamline__transactions_complete.sql create mode 100644 models/streamline/silver/_max_block_by_date.sql create mode 100644 models/streamline/silver/realtime/streamline__blocks_tx_realtime.sql create mode 100644 models/streamline/silver/realtime/streamline__transactions_realtime.sql create mode 100644 models/streamline/silver/streamline__blocks.sql create mode 100644 models/streamline/silver/streamline__chainhead.sql create mode 100644 package-lock.yml create mode 100644 packages.yml create mode 100644 profiles.yml create mode 100644 requirements.txt create mode 100644 snapshots/.gitkeep diff --git a/README.md b/README.md new file mode 100644 index 0000000..7faabb0 --- /dev/null +++ b/README.md @@ -0,0 +1,75 @@ + +## Profile Set Up + +#### Use the following within profiles.yml +---- + +```yml +movement: + target: dev + outputs: + dev: + type: snowflake + account: + role: + user: + password: + region: + database: MOVEMENT_DEV + warehouse: + schema: silver + threads: 4 + client_session_keep_alive: False + query_tag: +``` + +### Resources: +- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) +- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers +- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support +- Find [dbt events](https://events.getdbt.com) near you +- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices + +- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices + +## Applying Model Tags + +### Database / Schema level tags + +Database and schema tags are applied via the `add_database_or_schema_tags` macro. These tags are inherited by their downstream objects. To add/modify tags call the appropriate tag set function within the macro. + +``` +{{ set_database_tag_value('SOME_DATABASE_TAG_KEY','SOME_DATABASE_TAG_VALUE') }} +{{ set_schema_tag_value('SOME_SCHEMA_TAG_KEY','SOME_SCHEMA_TAG_VALUE') }} +``` + +### Model tags + +To add/update a model's snowflake tags, add/modify the `meta` model property under `config`. Only table level tags are supported at this time via DBT. + +``` +{{ config( + ..., + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'SOME_PURPOSE' + } + } + }, + ... +) }} +``` + +By default, model tags are not pushed to snowflake on each load. You can push a tag update for a model by specifying the `UPDATE_SNOWFLAKE_TAGS` project variable during a run. + +``` +dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":True}' -s models/core/core__fact_swaps.sql +``` + +### Querying for existing tags on a model in snowflake + +``` +select * +from table(m1.information_schema.tag_references('movement.core.fact_blocks', 'table')); +``` \ No newline at end of file diff --git a/analyses/.gitkeep b/analyses/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv new file mode 100644 index 0000000..32c758c --- /dev/null +++ b/data/github_actions__workflows.csv @@ -0,0 +1,6 @@ +workflow_name,workflow_schedule +dbt_run_streamline_blocks_tx_realtime,"0,20,40 * * * *" +dbt_run_streamline_transactions_realtime,"15,55 * * * *" +dbt_run_incremental_core,"15,35,55 * * * *" +dbt_test_intraday,"20 */4 * * *" +dbt_test_tasks,"0,30 * * * *" \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml new file mode 100644 index 0000000..6cfc05e --- /dev/null +++ b/dbt_project.yml @@ -0,0 +1,103 @@ +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: "movement_models" +version: "1.0.0" +config-version: 2 + +require-dbt-version: ">=1.8.0" + +# This setting configures which "profile" dbt uses for this project. +profile: "movement" + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that models in this project can be +# found in the "models/" directory. You probably won't need to change these! +model-paths: ["models"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +seed-paths: ["data"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" # directory which will store compiled SQL files +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_modules" + - "dbt_packages" + +tests: + +store_failures: true # all tests + +on-run-start: + - "{{ create_sps() }}" + - "{{ create_udfs() }}" + +on-run-end: + - '{{ apply_meta_as_tags(results) }}' + +dispatch: + - macro_namespace: dbt + search_order: + - movement-models + - dbt_snowflake_query_tags + - dbt + +query-comment: + comment: '{{ dbt_snowflake_query_tags.get_query_comment(node) }}' + append: true # Snowflake removes prefixed comments. + +# Configuring models +# Full documentation: https://docs.getdbt.com/docs/configuring-models + +models: + +copy_grants: true + +persist_docs: + relation: true + columns: true + +on_schema_change: "append_new_columns" + +# In this example config, we tell dbt to build all models in the example/ directory +# as tables. These settings can be overridden in the individual model files +# using the `{{ config(...) }}` macro. + +vars: + "dbt_date:time_zone": GMT + STREAMLINE_INVOKE_STREAMS: false + STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: false + UPDATE_UDFS_AND_SPS: false + UPDATE_SNOWFLAKE_TAGS: True + OBSERV_FULL_TEST: false + START_GHA_TASKS: false + BRONZE_LOOKBACK_DAYS: '{{ env_var("BRONZE_LOOKBACK_DAYS", 3) }}' + +#### STREAMLINE 2.0 BEGIN #### + + API_INTEGRATION: '{{ var("config")[target.name]["API_INTEGRATION"] if var("config")[target.name] else var("config")["dev"]["API_INTEGRATION"] }}' + EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] if var("config")[target.name] else var("config")["dev"]["EXTERNAL_FUNCTION_URI"] }}' + ROLES: '{{ var("config")[target.name]["ROLES"] }}' + + config: + # The keys correspond to dbt profiles and are case sensitive + dev: + API_INTEGRATION: aws_m1_api_dev + EXTERNAL_FUNCTION_URI: https://z2wyjkp9r7.execute-api.us-east-1.amazonaws.com/stg/ + ROLES: + - AWS_LAMBDA_MOVEMENT_API + - INTERNAL_DEV + dev-2xl: + API_INTEGRATION: aws_m1_api_dev + EXTERNAL_FUNCTION_URI: https://z2wyjkp9r7.execute-api.us-east-1.amazonaws.com/stg/ + ROLES: + - AWS_LAMBDA_MOVEMENT_API + - INTERNAL_DEV + + prod: + API_INTEGRATION: aws_movement_api_prod + EXTERNAL_FUNCTION_URI: https://d0t060jjxf.execute-api.us-east-1.amazonaws.com/prod/ + ROLES: + - AWS_LAMBDA_MOVEMENT_API + - INTERNAL_DEV + - DBT_CLOUD_MOVEMENT + +#### STREAMLINE 2.0 END #### \ No newline at end of file diff --git a/docs/.gitkeep b/docs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/macros/.gitkeep b/macros/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/macros/create_sps.sql b/macros/create_sps.sql new file mode 100644 index 0000000..9fa7c66 --- /dev/null +++ b/macros/create_sps.sql @@ -0,0 +1,6 @@ +{% macro create_sps() %} + {% if target.database == 'MOVEMENT' %} + CREATE schema IF NOT EXISTS _internal; +{{ sp_create_prod_clone('_internal') }}; + {% endif %} +{% endmacro %} diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql new file mode 100644 index 0000000..3515bbf --- /dev/null +++ b/macros/create_udfs.sql @@ -0,0 +1,10 @@ +{% macro create_udfs() %} + {% if var("UPDATE_UDFS_AND_SPS") %} + {% set sql %} + CREATE schema if NOT EXISTS silver; +{{ create_udf_bulk_rest_api_v2() }} + + {% endset %} + {% do run_query(sql) %} + {% endif %} +{% endmacro %} diff --git a/macros/custom_naming_macros.sql b/macros/custom_naming_macros.sql new file mode 100644 index 0000000..cb6fc87 --- /dev/null +++ b/macros/custom_naming_macros.sql @@ -0,0 +1,23 @@ +{% macro generate_schema_name( + custom_schema_name = none, + node = none + ) -%} + {% set node_name = node.name %} + {% set split_name = node_name.split('__') %} + {{ split_name [0] | trim }} +{%- endmacro %} + +{% macro generate_alias_name( + custom_alias_name = none, + node = none + ) -%} + {% set node_name = node.name %} + {% set split_name = node_name.split('__') %} + {{ split_name [1] | trim }} +{%- endmacro %} + +{% macro generate_tmp_view_name(model_name) -%} + {% set node_name = model_name.name %} + {% set split_name = node_name.split('__') %} + {{ target.database ~ '.' ~ split_name[0] ~ '.' ~ split_name [1] ~ '__dbt_tmp' | trim }} +{%- endmacro %} diff --git a/macros/custom_query_tag.sql b/macros/custom_query_tag.sql new file mode 100644 index 0000000..809e1bb --- /dev/null +++ b/macros/custom_query_tag.sql @@ -0,0 +1,11 @@ +{% macro set_query_tag() -%} + {% set new_json = {"repo":project_name, "object":this.table, "profile":target.profile_name, "env":target.name, "existing_tag":get_current_query_tag() } %} +{% set new_query_tag = tojson(new_json) | as_text %} + {% if new_query_tag %} + {% set original_query_tag = get_current_query_tag() %} + {{ log("Setting query_tag to '" ~ new_query_tag ~ "'. Will reset to '" ~ original_query_tag ~ "' after materialization.") }} + {% do run_query("alter session set query_tag = '{}'".format(new_query_tag)) %} + {{ return(original_query_tag)}} + {% endif %} + {{ return(none)}} +{% endmacro %} \ No newline at end of file diff --git a/macros/dbt/get_merge_sql.sql b/macros/dbt/get_merge_sql.sql new file mode 100644 index 0000000..e1a8be2 --- /dev/null +++ b/macros/dbt/get_merge_sql.sql @@ -0,0 +1,16 @@ +{% macro get_merge_sql( + target, + source, + unique_key, + dest_columns, + incremental_predicates + ) -%} + {% set merge_sql = fsc_utils.get_merge_sql( + target, + source, + unique_key, + dest_columns, + incremental_predicates + ) %} + {{ return(merge_sql) }} +{% endmacro %} diff --git a/macros/dbt/get_tmp_relation_type.sql b/macros/dbt/get_tmp_relation_type.sql new file mode 100644 index 0000000..3bb7438 --- /dev/null +++ b/macros/dbt/get_tmp_relation_type.sql @@ -0,0 +1,8 @@ +{% macro dbt_snowflake_get_tmp_relation_type( + strategy, + unique_key, + language + ) %} + -- always table + {{ return('table') }} +{% endmacro %} diff --git a/macros/run_sp_create_prod_clone.sql b/macros/run_sp_create_prod_clone.sql new file mode 100644 index 0000000..2e0cfb1 --- /dev/null +++ b/macros/run_sp_create_prod_clone.sql @@ -0,0 +1,10 @@ +{% macro run_sp_create_prod_clone() %} + {% set clone_query %} + call m1._internal.create_prod_clone( + 'movement', + 'movement_dev', + 'internal_dev' + ); +{% endset %} + {% do run_query(clone_query) %} +{% endmacro %} diff --git a/macros/sp_create_prod_clone.sql b/macros/sp_create_prod_clone.sql new file mode 100644 index 0000000..20ee897 --- /dev/null +++ b/macros/sp_create_prod_clone.sql @@ -0,0 +1,44 @@ +{% macro sp_create_prod_clone(target_schema) -%} + +create or replace procedure {{ target_schema }}.create_prod_clone(source_db_name string, destination_db_name string, role_name string) +returns boolean +language javascript +execute as caller +as +$$ + snowflake.execute({sqlText: `BEGIN TRANSACTION;`}); + try { + snowflake.execute({sqlText: `CREATE OR REPLACE DATABASE ${DESTINATION_DB_NAME} CLONE ${SOURCE_DB_NAME}`}); + snowflake.execute({sqlText: `DROP SCHEMA IF EXISTS ${DESTINATION_DB_NAME}._INTERNAL`}); /* this only needs to be in prod */ + + snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL SCHEMAS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL FUNCTIONS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL PROCEDURES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL VIEWS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL STAGES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL TABLES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE FUNCTIONS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE PROCEDURES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE VIEWS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE STAGES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE TABLES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); + + snowflake.execute({sqlText: `GRANT OWNERSHIP ON DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}) + + var existing_tags = snowflake.execute({sqlText: `SHOW TAGS IN DATABASE ${DESTINATION_DB_NAME};`}); + while (existing_tags.next()) { + var schema = existing_tags.getColumnValue(4); + var tag_name = existing_tags.getColumnValue(2) + snowflake.execute({sqlText: `GRANT OWNERSHIP ON TAG ${DESTINATION_DB_NAME}.${schema}.${tag_name} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + } + + snowflake.execute({sqlText: `COMMIT;`}); + } catch (err) { + snowflake.execute({sqlText: `ROLLBACK;`}); + throw(err); + } + + return true +$$ + +{%- endmacro %} \ No newline at end of file diff --git a/macros/streamline/api_integrations.sql b/macros/streamline/api_integrations.sql new file mode 100644 index 0000000..d34e467 --- /dev/null +++ b/macros/streamline/api_integrations.sql @@ -0,0 +1,21 @@ +{% macro create_aws_movement_api() %} + {{ log( + "Creating integration for target:" ~ target + ) }} + + {% if target.name == "prod" %} + {% set sql %} + CREATE api integration IF NOT EXISTS aws_movement_api_prod api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::924682671219:role/movement-api-prod-rolesnowflakeudfsAF733095-uYaV9o2uJLDU' api_allowed_prefixes = ( + 'https://d0t060jjxf.execute-api.us-east-1.amazonaws.com/prod/' + ) enabled = TRUE; +{% endset %} + {% do run_query(sql) %} + {% else %} + {% set sql %} + CREATE api integration IF NOT EXISTS aws_m1_api_dev api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::704693948482:role/m1-api-stg-rolesnowflakeudfsAF733095-bEigMvFi81Fd' api_allowed_prefixes = ( + 'https://z2wyjkp9r7.execute-api.us-east-1.amazonaws.com/stg/' + ) enabled = TRUE; +{% endset %} + {% do run_query(sql) %} + {% endif %} +{% endmacro %} diff --git a/macros/streamline/models.sql b/macros/streamline/models.sql new file mode 100644 index 0000000..503e680 --- /dev/null +++ b/macros/streamline/models.sql @@ -0,0 +1,70 @@ +{% macro streamline_external_table_query_v2( + model, + partition_function + ) %} + + {% set days = var("BRONZE_LOOKBACK_DAYS")%} + + WITH meta AS ( + SELECT + last_modified AS inserted_timestamp, + file_name, + {{ partition_function }} AS partition_key + FROM + TABLE( + information_schema.external_table_file_registration_history( + start_time => DATEADD('day', -ABS({{days}}), CURRENT_TIMESTAMP()), + table_name => '{{ source( "bronze_streamline", model) }}') + ) A + ) + SELECT + s.*, + b.file_name, + inserted_timestamp + FROM + {{ source( + "bronze_streamline", + model + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b.partition_key = s.partition_key + WHERE + b.partition_key = s.partition_key + AND DATA :error IS NULL +{% endmacro %} + +{% macro streamline_external_table_FR_query_v2( + model, + partition_function + ) %} + WITH meta AS ( + SELECT + registered_on AS inserted_timestamp, + file_name, + {{ partition_function }} AS partition_key + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", model) }}' + ) + ) A + ) +SELECT + s.*, + b.file_name, + inserted_timestamp +FROM + {{ source( + "bronze_streamline", + model + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b.partition_key = s.partition_key +WHERE + b.partition_key = s.partition_key + AND DATA :error IS NULL +{% endmacro %} diff --git a/macros/streamline/streamline_udfs.sql b/macros/streamline/streamline_udfs.sql new file mode 100644 index 0000000..f6c3948 --- /dev/null +++ b/macros/streamline/streamline_udfs.sql @@ -0,0 +1,10 @@ +{% macro create_udf_bulk_rest_api_v2() %} + CREATE + OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_rest_api_v2( + json OBJECT + ) returns ARRAY api_integration = {% if target.name == "prod" %} + aws_movement_api_prod AS 'https://d0t060jjxf.execute-api.us-east-1.amazonaws.com/prod/udf_bulk_rest_api' + {% else %} + aws_m1_api_dev AS 'https://z2wyjkp9r7.execute-api.us-east-1.amazonaws.com/stg/udf_bulk_rest_api' + {%- endif %}; +{% endmacro %} diff --git a/macros/tags/add_database_or_schema_tags.sql b/macros/tags/add_database_or_schema_tags.sql new file mode 100644 index 0000000..eb00e45 --- /dev/null +++ b/macros/tags/add_database_or_schema_tags.sql @@ -0,0 +1,6 @@ +{% macro add_database_or_schema_tags() %} + {{ set_database_tag_value( + 'BLOCKCHAIN_NAME', + 'movement' + ) }} +{% endmacro %} diff --git a/macros/tags/snowflake_tagging.sql b/macros/tags/snowflake_tagging.sql new file mode 100644 index 0000000..bc25e69 --- /dev/null +++ b/macros/tags/snowflake_tagging.sql @@ -0,0 +1,127 @@ +{% macro apply_meta_as_tags(results) %} + {% if var("UPDATE_SNOWFLAKE_TAGS") %} + {{ log('apply_meta_as_tags', info=False) }} + {{ log(results, info=False) }} + {% if execute %} + + {%- set tags_by_schema = {} -%} + {% for res in results -%} + {% if res.node.meta.database_tags %} + + {%- set model_database = res.node.database -%} + {%- set model_schema = res.node.schema -%} + {%- set model_schema_full = model_database+'.'+model_schema -%} + {%- set model_alias = res.node.alias -%} + + {% if model_schema_full not in tags_by_schema.keys() %} + {{ log('need to fetch tags for schema '+model_schema_full, info=False) }} + {%- call statement('main', fetch_result=True) -%} + show tags in {{model_database}}.{{model_schema}} + {%- endcall -%} + {%- set _ = tags_by_schema.update({model_schema_full: load_result('main')['table'].columns.get('name').values()|list}) -%} + {{ log('Added tags to cache', info=False) }} + {% else %} + {{ log('already have tag info for schema', info=False) }} + {% endif %} + + {%- set current_tags_in_schema = tags_by_schema[model_schema_full] -%} + {{ log('current_tags_in_schema:', info=False) }} + {{ log(current_tags_in_schema, info=False) }} + {{ log("========== Processing tags for "+model_schema_full+"."+model_alias+" ==========", info=False) }} + + {% set line -%} + node: {{ res.node.unique_id }}; status: {{ res.status }} (message: {{ res.message }}) + node full: {{ res.node}} + meta: {{ res.node.meta}} + materialized: {{ res.node.config.materialized }} + {%- endset %} + {{ log(line, info=False) }} + + {%- call statement('main', fetch_result=True) -%} + select LEVEL,UPPER(TAG_NAME) as TAG_NAME,TAG_VALUE from table(information_schema.tag_references_all_columns('{{model_schema}}.{{model_alias}}', 'table')) + {%- endcall -%} + {%- set existing_tags_for_table = load_result('main')['data'] -%} + {{ log('Existing tags for table:', info=False) }} + {{ log(existing_tags_for_table, info=False) }} + + {{ log('--', info=False) }} + {% for table_tag in res.node.meta.database_tags.table %} + + {{ create_tag_if_missing(current_tags_in_schema,table_tag|upper) }} + {% set desired_tag_value = res.node.meta.database_tags.table[table_tag] %} + + {{set_table_tag_value_if_different(model_schema,model_alias,table_tag,desired_tag_value,existing_tags_for_table)}} + {% endfor %} + {{ log("========== Finished processing tags for "+model_alias+" ==========", info=False) }} + {% endif %} + {% endfor %} + {% endif %} + {% endif %} +{% endmacro %} + + +{% macro create_tag_if_missing(all_tag_names,table_tag) %} + {% if table_tag not in all_tag_names %} + {{ log('Creating missing tag '+table_tag, info=False) }} + {%- call statement('main', fetch_result=True) -%} + create tag if not exists silver.{{table_tag}} + {%- endcall -%} + {{ log(load_result('main').data, info=False) }} + {% else %} + {{ log('Tag already exists: '+table_tag, info=False) }} + {% endif %} +{% endmacro %} + +{% macro set_table_tag_value_if_different(model_schema,table_name,tag_name,desired_tag_value,existing_tags) %} + {{ log('Ensuring tag '+tag_name+' has value '+desired_tag_value+' at table level', info=False) }} + {%- set existing_tag_for_table = existing_tags|selectattr('0','equalto','TABLE')|selectattr('1','equalto',tag_name|upper)|list -%} + {{ log('Filtered tags for table:', info=False) }} + {{ log(existing_tag_for_table[0], info=False) }} + {% if existing_tag_for_table|length > 0 and existing_tag_for_table[0][2]==desired_tag_value %} + {{ log('Correct tag value already exists', info=False) }} + {% else %} + {{ log('Setting tag value for '+tag_name+' to value '+desired_tag_value, info=False) }} + {%- call statement('main', fetch_result=True) -%} + alter table {{model_schema}}.{{table_name}} set tag {{tag_name}} = '{{desired_tag_value}}' + {%- endcall -%} + {{ log(load_result('main').data, info=False) }} + {% endif %} +{% endmacro %} + +{% macro set_column_tag_value_if_different(table_name,column_name,tag_name,desired_tag_value,existing_tags) %} + {{ log('Ensuring tag '+tag_name+' has value '+desired_tag_value+' at column level', info=False) }} + {%- set existing_tag_for_column = existing_tags|selectattr('0','equalto','COLUMN')|selectattr('1','equalto',tag_name|upper)|list -%} + {{ log('Filtered tags for column:', info=False) }} + {{ log(existing_tag_for_column[0], info=False) }} + {% if existing_tag_for_column|length > 0 and existing_tag_for_column[0][2]==desired_tag_value %} + {{ log('Correct tag value already exists', info=False) }} + {% else %} + {{ log('Setting tag value for '+tag_name+' to value '+desired_tag_value, info=False) }} + {%- call statement('main', fetch_result=True) -%} + alter table {{table_name}} modify column {{column_name}} set tag {{tag_name}} = '{{desired_tag_value}}' + {%- endcall -%} + {{ log(load_result('main').data, info=False) }} + {% endif %} +{% endmacro %} + +{% macro set_database_tag_value(tag_name,tag_value) %} + {% set query %} + create tag if not exists silver.{{tag_name}} + {% endset %} + {% do run_query(query) %} + {% set query %} + alter database {{target.database}} set tag {{target.database}}.silver.{{tag_name}} = '{{tag_value}}' + {% endset %} + {% do run_query(query) %} +{% endmacro %} + +{% macro set_schema_tag_value(target_schema,tag_name,tag_value) %} + {% set query %} + create tag if not exists silver.{{tag_name}} + {% endset %} + {% do run_query(query) %} + {% set query %} + alter schema {{target.database}}.{{target_schema}} set tag {{target.database}}.silver.{{tag_name}} = '{{tag_value}}' + {% endset %} + {% do run_query(query) %} +{% endmacro %} \ No newline at end of file diff --git a/macros/tests/compare_model_subset.sql b/macros/tests/compare_model_subset.sql new file mode 100644 index 0000000..18aa624 --- /dev/null +++ b/macros/tests/compare_model_subset.sql @@ -0,0 +1,29 @@ +{% test compare_model_subset(model, compare_model, compare_columns, model_condition) %} + +{% set compare_cols_csv = compare_columns | join(', ') %} + +with a as ( + select {{compare_cols_csv}} from {{ model }} + {{ model_condition }} +), +b as ( + select {{compare_cols_csv}} from {{ compare_model }} +), +a_minus_b as ( + select * from a + except + select * from b +), +b_minus_a as ( + select * from b + except + select * from a +), +unioned as ( + select 'in_actual_not_in_expected' as which_diff, a_minus_b.* from a_minus_b + union all + select 'in_expected_not_in_actual' as which_diff, b_minus_a.* from b_minus_a +) +select * from unioned + +{% endtest %} \ No newline at end of file diff --git a/macros/tests/sequence_gaps.sql b/macros/tests/sequence_gaps.sql new file mode 100644 index 0000000..84a9aa9 --- /dev/null +++ b/macros/tests/sequence_gaps.sql @@ -0,0 +1,37 @@ +{% macro sequence_gaps( + table, + partition_by, + column + ) %} + {%- set partition_sql = partition_by | join(", ") -%} + {%- set previous_column = "prev_" ~ column -%} + WITH source AS ( + SELECT + {{ partition_sql + "," if partition_sql }} + {{ column }}, + LAG( + {{ column }}, + 1 + ) over ( + {{ "PARTITION BY " ~ partition_sql if partition_sql }} + ORDER BY + {{ column }} ASC + ) AS {{ previous_column }} + FROM + {{ table }} + WHERE + block_timestamp::date <= current_date - 1 + ) +SELECT + {{ partition_sql + "," if partition_sql }} + {{ previous_column }}, + {{ column }}, + {{ column }} - {{ previous_column }} + - 1 AS gap +FROM + source +WHERE + {{ column }} - {{ previous_column }} <> 1 +ORDER BY + gap DESC +{% endmacro %} diff --git a/macros/tests/tx_gaps.sql b/macros/tests/tx_gaps.sql new file mode 100644 index 0000000..82b449f --- /dev/null +++ b/macros/tests/tx_gaps.sql @@ -0,0 +1,33 @@ +{% macro tx_gaps( + model + ) %} + WITH block_base AS ( + SELECT + block_id, + tx_count + FROM + {{ ref('silver__blocks') }} + ), + model_name AS ( + SELECT + block_id, + COUNT( + DISTINCT tx_id + ) AS model_tx_count + FROM + {{ model }} + GROUP BY + block_id + ) +SELECT + block_base.block_id, + tx_count, + model_name.block_id, + model_tx_count +FROM + block_base + LEFT JOIN model_name + ON block_base.block_id = model_name.block_id +WHERE + tx_count <> model_tx_count +{% endmacro %} diff --git a/macros/utils.sql b/macros/utils.sql new file mode 100644 index 0000000..2e3b3cb --- /dev/null +++ b/macros/utils.sql @@ -0,0 +1,42 @@ +{% macro if_data_call_wait() %} + {% if var( + "STREAMLINE_INVOKE_STREAMS" + ) %} + {% set query %} + SELECT + 1 + WHERE + EXISTS( + SELECT + 1 + FROM + {{ model.schema ~ "." ~ model.alias }} + LIMIT + 1 + ) {% endset %} + {% if execute %} + {% set results = run_query( + query + ) %} + {% if results %} + {{ log( + "Waiting...", + info = True + ) }} + + {% set wait_query %} + SELECT + system$wait( + {{ var( + "WAIT", + 600 + ) }} + ) {% endset %} + {% do run_query(wait_query) %} + {% else %} + SELECT + NULL; + {% endif %} + {% endif %} + {% endif %} +{% endmacro %} diff --git a/models/bronze/core/bronze__blocks_tx.sql b/models/bronze/core/bronze__blocks_tx.sql new file mode 100644 index 0000000..a3e3432 --- /dev/null +++ b/models/bronze/core/bronze__blocks_tx.sql @@ -0,0 +1,7 @@ +{{ config ( + materialized = 'view' +) }} +{{ streamline_external_table_query_v2( + model = "blocks_tx", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" +) }} diff --git a/models/bronze/core/bronze__blocks_tx_FR.sql b/models/bronze/core/bronze__blocks_tx_FR.sql new file mode 100644 index 0000000..6541673 --- /dev/null +++ b/models/bronze/core/bronze__blocks_tx_FR.sql @@ -0,0 +1,7 @@ +{{ config ( + materialized = 'view' +) }} +{{ streamline_external_table_FR_query_v2( + model = "blocks_tx", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" +) }} diff --git a/models/bronze/core/bronze__transactions.sql b/models/bronze/core/bronze__transactions.sql new file mode 100644 index 0000000..a7af4cc --- /dev/null +++ b/models/bronze/core/bronze__transactions.sql @@ -0,0 +1,7 @@ +{{ config ( + materialized = 'view' +) }} +{{ streamline_external_table_query_v2( + model = "transactions", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" +) }} diff --git a/models/bronze/core/bronze__transactions_FR.sql b/models/bronze/core/bronze__transactions_FR.sql new file mode 100644 index 0000000..7ad9586 --- /dev/null +++ b/models/bronze/core/bronze__transactions_FR.sql @@ -0,0 +1,7 @@ +{{ config ( + materialized = 'view' +) }} +{{ streamline_external_table_FR_query_v2( + model = "transactions", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" +) }} diff --git a/models/bronze/prices/bronze__complete_native_prices.sql b/models/bronze/prices/bronze__complete_native_prices.sql new file mode 100644 index 0000000..02ef0fb --- /dev/null +++ b/models/bronze/prices/bronze__complete_native_prices.sql @@ -0,0 +1,29 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + HOUR, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_native_prices_id, + _invocation_id +FROM + {{ source( + 'crosschain_silver', + 'complete_native_prices' + ) }} +WHERE + blockchain = 'sei' + AND symbol = 'SEI' diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md new file mode 100644 index 0000000..9dd1095 --- /dev/null +++ b/models/descriptions/__overview__.md @@ -0,0 +1,70 @@ +{% docs __overview__ %} + +# Welcome to the Flipside Crypto m1 Models Documentation + +## **What does this documentation cover?** +The documentation included here details the design of the m1 + tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/flipsideCrypto/m1-models/) + +## **How do I use these docs?** +The easiest way to navigate this documentation is to use the Quick Links below. These links will take you to the documentation for each table, which contains a description, a list of the columns, and other helpful information. + +If you are experienced with dbt docs, feel free to use the sidebar to navigate the documentation, as well as explore the relationships between tables and the logic building them. + +There is more information on how to use dbt docs in the last section of this document. + +## **Quick Links to Table Documentation** + +**Click on the links below to jump to the documentation for each schema.** + +### Core Fact Tables (`m1`.`CORE`.``) +- [fact_blocks](#!/model/model.movement_models.core__fact_blocks) +- [fact_events](#!/model/model.movement_models.core__fact_events) +- [fact_changes](#!/model/model.movement_models.core__fact_changes) +- [fact_transactions](#!/model/model.movement_models.core__fact_transactions) +- [fact_transactions_block_metadata](#!/model/model.movement_models.core__fact_transactions_block_metadata) +- [fact_transactions_state_checkpoint](#!/model/model.movement_models.core__fact_transactions_state_checkpoint) + + + + +## **Data Model Overview** + +The m1 models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** + +- Bronze: Data is loaded in from the source as a view +- Silver: All necessary parsing, filtering, de-duping, and other transformations are done here +- Gold (or core): Final views and tables that are available publicly + +The dimension tables are sourced from a variety of on-chain and off-chain sources. + +Convenience views (denoted ez_) are a combination of different fact and dimension tables. These views are built to make it easier to query the data. + +## **Using dbt docs** +### Navigation + +You can use the ```Project``` and ```Database``` navigation tabs on the left side of the window to explore the models in the project. + +### Database Tab + +This view shows relations (tables and views) grouped into database schemas. Note that ephemeral models are *not* shown in this interface, as they do not exist in the database. + +### Graph Exploration + +You can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models. + +On model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the Expand button at the top-right of this lineage pane, you'll be able to see all of the models that are used to build, or are built from, the model you're exploring. + +Once expanded, you'll be able to use the ```--models``` and ```--exclude``` model selection syntax to filter the models in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax). + +Note that you can also right-click on models to interactively filter and explore the graph. + + +### **More information** +- [Flipside](https://flipsidecrypto.xyz/) +- [Velocity](https://app.flipsidecrypto.com/velocity?nav=Discover) +- [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials) +- [Github](https://github.com/FlipsideCrypto/m1-models) +- [What is dbt?](https://docs.getdbt.com/docs/introduction) + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/accumulator_root_hash.md b/models/descriptions/accumulator_root_hash.md new file mode 100644 index 0000000..912fbd5 --- /dev/null +++ b/models/descriptions/accumulator_root_hash.md @@ -0,0 +1,5 @@ +{% docs accumulator_root_hash %} + +The root hash of a Merkle accumulator. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/address.md b/models/descriptions/address.md new file mode 100644 index 0000000..36215fe --- /dev/null +++ b/models/descriptions/address.md @@ -0,0 +1,5 @@ +{% docs address %} + +Address unique to an individual wallet, validator, or token. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/address_change.md b/models/descriptions/address_change.md new file mode 100644 index 0000000..74e0e44 --- /dev/null +++ b/models/descriptions/address_change.md @@ -0,0 +1,5 @@ +{% docs address_change %} + +The top level address for this change. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/address_event.md b/models/descriptions/address_event.md new file mode 100644 index 0000000..e97de9f --- /dev/null +++ b/models/descriptions/address_event.md @@ -0,0 +1,5 @@ +{% docs address_event %} + +The top level address for this event. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/amount.md b/models/descriptions/amount.md new file mode 100644 index 0000000..a01c8e4 --- /dev/null +++ b/models/descriptions/amount.md @@ -0,0 +1,5 @@ +{% docs amount %} + +The non-decimal adjusted amount of a token. For example, if a token has 18 decimals, then the amount of 1 token is 10^18. + +{% enddocs %} diff --git a/models/descriptions/block_hash.md b/models/descriptions/block_hash.md new file mode 100644 index 0000000..1658154 --- /dev/null +++ b/models/descriptions/block_hash.md @@ -0,0 +1,5 @@ +{% docs block_hash %} + +The hash of the block header for a given block. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/block_number.md b/models/descriptions/block_number.md new file mode 100644 index 0000000..ba7fa4f --- /dev/null +++ b/models/descriptions/block_number.md @@ -0,0 +1,5 @@ +{% docs block_number %} + +Also known as block height. The block number, which indicates the length of the blockchain, increases after the addition of each new block. + +{% enddocs %} diff --git a/models/descriptions/block_timestamp.md b/models/descriptions/block_timestamp.md new file mode 100644 index 0000000..0c1a1b8 --- /dev/null +++ b/models/descriptions/block_timestamp.md @@ -0,0 +1,5 @@ +{% docs block_timestamp %} + +The date and time at which the block was produced. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/blockchain.md b/models/descriptions/blockchain.md new file mode 100644 index 0000000..b5098dc --- /dev/null +++ b/models/descriptions/blockchain.md @@ -0,0 +1,5 @@ +{% docs blockchain %} + +The name of the blockchain + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/change_address.md b/models/descriptions/change_address.md new file mode 100644 index 0000000..c1f0173 --- /dev/null +++ b/models/descriptions/change_address.md @@ -0,0 +1,5 @@ +{% docs change_address %} + +The first segment of the inner change type + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/change_data.md b/models/descriptions/change_data.md new file mode 100644 index 0000000..0f669dc --- /dev/null +++ b/models/descriptions/change_data.md @@ -0,0 +1,5 @@ +{% docs change_data %} + +The "data" object within this change. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/change_index.md b/models/descriptions/change_index.md new file mode 100644 index 0000000..f5b008e --- /dev/null +++ b/models/descriptions/change_index.md @@ -0,0 +1,5 @@ +{% docs change_index %} + +Unique identifier for the change. This is a monotonically increasing integer that is incremented for each change. This is useful for determining the order of changes. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/change_module.md b/models/descriptions/change_module.md new file mode 100644 index 0000000..0aecd3c --- /dev/null +++ b/models/descriptions/change_module.md @@ -0,0 +1,5 @@ +{% docs change_module %} + +The second segment of the inner change type + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/change_resource.md b/models/descriptions/change_resource.md new file mode 100644 index 0000000..54e227a --- /dev/null +++ b/models/descriptions/change_resource.md @@ -0,0 +1,5 @@ +{% docs change_resource %} + +The third segment of the inner change type + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/change_type.md b/models/descriptions/change_type.md new file mode 100644 index 0000000..aeef1e9 --- /dev/null +++ b/models/descriptions/change_type.md @@ -0,0 +1,5 @@ +{% docs change_type %} + +The "type" object from within this change. Values are: delete_resource, delete_table_item, write_module, write_resource, write_table_item. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/changes.md b/models/descriptions/changes.md new file mode 100644 index 0000000..8b718ad --- /dev/null +++ b/models/descriptions/changes.md @@ -0,0 +1,5 @@ +{% docs changes %} + +The changes that the transaction executed. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/creation_number.md b/models/descriptions/creation_number.md new file mode 100644 index 0000000..31e5dac --- /dev/null +++ b/models/descriptions/creation_number.md @@ -0,0 +1,5 @@ +{% docs creation_number %} + +Ceation number corresponding to the event stream originating from the given account. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/creator.md b/models/descriptions/creator.md new file mode 100644 index 0000000..ff72c36 --- /dev/null +++ b/models/descriptions/creator.md @@ -0,0 +1,5 @@ +{% docs creator %} + +Name of the label creator - for now, this will always be "Flipside." + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/defi__fact_bridge_activity.md b/models/descriptions/defi__fact_bridge_activity.md new file mode 100644 index 0000000..5e1e5eb --- /dev/null +++ b/models/descriptions/defi__fact_bridge_activity.md @@ -0,0 +1,55 @@ + +{% docs bridge_platform %} + +The platform or protocol from which the bridge transaction or event originates. + +{% enddocs %} + +{% docs bridge_sender %} + +The address that initiated the bridge deposit or transfer. This address is the sender of the tokens/assets being bridged to the destination chain. + +{% enddocs %} + +{% docs bridge_receiver %} + +The designated address set to receive the bridged tokens on the target chain after the completion of the bridge transaction. For non-evm chains, the hex address is decoded/encoded to match the data format of the destination chain, where possible. + +{% enddocs %} + +{% docs destination_chain %} + +The name of the blockchain network to which the assets are being bridged. + +{% enddocs %} + +{% docs destination_chain_id %} + +The numeric identifier associated with the destination blockchain network. This is specific to the chain and helps in uniquely identifying it. + +{% enddocs %} + +{% docs bridge_address %} + +The address of the contract responsible for handling the bridge deposit or transfer. This contract mediates the transfer and ensures that assets are sent and received appropriately. + +{% enddocs %} + + +{% docs bridge_token_address %} + +The address associated with the token that is being bridged. It provides a unique identifier for the token within its origin blockchain. + +{% enddocs %} + +{% docs source_chain_id %} + +The numeric identifier associated with the source blockchain network. This is specific to the chain and helps in uniquely identifying it. + +{% enddocs %} + +{% docs source_chain %} + +The name of the blockchain network from which the assets are being bridged. + +{% enddocs %} diff --git a/models/descriptions/epoch.md b/models/descriptions/epoch.md new file mode 100644 index 0000000..1befe29 --- /dev/null +++ b/models/descriptions/epoch.md @@ -0,0 +1,5 @@ +{% docs epoch %} + +An epoch in the Aptos blockchain is defined as a duration of time, in seconds, during which a number of blocks are voted on by the validators, the validator set is updated, and the rewards are distributed to the validators. The Aptos mainnet epoch is set as 7200 seconds (two hours). + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/event_address.md b/models/descriptions/event_address.md new file mode 100644 index 0000000..27c91b9 --- /dev/null +++ b/models/descriptions/event_address.md @@ -0,0 +1,5 @@ +{% docs event_address %} + +The first segment of the event type + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/event_data.md b/models/descriptions/event_data.md new file mode 100644 index 0000000..20f0f1b --- /dev/null +++ b/models/descriptions/event_data.md @@ -0,0 +1,5 @@ +{% docs event_data %} + +The "data" object within this event. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/event_index.md b/models/descriptions/event_index.md new file mode 100644 index 0000000..b11e8ab --- /dev/null +++ b/models/descriptions/event_index.md @@ -0,0 +1,5 @@ +{% docs event_index %} + +Unique identifier for the event. This is a monotonically increasing integer that is incremented for each event. This is useful for determining the order of changes. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/event_module.md b/models/descriptions/event_module.md new file mode 100644 index 0000000..0cd680f --- /dev/null +++ b/models/descriptions/event_module.md @@ -0,0 +1,5 @@ +{% docs event_module %} + +The second segment of the event type + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/event_resource.md b/models/descriptions/event_resource.md new file mode 100644 index 0000000..171c2e1 --- /dev/null +++ b/models/descriptions/event_resource.md @@ -0,0 +1,5 @@ +{% docs event_resource %} + +The third segment of the event type + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/event_root_hash.md b/models/descriptions/event_root_hash.md new file mode 100644 index 0000000..496edae --- /dev/null +++ b/models/descriptions/event_root_hash.md @@ -0,0 +1,5 @@ +{% docs event_root_hash %} + +The root hash for the event. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/event_type.md b/models/descriptions/event_type.md new file mode 100644 index 0000000..59986ef --- /dev/null +++ b/models/descriptions/event_type.md @@ -0,0 +1,5 @@ +{% docs event_type %} + +The full three-part descriptive type from event. The event type consists of the event_address :: event_module :: event_resource. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/events.md b/models/descriptions/events.md new file mode 100644 index 0000000..c96082b --- /dev/null +++ b/models/descriptions/events.md @@ -0,0 +1,5 @@ +{% docs events %} + +The events that the transaction executed. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/expiration_timestamp_secs.md b/models/descriptions/expiration_timestamp_secs.md new file mode 100644 index 0000000..0386f71 --- /dev/null +++ b/models/descriptions/expiration_timestamp_secs.md @@ -0,0 +1,5 @@ +{% docs expiration_timestamp_secs %} + +The time at which the transaction ceases to valid. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/first_version.md b/models/descriptions/first_version.md new file mode 100644 index 0000000..1a5825b --- /dev/null +++ b/models/descriptions/first_version.md @@ -0,0 +1,5 @@ +{% docs first_version %} + +The version number of the first transaction in the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/from_address_transfer.md b/models/descriptions/from_address_transfer.md new file mode 100644 index 0000000..fc8b5a1 --- /dev/null +++ b/models/descriptions/from_address_transfer.md @@ -0,0 +1,5 @@ +{% docs from_address_transfer %} + +The account address that sent the transfer. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/gas_unit_price.md b/models/descriptions/gas_unit_price.md new file mode 100644 index 0000000..0a53fd5 --- /dev/null +++ b/models/descriptions/gas_unit_price.md @@ -0,0 +1,5 @@ +{% docs gas_unit_price %} + +The cost per unit of gas, determining the transaction fee paid by the sender for each unit of computational resource consumed + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/gas_used.md b/models/descriptions/gas_used.md new file mode 100644 index 0000000..785880e --- /dev/null +++ b/models/descriptions/gas_used.md @@ -0,0 +1,5 @@ +{% docs gas_used %} + +The amount of gas used for the transaction + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/handle_change.md b/models/descriptions/handle_change.md new file mode 100644 index 0000000..29db8eb --- /dev/null +++ b/models/descriptions/handle_change.md @@ -0,0 +1,5 @@ +{% docs handle_change %} + +The top level handle for this change. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/inner_change_type.md b/models/descriptions/inner_change_type.md new file mode 100644 index 0000000..1ae3156 --- /dev/null +++ b/models/descriptions/inner_change_type.md @@ -0,0 +1,5 @@ +{% docs inner_change_type %} + +The full three-part descriptive change type from change. The inner change type consists of the change_address :: change_module :: change_resource. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/inserted_timestamp.md b/models/descriptions/inserted_timestamp.md new file mode 100644 index 0000000..14703c3 --- /dev/null +++ b/models/descriptions/inserted_timestamp.md @@ -0,0 +1,5 @@ +{% docs inserted_timestamp %} + +The utc timestamp at which the row was inserted into the table. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/key_change.md b/models/descriptions/key_change.md new file mode 100644 index 0000000..a277e5f --- /dev/null +++ b/models/descriptions/key_change.md @@ -0,0 +1,5 @@ +{% docs key_change %} + +The key value for the write_table_item change + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/label.md b/models/descriptions/label.md new file mode 100644 index 0000000..51af3dc --- /dev/null +++ b/models/descriptions/label.md @@ -0,0 +1,5 @@ +{% docs label %} + +The label or name of the address. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/label_subtype.md b/models/descriptions/label_subtype.md new file mode 100644 index 0000000..113de57 --- /dev/null +++ b/models/descriptions/label_subtype.md @@ -0,0 +1,5 @@ +{% docs label_subtype %} + +Adds more detail to the label type. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/label_type.md b/models/descriptions/label_type.md new file mode 100644 index 0000000..c81472b --- /dev/null +++ b/models/descriptions/label_type.md @@ -0,0 +1,5 @@ +{% docs label_type %} + +A broad category that describes what a label is representing. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/last_version.md b/models/descriptions/last_version.md new file mode 100644 index 0000000..a56d7b6 --- /dev/null +++ b/models/descriptions/last_version.md @@ -0,0 +1,5 @@ +{% docs last_version %} + +The version number of the last transaction in the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/max_gas_amount.md b/models/descriptions/max_gas_amount.md new file mode 100644 index 0000000..45da165 --- /dev/null +++ b/models/descriptions/max_gas_amount.md @@ -0,0 +1,5 @@ +{% docs max_gas_amount %} + +The maximum amount of gas allocated for the execution of a transaction + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/modified_timestamp.md b/models/descriptions/modified_timestamp.md new file mode 100644 index 0000000..c4d18ee --- /dev/null +++ b/models/descriptions/modified_timestamp.md @@ -0,0 +1,5 @@ +{% docs modified_timestamp %} + +The utc timestamp at which the row was last modified. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/payload.md b/models/descriptions/payload.md new file mode 100644 index 0000000..bc8f12b --- /dev/null +++ b/models/descriptions/payload.md @@ -0,0 +1,5 @@ +{% docs payload %} + +The data that is being carried by a transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/payload_function.md b/models/descriptions/payload_function.md new file mode 100644 index 0000000..912a1a1 --- /dev/null +++ b/models/descriptions/payload_function.md @@ -0,0 +1,5 @@ +{% docs payload_function %} + +The function that is being called in the transaction payload. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/pk.md b/models/descriptions/pk.md new file mode 100644 index 0000000..8dfdfb5 --- /dev/null +++ b/models/descriptions/pk.md @@ -0,0 +1,5 @@ +{% docs pk %} + +The unique identifier for each row in the table. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/proposer.md b/models/descriptions/proposer.md new file mode 100644 index 0000000..d66b281 --- /dev/null +++ b/models/descriptions/proposer.md @@ -0,0 +1,5 @@ +{% docs proposer %} + +The block proposer. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/round.md b/models/descriptions/round.md new file mode 100644 index 0000000..f24284a --- /dev/null +++ b/models/descriptions/round.md @@ -0,0 +1,5 @@ +{% docs round %} + +A round number is a shared counter used to select leaders during an epoch of the consensus protocol. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/sender.md b/models/descriptions/sender.md new file mode 100644 index 0000000..b86343b --- /dev/null +++ b/models/descriptions/sender.md @@ -0,0 +1,5 @@ +{% docs sender %} + +Sender is the address of the originator account for a transaction. A transaction must be signed by the originator. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/sequence_number.md b/models/descriptions/sequence_number.md new file mode 100644 index 0000000..7738bf6 --- /dev/null +++ b/models/descriptions/sequence_number.md @@ -0,0 +1,5 @@ +{% docs sequence_number %} + +The sequence number for an account indicates the number of transactions that have been submitted and committed on chain from that account. It is incremented every time a transaction sent from that account is executed or aborted and stored in the blockchain. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/signature.md b/models/descriptions/signature.md new file mode 100644 index 0000000..121914a --- /dev/null +++ b/models/descriptions/signature.md @@ -0,0 +1,5 @@ +{% docs signature %} + +A signature is the result of hashing the signing message with the client's private key. By default Aptos uses the Ed25519 scheme to generate the signature of the raw transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/stats/stats_core.md b/models/descriptions/stats/stats_core.md new file mode 100644 index 0000000..3e45be9 --- /dev/null +++ b/models/descriptions/stats/stats_core.md @@ -0,0 +1,71 @@ +{% docs ez_core_metrics_hourly_table_doc %} + +A convenience table that aggregates block and transaction related metrics using various aggregate functions such as SUM, COUNT, MIN and MAX from the fact_transactions table, on an hourly basis. Stats for the current hour will be updated as new data arrives. + +{% enddocs %} + +{% docs block_timestamp_hour %} + +The hour of the timestamp of the block. + +{% enddocs %} + +{% docs block_number_min %} + +The minimum block number in the hour. + +{% enddocs %} + +{% docs block_number_max %} + +The maximum block number in the hour. + +{% enddocs %} + +{% docs block_count %} + +The number of blocks in the hour. + +{% enddocs %} + +{% docs transaction_count %} + +The number of transactions in the hour. + +{% enddocs %} + +{% docs transaction_count_success %} + +The number of successful transactions in the hour. + +{% enddocs %} + +{% docs transaction_count_failed %} + +The number of failed transactions in the hour. + +{% enddocs %} + +{% docs unique_sender_count %} + +The number of unique sender address in the hour. + +{% enddocs %} + +{% docs unique_payload_function_count %} + +The number of unique payload functions in the hour. + +{% enddocs %} + +{% docs total_fees_native %} + +The sum of all fees in the hour, in the native fee currency. + +{% enddocs %} + +{% docs total_fees_usd %} + +The sum of all fees in the hour, in USD. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/success.md b/models/descriptions/success.md new file mode 100644 index 0000000..6b20598 --- /dev/null +++ b/models/descriptions/success.md @@ -0,0 +1,5 @@ +{% docs success %} + +The boolean value indicating whether the transaction was successful or not. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tables/core__fact_blocks.md b/models/descriptions/tables/core__fact_blocks.md new file mode 100644 index 0000000..a5d2fb5 --- /dev/null +++ b/models/descriptions/tables/core__fact_blocks.md @@ -0,0 +1,8 @@ +{% docs core__fact_blocks %} + +This table contains "block" level data for the m1 blockchain. This table can be used to analyze trends at a block level, for example total transactions over time. +"The m1 blockchain doesn't have an explicit notion of a block — it only uses blocks for batching and executing transactions. +A transaction at height 0 is the first transaction (genesis transaction), and a transaction at height 100 is the 101st transaction in the transaction store." + + +{% enddocs %} diff --git a/models/descriptions/tables/core__fact_changes.md b/models/descriptions/tables/core__fact_changes.md new file mode 100644 index 0000000..3b66c3e --- /dev/null +++ b/models/descriptions/tables/core__fact_changes.md @@ -0,0 +1,5 @@ +{% docs core__fact_changes %} + +This table contains the flattened changes from the transaction. Each change will have a unique change index within a transaction. + +{% enddocs %} diff --git a/models/descriptions/tables/core__fact_events.md b/models/descriptions/tables/core__fact_events.md new file mode 100644 index 0000000..ad7a9d0 --- /dev/null +++ b/models/descriptions/tables/core__fact_events.md @@ -0,0 +1,5 @@ +{% docs core__fact_events %} + +This table contains the flattened events from the transaction. Each event will have a unique event index within a transaction. + +{% enddocs %} diff --git a/models/descriptions/tables/core__fact_transactions.md b/models/descriptions/tables/core__fact_transactions.md new file mode 100644 index 0000000..4ee3daf --- /dev/null +++ b/models/descriptions/tables/core__fact_transactions.md @@ -0,0 +1,6 @@ +{% docs core__fact_transactions %} + +This table contains transaction level data for the Aptos blockchain. Each transaction will have a unique transaction hash and version. +For more information see [docs.movementlabs.xyz docs] Each transaction will have a unique transaction hash + +{% enddocs %} diff --git a/models/descriptions/tables/core__fact_transactions_block_metadata.md b/models/descriptions/tables/core__fact_transactions_block_metadata.md new file mode 100644 index 0000000..819eeac --- /dev/null +++ b/models/descriptions/tables/core__fact_transactions_block_metadata.md @@ -0,0 +1,5 @@ +{% docs core__fact_transactions_block_metadata %} + +These transactions are inserted at the beginning of the block. A BlockMetadata transaction can also mark the end of an epoch and trigger reward distribution to validators. + +{% enddocs %} diff --git a/models/descriptions/tables/core__fact_transactions_state_checkpoint.md b/models/descriptions/tables/core__fact_transactions_state_checkpoint.md new file mode 100644 index 0000000..3b4b995 --- /dev/null +++ b/models/descriptions/tables/core__fact_transactions_state_checkpoint.md @@ -0,0 +1,5 @@ +{% docs core__fact_transactions_state_checkpoint %} + +These transactions are appended at the end of the block and is used as a checkpoint milestone. + +{% enddocs %} diff --git a/models/descriptions/to_address_transfer.md b/models/descriptions/to_address_transfer.md new file mode 100644 index 0000000..6144397 --- /dev/null +++ b/models/descriptions/to_address_transfer.md @@ -0,0 +1,5 @@ +{% docs to_address_transfer %} + +The account address that received the transfer. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/token_address.md b/models/descriptions/token_address.md new file mode 100644 index 0000000..766881f --- /dev/null +++ b/models/descriptions/token_address.md @@ -0,0 +1,5 @@ +{% docs token_address %} + +The full address of the token. This string contains the account,module, and resource. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/transfer_event.md b/models/descriptions/transfer_event.md new file mode 100644 index 0000000..3862b6f --- /dev/null +++ b/models/descriptions/transfer_event.md @@ -0,0 +1,5 @@ +{% docs transfer_event %} + +The type of transfer event. Value will either be 'WithdrawEvent' or 'DepositEvent' + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tx_count.md b/models/descriptions/tx_count.md new file mode 100644 index 0000000..74978b9 --- /dev/null +++ b/models/descriptions/tx_count.md @@ -0,0 +1,5 @@ +{% docs tx_count %} + +The count of transactions in this block. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tx_hash.md b/models/descriptions/tx_hash.md new file mode 100644 index 0000000..a848c23 --- /dev/null +++ b/models/descriptions/tx_hash.md @@ -0,0 +1,5 @@ +{% docs tx_hash %} + +Transaction hash is a unique 66-character identifier that is generated when a transaction is executed. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tx_type.md b/models/descriptions/tx_type.md new file mode 100644 index 0000000..528e1bd --- /dev/null +++ b/models/descriptions/tx_type.md @@ -0,0 +1,5 @@ +{% docs tx_type %} + +The type of the transaction. Values will be one of "block_metadata_transaction","state_checkpoint_transaction","user_transaction". + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/value_change.md b/models/descriptions/value_change.md new file mode 100644 index 0000000..87d41a8 --- /dev/null +++ b/models/descriptions/value_change.md @@ -0,0 +1,5 @@ +{% docs value_change %} + +The value for the write_table_item change + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/version.md b/models/descriptions/version.md new file mode 100644 index 0000000..ac9836e --- /dev/null +++ b/models/descriptions/version.md @@ -0,0 +1,5 @@ +{% docs version %} + +The version also know as the height of the transactions that have been executed on the Aptos blockchain. The first transaction in the blockchain has a version of 0. The version number is incremented by 1 for each transaction that is executed on the blockchain. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/vm_status.md b/models/descriptions/vm_status.md new file mode 100644 index 0000000..1942480 --- /dev/null +++ b/models/descriptions/vm_status.md @@ -0,0 +1,5 @@ +{% docs vm_status %} + +For failed transactions, this fields provides context to why the transaction failed. For successful transactions, this field will be set to `Executed successfully`. + +{% enddocs %} \ No newline at end of file diff --git a/models/evm/bronze/api_udf/bronze_evm_api__contract_abis.sql b/models/evm/bronze/api_udf/bronze_evm_api__contract_abis.sql new file mode 100644 index 0000000..31c285a --- /dev/null +++ b/models/evm/bronze/api_udf/bronze_evm_api__contract_abis.sql @@ -0,0 +1,53 @@ +{{ config( + materialized = 'incremental', + unique_key = "contract_address", + full_refresh = false, + tags = ['noncore'] +) }} + +WITH base AS ( + + SELECT + contract_address + FROM + {{ ref('silver_evm__relevant_contracts') }} + WHERE + total_interaction_count >= 100 + +{% if is_incremental() %} +EXCEPT +SELECT + contract_address +FROM + {{ this }} +{% endif %} +LIMIT + 50 +), row_nos AS ( + SELECT + contract_address, + ROW_NUMBER() over ( + ORDER BY + contract_address + ) AS row_no + FROM + base +), +batched AS ({% for item in range(51) %} +SELECT + rn.contract_address, CONCAT('https://seitrace.com/pacific-1/api/v2/smart-contracts/', contract_address) AS url, IFNULL(live.udf_api(url) :data :abi, ARRAY_CONSTRUCT('ABI unavailable')) AS abi_data, SYSDATE() AS _inserted_timestamp +FROM + row_nos rn +WHERE + row_no = {{ item }} + + {% if not loop.last %} + UNION ALL + {% endif %} +{% endfor %}) +SELECT + contract_address, + abi_data, + _inserted_timestamp +FROM + batched diff --git a/models/evm/bronze/api_udf/bronze_evm_api__contract_abis.yml b/models/evm/bronze/api_udf/bronze_evm_api__contract_abis.yml new file mode 100644 index 0000000..7913879 --- /dev/null +++ b/models/evm/bronze/api_udf/bronze_evm_api__contract_abis.yml @@ -0,0 +1,22 @@ +version: 2 +models: + - name: bronze_evm_api__contract_abis + + columns: + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: CONTRACT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARCHAR + - dbt_expectations.expect_column_values_to_match_regex: + regex: "^(0x)[0-9a-fA-F]{40}$" \ No newline at end of file diff --git a/models/evm/bronze/api_udf/bronze_evm_api__token_reads.sql b/models/evm/bronze/api_udf/bronze_evm_api__token_reads.sql new file mode 100644 index 0000000..1bdd0d2 --- /dev/null +++ b/models/evm/bronze/api_udf/bronze_evm_api__token_reads.sql @@ -0,0 +1,126 @@ +{{ config( + materialized = 'incremental', + unique_key = "contract_address", + full_refresh = false, + tags = ['core','recent_evm_test'] +) }} + +WITH base AS ( + + SELECT + contract_address, + latest_event_block AS latest_block + FROM + {{ ref('silver_evm__relevant_contracts') }} + WHERE + total_event_count >= 25 + +{% if is_incremental() %} +AND contract_address NOT IN ( + SELECT + contract_address + FROM + {{ this }} +) +{% endif %} +ORDER BY + total_event_count DESC +LIMIT + 200 +), function_sigs AS ( + SELECT + '0x313ce567' AS function_sig, + 'decimals' AS function_name + UNION ALL + SELECT + '0x06fdde03', + 'name' + UNION ALL + SELECT + '0x95d89b41', + 'symbol' +), +all_reads AS ( + SELECT + * + FROM + base + JOIN function_sigs + ON 1 = 1 +), +ready_reads AS ( + SELECT + contract_address, + latest_block, + function_sig, + RPAD( + function_sig, + 64, + '0' + ) AS input, + utils.udf_json_rpc_call( + 'eth_call', + [{'to': contract_address, 'from': null, 'data': input}, utils.udf_int_to_hex(latest_block)], + concat_ws( + '-', + contract_address, + input, + latest_block + ) + ) AS rpc_request + FROM + all_reads +), +batch_reads AS ( + SELECT + ARRAY_AGG(rpc_request) AS batch_rpc_request + FROM + ready_reads +), +node_call AS ( + SELECT + *, + live.udf_api( + 'POST', + '{Service}/{Authentication}',{}, + batch_rpc_request, + 'Vault/prod/sei/quicknode/mainnet' + ) AS response + FROM + batch_reads + WHERE + EXISTS ( + SELECT + 1 + FROM + ready_reads + LIMIT + 1 + ) +), flat_responses AS ( + SELECT + VALUE :id :: STRING AS call_id, + VALUE :result :: STRING AS read_result + FROM + node_call, + LATERAL FLATTEN ( + input => response :data + ) +) +SELECT + SPLIT_PART( + call_id, + '-', + 1 + ) AS contract_address, + SPLIT_PART( + call_id, + '-', + 3 + ) AS block_number, + LEFT(SPLIT_PART(call_id, '-', 2), 10) AS function_sig, + NULL AS function_input, + read_result, + SYSDATE() :: TIMESTAMP AS _inserted_timestamp +FROM + flat_responses diff --git a/models/evm/bronze/api_udf/bronze_evm_api__token_reads.yml b/models/evm/bronze/api_udf/bronze_evm_api__token_reads.yml new file mode 100644 index 0000000..1a87770 --- /dev/null +++ b/models/evm/bronze/api_udf/bronze_evm_api__token_reads.yml @@ -0,0 +1,18 @@ +version: 2 +models: + - name: bronze_evm_api__token_reads + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - CONTRACT_ADDRESS + - FUNCTION_SIG + columns: + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/evm/silver/abis/event_logs/silver_evm__complete_event_abis.sql b/models/evm/silver/abis/event_logs/silver_evm__complete_event_abis.sql new file mode 100644 index 0000000..0a1bd07 --- /dev/null +++ b/models/evm/silver/abis/event_logs/silver_evm__complete_event_abis.sql @@ -0,0 +1,216 @@ +{{ config ( + materialized = 'incremental', + unique_key = ['parent_contract_address','event_signature','start_block'], + merge_exclude_columns = ["inserted_timestamp"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['abis'] +) }} + +WITH new_abis AS ( + + SELECT + DISTINCT contract_address + FROM + {{ ref('silver_evm__flat_event_abis') }} + +{% if is_incremental() %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '12 hours' + FROM + {{ this }} + ) +{% endif %} +), +proxies AS ( + SELECT + p0.created_block, + p0.proxy_created_block, + p0.contract_address, + p0.proxy_address, + p0.start_block, + p0._id, + p0._inserted_timestamp + FROM + {{ ref('silver_evm__proxies') }} + p0 + JOIN new_abis na0 + ON p0.contract_address = na0.contract_address + UNION + SELECT + p1.created_block, + p1.proxy_created_block, + p1.contract_address, + p1.proxy_address, + p1.start_block, + p1._id, + p1._inserted_timestamp + FROM + {{ ref('silver_evm__proxies') }} + p1 + JOIN new_abis na1 + ON p1.proxy_address = na1.contract_address +), +all_relevant_contracts AS ( + SELECT + DISTINCT contract_address + FROM + proxies + UNION + SELECT + DISTINCT proxy_address AS contract_address + FROM + proxies + UNION + SELECT + contract_address + FROM + new_abis +), +flat_abis AS ( + SELECT + contract_address, + event_name, + abi, + simple_event_name, + event_signature, + NAME, + inputs, + event_type, + _inserted_timestamp + FROM + {{ ref('silver_evm__flat_event_abis') }} + JOIN all_relevant_contracts USING (contract_address) +), +base AS ( + SELECT + ea.contract_address, + event_name, + abi, + simple_event_name, + event_signature, + NAME, + inputs, + event_type, + ea._inserted_timestamp, + pb._inserted_timestamp AS proxy_inserted_timestamp, + pb.start_block, + pb.proxy_created_block, + pb.contract_address AS base_contract_address, + 1 AS priority + FROM + flat_abis ea + JOIN proxies pb + ON ea.contract_address = pb.proxy_address + UNION ALL + SELECT + eab.contract_address, + event_name, + abi, + simple_event_name, + event_signature, + NAME, + inputs, + event_type, + eab._inserted_timestamp, + pbb._inserted_timestamp AS proxy_inserted_timestamp, + pbb.created_block AS start_block, + pbb.proxy_created_block, + pbb.contract_address AS base_contract_address, + 2 AS priority + FROM + flat_abis eab + JOIN ( + SELECT + DISTINCT contract_address, + created_block, + proxy_created_block, + _inserted_timestamp + FROM + proxies + ) pbb + ON eab.contract_address = pbb.contract_address + UNION ALL + SELECT + contract_address, + event_name, + abi, + simple_event_name, + event_signature, + NAME, + inputs, + event_type, + _inserted_timestamp, + NULL AS proxy_inserted_timestamp, + 0 AS start_block, + NULL AS proxy_created_block, + contract_address AS base_contract_address, + 3 AS priority + FROM + flat_abis eac + WHERE + contract_address NOT IN ( + SELECT + DISTINCT contract_address + FROM + proxies + ) +), +new_records AS ( + SELECT + base_contract_address AS parent_contract_address, + event_name, + abi, + start_block, + proxy_created_block, + simple_event_name, + event_signature, + NAME, + inputs, + event_type, + _inserted_timestamp, + proxy_inserted_timestamp + FROM + base qualify ROW_NUMBER() over ( + PARTITION BY parent_contract_address, + NAME, + event_type, + event_signature, + start_block + ORDER BY + priority ASC, + _inserted_timestamp DESC, + proxy_created_block DESC nulls last, + proxy_inserted_timestamp DESC nulls last + ) = 1 +) +SELECT + parent_contract_address, + event_name, + abi, + start_block, + proxy_created_block, + simple_event_name, + event_signature, + IFNULL(LEAD(start_block) over (PARTITION BY parent_contract_address, event_signature +ORDER BY + start_block) -1, 1e18) AS end_block, + _inserted_timestamp, + proxy_inserted_timestamp, + SYSDATE() AS _updated_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['parent_contract_address','event_signature','start_block'] + ) }} AS complete_event_abis_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + new_records qualify ROW_NUMBER() over ( + PARTITION BY parent_contract_address, + event_name, + event_signature, + start_block + ORDER BY + _inserted_timestamp DESC + ) = 1 diff --git a/models/evm/silver/abis/event_logs/silver_evm__complete_event_abis.yml b/models/evm/silver/abis/event_logs/silver_evm__complete_event_abis.yml new file mode 100644 index 0000000..4a4bdae --- /dev/null +++ b/models/evm/silver/abis/event_logs/silver_evm__complete_event_abis.yml @@ -0,0 +1,9 @@ +version: 2 +models: + - name: silver_evm__complete_event_abis + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - PARENT_CONTRACT_ADDRESS + - EVENT_SIGNATURE + - START_BLOCK \ No newline at end of file diff --git a/models/evm/silver/abis/event_logs/silver_evm__flat_event_abis.sql b/models/evm/silver/abis/event_logs/silver_evm__flat_event_abis.sql new file mode 100644 index 0000000..626f293 --- /dev/null +++ b/models/evm/silver/abis/event_logs/silver_evm__flat_event_abis.sql @@ -0,0 +1,112 @@ +{{ config ( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'contract_address', + cluster_by = '_inserted_timestamp::date', + tags = ['abis'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY (contract_address)" +) }} + +WITH abi_base AS ( + + SELECT + contract_address, + DATA, + _inserted_timestamp + FROM + {{ ref('silver_evm__abis') }} + +{% if is_incremental() %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '24 hours' + FROM + {{ this }} + ) +{% endif %} +), +flat_abi AS ( + SELECT + contract_address, + _inserted_timestamp, + DATA, + VALUE :inputs AS inputs, + VALUE :payable :: BOOLEAN AS payable, + VALUE :stateMutability :: STRING AS stateMutability, + VALUE :type :: STRING AS TYPE, + VALUE :anonymous :: BOOLEAN AS anonymous, + VALUE :name :: STRING AS NAME + FROM + abi_base, + LATERAL FLATTEN ( + input => DATA + ) + WHERE + TYPE = 'event' qualify ROW_NUMBER() over ( + PARTITION BY contract_address, + NAME, + inputs + ORDER BY + LENGTH(inputs) + ) = 1 +), +event_types AS ( + SELECT + contract_address, + _inserted_timestamp, + inputs, + anonymous, + NAME, + ARRAY_AGG( + VALUE :type :: STRING + ) AS event_type + FROM + flat_abi, + LATERAL FLATTEN ( + input => inputs + ) + GROUP BY + contract_address, + _inserted_timestamp, + inputs, + anonymous, + NAME +), +apply_udfs AS ( + SELECT + contract_address, + NAME AS event_name, + PARSE_JSON( + OBJECT_CONSTRUCT( + 'anonymous', + anonymous, + 'inputs', + inputs, + 'name', + NAME, + 'type', + 'event' + ) :: STRING + ) AS abi, + utils.udf_evm_text_signature(abi) AS simple_event_name, + utils.udf_keccak256(simple_event_name) AS event_signature, + NAME, + inputs, + event_type, + _inserted_timestamp + FROM + event_types +) +SELECT + contract_address, + event_name, + abi, + simple_event_name, + event_signature, + NAME, + inputs, + event_type, + _inserted_timestamp +FROM + apply_udfs diff --git a/models/evm/silver/abis/silver_evm__abis.sql b/models/evm/silver/abis/silver_evm__abis.sql new file mode 100644 index 0000000..c999af4 --- /dev/null +++ b/models/evm/silver/abis/silver_evm__abis.sql @@ -0,0 +1,181 @@ +{{ config ( + materialized = "incremental", + unique_key = "contract_address", + merge_exclude_columns = ["inserted_timestamp"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(contract_address,abi_hash,bytecode), SUBSTRING(contract_address,abi_hash,bytecode)", + tags = ['abis'] +) }} + +WITH override_abis AS ( + + SELECT + contract_address, + PARSE_JSON(DATA) AS abi, + TO_TIMESTAMP_LTZ(SYSDATE()) AS _inserted_timestamp, + 'flipside' AS abi_source, + 'flipside' AS discord_username, + SHA2(abi) AS abi_hash, + 1 AS priority + FROM + {{ ref('silver_evm__override_abis') }} + WHERE + contract_address IS NOT NULL +), +verified_abis AS ( + SELECT + contract_address, + DATA, + _inserted_timestamp, + abi_source, + discord_username, + abi_hash, + 2 AS priority + FROM + {{ ref('silver_evm__verified_abis') }} + WHERE + abi_source = 'seitrace' + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} + WHERE + abi_source = 'seitrace' +) +{% endif %} +), +user_abis AS ( + SELECT + contract_address, + DATA, + _inserted_timestamp, + abi_source, + discord_username, + abi_hash, + 2 AS priority + FROM + {{ ref('silver_evm__verified_abis') }} + WHERE + abi_source = 'user' + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} + WHERE + abi_source = 'user' +) +{% endif %} +), +bytecode_abis AS ( + SELECT + contract_address, + abi, + abi_hash, + 'bytecode_matched' AS abi_source, + NULL AS discord_username, + _inserted_timestamp, + 3 AS priority + FROM + {{ ref('silver_evm__bytecode_abis') }} + WHERE + 1 = 1 + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} + WHERE + abi_source = 'bytecode_matched' +) +{% endif %} +), +all_abis AS ( + SELECT + contract_address, + abi AS DATA, + _inserted_timestamp, + abi_source, + discord_username, + abi_hash, + priority + FROM + override_abis + UNION + SELECT + contract_address, + DATA, + _inserted_timestamp, + abi_source, + discord_username, + abi_hash, + priority + FROM + verified_abis + UNION + SELECT + contract_address, + DATA, + _inserted_timestamp, + abi_source, + discord_username, + abi_hash, + priority + FROM + user_abis + UNION + SELECT + contract_address, + abi AS DATA, + _inserted_timestamp, + abi_source, + discord_username, + abi_hash, + priority + FROM + bytecode_abis +), +priority_abis AS ( + SELECT + contract_address, + DATA, + _inserted_timestamp, + abi_source, + discord_username, + abi_hash, + priority + FROM + all_abis qualify(ROW_NUMBER() over(PARTITION BY contract_address + ORDER BY + priority ASC)) = 1 +) +SELECT + p.contract_address, + p.data, + p._inserted_timestamp, + p.abi_source, + p.discord_username, + p.abi_hash, + created_contract_input AS bytecode, + {{ dbt_utils.generate_surrogate_key( + ['contract_address'] + ) }} AS abis_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + priority_abis p + LEFT JOIN {{ ref('silver_evm__created_contracts') }} + ON p.contract_address = created_contract_address diff --git a/models/evm/silver/abis/silver_evm__abis.yml b/models/evm/silver/abis/silver_evm__abis.yml new file mode 100644 index 0000000..311202e --- /dev/null +++ b/models/evm/silver/abis/silver_evm__abis.yml @@ -0,0 +1,8 @@ + +version: 2 +models: + - name: silver_evm__abis + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - CONTRACT_ADDRESS \ No newline at end of file diff --git a/models/evm/silver/abis/silver_evm__bytecode_abis.sql b/models/evm/silver/abis/silver_evm__bytecode_abis.sql new file mode 100644 index 0000000..57c4cae --- /dev/null +++ b/models/evm/silver/abis/silver_evm__bytecode_abis.sql @@ -0,0 +1,76 @@ +{{ config ( + materialized = "incremental", + unique_key = "contract_address", + tags = ['abis'] +) }} + +WITH contracts_with_abis AS ( + -- Identifying contracts with verified ABIs + + SELECT + created_contract_address AS contract_address + FROM + {{ ref('silver_evm__created_contracts') }} + JOIN {{ ref('silver_evm__verified_abis') }} A + ON A.contract_address = created_contract_address +), +contracts_without_abis AS ( + -- Contracts that are missing ABIs + SELECT + created_contract_address AS contract_address, + created_contract_input AS bytecode + FROM + {{ ref('silver_evm__created_contracts') }} + WHERE + created_contract_address NOT IN ( + SELECT + contract_address + FROM + contracts_with_abis + ) + +{% if is_incremental() %} +AND created_contract_address NOT IN ( + SELECT + contract_address + FROM + {{ this }} +) +{% endif %} +), +unique_bytecode_abis AS ( + -- Bytecodes from created_contracts with a unique ABI + SELECT + cc.created_contract_input AS bytecode, + va.data AS abi, + va.abi_hash + FROM + {{ ref('silver_evm__created_contracts') }} + cc + JOIN {{ ref('silver_evm__verified_abis') }} + va + ON cc.created_contract_address = va.contract_address + GROUP BY + cc.created_contract_input, + va.data, + va.abi_hash + HAVING + COUNT( + DISTINCT va.data + ) = 1 -- Ensuring there's only one ABI per bytecode +) -- Final matching +SELECT + contract_address, + abi, + abi_hash, + +{% if is_incremental() %} +SYSDATE() +{% else %} + TO_TIMESTAMP_NTZ('2000-01-01 00:00:00') +{% endif %} + +AS _inserted_timestamp +FROM + contracts_without_abis + JOIN unique_bytecode_abis USING (bytecode) diff --git a/models/evm/silver/abis/silver_evm__bytecode_abis.yml b/models/evm/silver/abis/silver_evm__bytecode_abis.yml new file mode 100644 index 0000000..ed154ad --- /dev/null +++ b/models/evm/silver/abis/silver_evm__bytecode_abis.yml @@ -0,0 +1,8 @@ +version: 2 +models: + - name: silver_evm__bytecode_abis + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - CONTRACT_ADDRESS + - ABI_HASH \ No newline at end of file diff --git a/models/evm/silver/abis/silver_evm__override_abis.sql b/models/evm/silver/abis/silver_evm__override_abis.sql new file mode 100644 index 0000000..88b8439 --- /dev/null +++ b/models/evm/silver/abis/silver_evm__override_abis.sql @@ -0,0 +1,8 @@ +{{ config( + materialized = 'view', + tags = ['abis'] +) }} + +SELECT + NULL AS contract_address, + NULL AS DATA diff --git a/models/evm/silver/abis/silver_evm__proxies.sql b/models/evm/silver/abis/silver_evm__proxies.sql new file mode 100644 index 0000000..6cb696f --- /dev/null +++ b/models/evm/silver/abis/silver_evm__proxies.sql @@ -0,0 +1,104 @@ +{{ config ( + materialized = 'incremental', + unique_key = ['contract_address','proxy_address'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['abis'] +) }} + +WITH base AS ( + + SELECT + from_address, + to_address, + MIN(block_number) AS start_block, + MAX(_inserted_timestamp) AS _inserted_timestamp + FROM + {{ ref('silver_evm__traces') }} + WHERE + TYPE = 'DELEGATECALL' + AND trace_status = 'SUCCESS' + AND tx_status = 'SUCCESS' + AND from_address != to_address -- exclude self-calls + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '24 hours' + FROM + {{ this }} +) +{% endif %} +GROUP BY + from_address, + to_address +), +create_id AS ( + SELECT + from_address AS contract_address, + to_address AS proxy_address, + start_block, + CONCAT( + from_address, + '-', + to_address + ) AS _id, + _inserted_timestamp + FROM + base +), +heal AS ( + SELECT + contract_address, + proxy_address, + start_block, + _id, + _inserted_timestamp + FROM + create_id + +{% if is_incremental() %} +UNION ALL +SELECT + contract_address, + proxy_address, + start_block, + _id, + _inserted_timestamp +FROM + {{ this }} + JOIN create_id USING ( + contract_address, + proxy_address + ) +{% endif %} +), +FINAL AS ( + SELECT + contract_address, + proxy_address, + start_block, + _id, + _inserted_timestamp + FROM + heal qualify ROW_NUMBER() over ( + PARTITION BY contract_address, + proxy_address + ORDER BY + start_block ASC + ) = 1 +) +SELECT + f.contract_address, + f.proxy_address, + f.start_block, + f._id, + f._inserted_timestamp, + C.block_number AS created_block, + p.block_number AS proxy_created_block +FROM + FINAL f + JOIN {{ ref('silver_evm__created_contracts') }} C + ON f.contract_address = C.created_contract_address + JOIN {{ ref('silver_evm__created_contracts') }} + p + ON f.proxy_address = p.created_contract_address diff --git a/models/evm/silver/abis/silver_evm__proxies.yml b/models/evm/silver/abis/silver_evm__proxies.yml new file mode 100644 index 0000000..be9791a --- /dev/null +++ b/models/evm/silver/abis/silver_evm__proxies.yml @@ -0,0 +1,7 @@ +version: 2 +models: + - name: silver_evm__proxies + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _ID \ No newline at end of file diff --git a/models/evm/silver/abis/silver_evm__user_verified_abis.sql b/models/evm/silver/abis/silver_evm__user_verified_abis.sql new file mode 100644 index 0000000..f73c178 --- /dev/null +++ b/models/evm/silver/abis/silver_evm__user_verified_abis.sql @@ -0,0 +1,550 @@ +{{ config ( + materialized = "incremental", + unique_key = "id", + tags = ['abis'] +) }} + +WITH base AS ( + + SELECT + contract_address, + abi, + PARSE_JSON(abi) AS DATA, + SHA2(PARSE_JSON(abi)) AS abi_hash, + discord_username, + _inserted_timestamp + FROM + {{ source( + "crosschain_public", + "user_abis" + ) }} + WHERE + blockchain = 'sei' + AND NOT duplicate_abi + +{% if is_incremental() %} +AND contract_address NOT IN ( + SELECT + contract_address + FROM + {{ this }} +) +AND _inserted_timestamp > ( + SELECT + COALESCE( + MAX( + _inserted_timestamp + ), + '1970-01-01' + ) + FROM + {{ this }} +) +{% endif %} +ORDER BY + _inserted_timestamp ASC +LIMIT + 10 +), flat_event_abi AS ( + SELECT + contract_address, + _inserted_timestamp, + DATA, + VALUE :inputs AS inputs, + VALUE :payable :: BOOLEAN AS payable, + VALUE :stateMutability :: STRING AS stateMutability, + VALUE :type :: STRING AS TYPE, + VALUE :anonymous :: BOOLEAN AS anonymous, + VALUE :name :: STRING AS NAME + FROM + base, + LATERAL FLATTEN ( + input => DATA + ) + WHERE + TYPE = 'event' qualify ROW_NUMBER() over ( + PARTITION BY contract_address, + NAME, + inputs + ORDER BY + LENGTH(inputs) + ) = 1 +), +event_types AS ( + SELECT + contract_address, + _inserted_timestamp, + inputs, + anonymous, + NAME, + ARRAY_AGG( + VALUE :type :: STRING + ) AS event_type + FROM + flat_event_abi, + LATERAL FLATTEN ( + input => inputs + ) + GROUP BY + contract_address, + _inserted_timestamp, + inputs, + anonymous, + NAME +), +apply_event_udfs AS ( + SELECT + contract_address, + NAME AS event_name, + PARSE_JSON( + OBJECT_CONSTRUCT( + 'anonymous', + anonymous, + 'inputs', + inputs, + 'name', + NAME, + 'type', + 'event' + ) :: STRING + ) AS abi, + utils.udf_evm_text_signature(abi) AS simple_event_name, + utils.udf_keccak256(simple_event_name) AS event_signature, + NAME, + inputs, + event_type, + _inserted_timestamp + FROM + event_types +), +final_flat_event_abis AS ( + SELECT + contract_address, + event_name, + abi, + simple_event_name, + event_signature, + NAME, + inputs, + event_type, + _inserted_timestamp + FROM + apply_event_udfs +), +flat_function_abis AS ( + SELECT + contract_address, + DATA, + _inserted_timestamp, + VALUE :inputs AS inputs, + VALUE :outputs AS outputs, + VALUE :payable :: BOOLEAN AS payable, + VALUE :stateMutability :: STRING AS stateMutability, + VALUE :type :: STRING AS TYPE, + VALUE :name :: STRING AS NAME + FROM + base, + LATERAL FLATTEN ( + input => DATA + ) + WHERE + TYPE = 'function' +), +udf_function_abis AS ( + SELECT + *, + PARSE_JSON( + object_construct_keep_null( + 'inputs', + IFNULL( + inputs, + [] + ), + 'outputs', + IFNULL( + outputs, + [] + ), + 'name', + NAME, + 'type', + 'function' + ) :: STRING + ) AS abi, + utils.udf_evm_text_signature(abi) AS simple_function_name, + utils.udf_keccak256(simple_function_name) AS function_signature + FROM + flat_function_abis qualify ROW_NUMBER() over ( + PARTITION BY contract_address, + function_signature + ORDER BY + _inserted_timestamp DESC + ) = 1 +), +flat_inputs AS ( + SELECT + contract_address, + inputs, + NAME, + simple_function_name, + function_signature, + ARRAY_AGG( + VALUE :type :: STRING + ) AS inputs_type + FROM + udf_function_abis, + LATERAL FLATTEN ( + input => inputs + ) + GROUP BY + ALL +), +fill_missing_input_names AS ( + SELECT + contract_address, + NAME, + inputs_type, + simple_function_name, + function_signature, + VALUE :internalType :: STRING AS internalType, + VALUE :type :: STRING AS TYPE, + CASE + WHEN VALUE :name :: STRING = '' THEN CONCAT('input_', ROW_NUMBER() over (PARTITION BY contract_address, function_signature + ORDER BY + INDEX ASC) :: STRING) + ELSE VALUE :name :: STRING + END AS name_fixed, + inputs, + INDEX, + VALUE :components AS components + FROM + flat_inputs, + LATERAL FLATTEN ( + input => inputs + ) +), +final_flat_inputs AS ( + SELECT + contract_address, + NAME, + inputs_type, + simple_function_name, + function_signature, + ARRAY_AGG( + OBJECT_CONSTRUCT( + 'internalType', + internalType, + 'name', + name_fixed, + 'type', + TYPE, + 'components', + components + ) + ) within GROUP ( + ORDER BY + INDEX + ) AS inputs + FROM + fill_missing_input_names + GROUP BY + ALL +), +flat_outputs AS ( + SELECT + contract_address, + outputs, + simple_function_name, + function_signature, + NAME, + ARRAY_AGG( + VALUE :type :: STRING + ) AS outputs_type + FROM + udf_function_abis, + LATERAL FLATTEN ( + input => outputs + ) + GROUP BY + ALL +), +fill_missing_output_names AS ( + SELECT + contract_address, + NAME, + outputs_type, + simple_function_name, + function_signature, + VALUE :internalType :: STRING AS internalType, + VALUE :type :: STRING AS TYPE, + CASE + WHEN VALUE :name :: STRING = '' THEN CONCAT('output_', ROW_NUMBER() over (PARTITION BY contract_address, function_signature + ORDER BY + INDEX ASC) :: STRING) + ELSE VALUE :name :: STRING + END AS name_fixed, + outputs, + INDEX, + VALUE :components AS components + FROM + flat_outputs, + LATERAL FLATTEN ( + input => outputs + ) +), +final_flat_outputs AS ( + SELECT + contract_address, + NAME, + outputs_type, + simple_function_name, + function_signature, + ARRAY_AGG( + OBJECT_CONSTRUCT( + 'internalType', + internalType, + 'name', + name_fixed, + 'type', + TYPE, + 'components', + components + ) + ) within GROUP ( + ORDER BY + INDEX + ) AS outputs + FROM + fill_missing_output_names + GROUP BY + ALL +), +all_contracts AS ( + SELECT + A.contract_address, + A.name AS function_name, + i.inputs, + o.outputs, + i.inputs_type, + o.outputs_type, + A._inserted_timestamp, + A.function_signature, + A.simple_function_name + FROM + udf_function_abis A + LEFT JOIN final_flat_inputs i + ON A.contract_address = i.contract_address + AND A.function_signature = i.function_signature + LEFT JOIN final_flat_outputs o + ON A.contract_address = o.contract_address + AND A.function_signature = o.function_signature +), +apply_function_udfs AS ( + SELECT + contract_address, + function_name, + PARSE_JSON( + object_construct_keep_null( + 'inputs', + IFNULL( + inputs, + [] + ), + 'outputs', + IFNULL( + outputs, + [] + ), + 'name', + function_name, + 'type', + 'function' + ) :: STRING + ) AS abi, + simple_function_name, + function_signature, + inputs, + outputs, + inputs_type, + outputs_type, + _inserted_timestamp + FROM + all_contracts +), +final_function_abis AS ( + SELECT + contract_address, + function_name, + abi, + simple_function_name, + function_signature, + inputs, + outputs, + inputs_type, + outputs_type, + _inserted_timestamp + FROM + apply_function_udfs +), +new_abis AS ( + SELECT + DISTINCT contract_address + FROM + base +), +contracts AS ( + SELECT + contract_address + FROM + {{ ref('silver_evm__proxies') }} + JOIN new_abis USING (contract_address) +), +proxies AS ( + SELECT + p.proxy_address, + p.contract_address + FROM + {{ ref('silver_evm__proxies') }} + p + JOIN new_abis n + ON p.proxy_address = n.contract_address +), +final_groupings AS ( + SELECT + b.contract_address AS address, + C.contract_address, + proxy_address, + CASE + WHEN C.contract_address IS NOT NULL + AND proxy_address IS NOT NULL THEN 'contract' + WHEN C.contract_address IS NOT NULL THEN 'contract' + WHEN proxy_address IS NOT NULL THEN 'proxy' + WHEN C.contract_address IS NULL + AND proxy_address IS NULL THEN 'contract' + END AS TYPE, + p.contract_address AS proxy_parent, + CASE + WHEN TYPE = 'contract' THEN address + ELSE proxy_parent + END AS final_address + FROM + base b + LEFT JOIN ( + SELECT + DISTINCT contract_address + FROM + contracts + ) C + ON b.contract_address = C.contract_address + LEFT JOIN ( + SELECT + DISTINCT proxy_address, + contract_address + FROM + proxies + ) p + ON b.contract_address = proxy_address +), +identified_addresses AS ( + SELECT + DISTINCT address AS base_address, + final_address AS contract_address + FROM + final_groupings +), +function_mapping AS ( + SELECT + ia.base_address, + ia.contract_address, + LEFT( + function_signature, + 10 + ) AS function_sig + FROM + identified_addresses ia + JOIN final_function_abis ffa + ON ia.base_address = ffa.contract_address +), +valid_traces AS ( + SELECT + DISTINCT base_address + FROM + ( + SELECT + base_address + FROM + {{ ref('silver_evm__traces') }} + JOIN function_mapping + ON function_sig = LEFT( + input, + 10 + ) + AND IFF( + TYPE = 'DELEGATECALL', + from_address, + to_address + ) = contract_address + WHERE + block_timestamp > DATEADD('month', -12, SYSDATE()) + LIMIT + 50000) + ), event_mapping AS ( + SELECT + ia.base_address, + ia.contract_address, + event_signature + FROM + identified_addresses ia + JOIN final_flat_event_abis fea + ON ia.base_address = fea.contract_address + ), + valid_logs AS ( + SELECT + DISTINCT base_address + FROM + ( + SELECT + base_address + FROM + {{ ref('silver_evm__logs') }} + l + JOIN event_mapping ia + ON ia.contract_address = l.contract_address + AND event_signature = topics [0] :: STRING + WHERE + block_timestamp > DATEADD('month', -12, SYSDATE()) + LIMIT + 50000) + ), all_valid_addresses AS ( + SELECT + base_address + FROM + valid_traces + UNION + SELECT + base_address + FROM + valid_logs + ) + SELECT + contract_address, + abi, + discord_username, + _inserted_timestamp, + abi_hash, + CONCAT( + contract_address, + '-', + abi_hash + ) AS id + FROM + base + WHERE + contract_address IN ( + SELECT + base_address + FROM + all_valid_addresses + ) qualify(ROW_NUMBER() over(PARTITION BY contract_address + ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/evm/silver/abis/silver_evm__user_verified_abis.yml b/models/evm/silver/abis/silver_evm__user_verified_abis.yml new file mode 100644 index 0000000..23e6846 --- /dev/null +++ b/models/evm/silver/abis/silver_evm__user_verified_abis.yml @@ -0,0 +1,7 @@ +version: 2 +models: + - name: silver_evm__user_verified_abis + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ID \ No newline at end of file diff --git a/models/evm/silver/abis/silver_evm__verified_abis.sql b/models/evm/silver/abis/silver_evm__verified_abis.sql new file mode 100644 index 0000000..c3f5fd0 --- /dev/null +++ b/models/evm/silver/abis/silver_evm__verified_abis.sql @@ -0,0 +1,108 @@ +{{ config( + materialized = 'incremental', + unique_key = "contract_address", + tags = ['abis'] +) }} + +WITH base AS ( + + SELECT + contract_address, + PARSE_JSON( + abi_data + ) AS DATA, + _inserted_timestamp + FROM + {{ ref('bronze_evm_api__contract_abis') }} + WHERE + abi_data [0] :: STRING <> 'ABI unavailable' + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + COALESCE( + MAX( + _inserted_timestamp + ), + '1970-01-01' + ) + FROM + {{ this }} +) +{% endif %} +), +sei_trace_abis AS ( + SELECT + contract_address, + DATA, + _inserted_timestamp, + 'seitrace' AS abi_source + FROM + base +), +user_abis AS ( + SELECT + contract_address, + abi, + discord_username, + _inserted_timestamp, + 'user' AS abi_source, + abi_hash + FROM + {{ ref('silver_evm__user_verified_abis') }} + +{% if is_incremental() %} +WHERE + _inserted_timestamp >= ( + SELECT + COALESCE( + MAX( + _inserted_timestamp + ), + '1970-01-01' + ) + FROM + {{ this }} + WHERE + abi_source = 'user' + ) + AND contract_address NOT IN ( + SELECT + contract_address + FROM + {{ this }} + ) +{% endif %} +), +all_abis AS ( + SELECT + contract_address, + DATA, + _inserted_timestamp, + abi_source, + NULL AS discord_username, + SHA2(DATA) AS abi_hash + FROM + sei_trace_abis + UNION + SELECT + contract_address, + PARSE_JSON(abi) AS DATA, + _inserted_timestamp, + 'user' AS abi_source, + discord_username, + abi_hash + FROM + user_abis +) +SELECT + contract_address, + DATA, + _inserted_timestamp, + abi_source, + discord_username, + abi_hash +FROM + all_abis qualify(ROW_NUMBER() over(PARTITION BY contract_address +ORDER BY + _INSERTED_TIMESTAMP DESC)) = 1 diff --git a/models/evm/silver/abis/silver_evm__verified_abis.yml b/models/evm/silver/abis/silver_evm__verified_abis.yml new file mode 100644 index 0000000..f990c7d --- /dev/null +++ b/models/evm/silver/abis/silver_evm__verified_abis.yml @@ -0,0 +1,7 @@ +version: 2 +models: + - name: silver_evm__verified_abis + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - CONTRACT_ADDRESS \ No newline at end of file diff --git a/models/evm/silver/core/silver_evm__blocks.sql b/models/evm/silver/core/silver_evm__blocks.sql new file mode 100644 index 0000000..b9ef07c --- /dev/null +++ b/models/evm/silver/core/silver_evm__blocks.sql @@ -0,0 +1,81 @@ +-- depends_on: {{ ref('bronze_evm__streamline_blocks') }} +{{ config( + materialized = 'incremental', + unique_key = "block_number", + cluster_by = "block_timestamp::date", + tags = ['core'], + merge_exclude_columns = ["inserted_timestamp"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(hash,parent_hash,receipts_root,sha3_uncles,state_root,transactions_root)", +) }} + +SELECT + DATA, + VALUE :BLOCK_NUMBER :: INT AS block_number, + utils.udf_hex_to_int( + DATA :result :baseFeePerGas :: STRING + ) :: INT AS base_fee_per_gas, + utils.udf_hex_to_int( + DATA :result :difficulty :: STRING + ) :: INT AS difficulty, + DATA :result :extraData :: STRING AS extra_data, + utils.udf_hex_to_int( + DATA :result :gasLimit :: STRING + ) :: INT AS gas_limit, + utils.udf_hex_to_int( + DATA :result :gasUsed :: STRING + ) :: INT AS gas_used, + DATA :result :hash :: STRING AS HASH, + DATA :result :logsBloom :: STRING AS logs_bloom, + DATA :result :miner :: STRING AS miner, + DATA :result :mixHash :: STRING AS mixHash, + utils.udf_hex_to_int( + DATA :result :nonce :: STRING + ) :: INT AS nonce, + utils.udf_hex_to_int( + DATA :result :number :: STRING + ) :: INT AS NUMBER, + DATA :result :parentHash :: STRING AS parent_hash, + DATA :result :receiptsRoot :: STRING AS receipts_root, + DATA :result :sha3Uncles :: STRING AS sha3_uncles, + utils.udf_hex_to_int( + DATA :result :size :: STRING + ) :: INT AS SIZE, + DATA :result :stateRoot :: STRING AS state_root, + utils.udf_hex_to_int( + DATA :result :timestamp :: STRING + ) :: TIMESTAMP AS block_timestamp, + IFNULL( + utils.udf_hex_to_int( + DATA :result :totalDifficulty :: STRING + ) :: INT, + 0 + ) AS total_difficulty, + ARRAY_SIZE( + DATA :result :transactions + ) AS tx_count, + DATA :result :transactionsRoot :: STRING AS transactions_root, + DATA :result :uncles AS uncles, + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS blocks_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__streamline_blocks') }} +WHERE + inserted_timestamp >= ( + SELECT + MAX(inserted_timestamp) inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze_evm__streamline_FR_blocks') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number +ORDER BY + inserted_timestamp DESC)) = 1 diff --git a/models/evm/silver/core/silver_evm__confirmed_blocks.sql b/models/evm/silver/core/silver_evm__confirmed_blocks.sql new file mode 100644 index 0000000..57c6719 --- /dev/null +++ b/models/evm/silver/core/silver_evm__confirmed_blocks.sql @@ -0,0 +1,55 @@ +-- depends_on: {{ ref('bronze_evm__streamline_confirm_blocks') }} +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = "round(block_number,-3)", + tags = ['core'] +) }} + +WITH base AS ( + + SELECT + VALUE :BLOCK_NUMBER :: INT AS block_number, + DATA :result :hash :: STRING AS block_hash, + DATA :result :transactions txs, + inserted_timestamp + FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__streamline_confirm_blocks') }} +WHERE + inserted_timestamp >= ( + SELECT + IFNULL( + MAX( + inserted_timestamp + ), + '1970-01-01' :: TIMESTAMP + ) inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze_evm__streamline_FR_confirm_blocks') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number +ORDER BY + inserted_timestamp DESC)) = 1 +) +SELECT + block_number, + block_hash, + VALUE :: STRING AS tx_hash, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS confirmed_blocks_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + base, + LATERAL FLATTEN ( + input => txs + ) diff --git a/models/evm/silver/core/silver_evm__contracts.sql b/models/evm/silver/core/silver_evm__contracts.sql new file mode 100644 index 0000000..c1dbef4 --- /dev/null +++ b/models/evm/silver/core/silver_evm__contracts.sql @@ -0,0 +1,103 @@ +{{ config( + materialized = 'incremental', + unique_key = 'contract_address', + merge_exclude_columns = ["inserted_timestamp"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(contract_address, token_symbol, token_name), SUBSTRING(contract_address, token_symbol, token_name)", + tags = ['core','recent_evm_test'] +) }} + +WITH base_metadata AS ( + + SELECT + contract_address, + block_number, + function_sig AS function_signature, + read_result AS read_output, + _inserted_timestamp + FROM + {{ ref('bronze_evm_api__token_reads') }} + WHERE + read_result IS NOT NULL + AND read_result <> '0x' + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} +) +{% endif %} +), +token_names AS ( + SELECT + contract_address, + block_number, + function_signature, + read_output, + utils.udf_hex_to_string(SUBSTR(read_output,(64 * 2 + 3), len(read_output))) AS token_name + FROM + base_metadata + WHERE + function_signature = '0x06fdde03' + AND token_name IS NOT NULL), + token_symbols AS ( + SELECT + contract_address, + block_number, + function_signature, + read_output, + utils.udf_hex_to_string(SUBSTR(read_output,(64 * 2 + 3), len(read_output))) AS token_symbol + FROM + base_metadata + WHERE + function_signature = '0x95d89b41' + AND token_symbol IS NOT NULL), + token_decimals AS ( + SELECT + contract_address, + utils.udf_hex_to_int( + read_output :: STRING + ) AS token_decimals, + LENGTH(token_decimals) AS dec_length + FROM + base_metadata + WHERE + function_signature = '0x313ce567' + AND read_output IS NOT NULL + AND read_output <> '0x' + ), + contracts AS ( + SELECT + contract_address, + MAX(_inserted_timestamp) AS _inserted_timestamp + FROM + base_metadata + GROUP BY + 1 + ) + SELECT + c1.contract_address AS contract_address, + token_name, + token_decimals :: INTEGER AS token_decimals, + token_symbol, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['c1.contract_address'] + ) }} AS contracts_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id + FROM + contracts c1 + LEFT JOIN token_names + ON c1.contract_address = token_names.contract_address + LEFT JOIN token_symbols + ON c1.contract_address = token_symbols.contract_address + LEFT JOIN token_decimals + ON c1.contract_address = token_decimals.contract_address + AND dec_length < 3 qualify(ROW_NUMBER() over(PARTITION BY c1.contract_address + ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/evm/silver/core/silver_evm__contracts.yml b/models/evm/silver/core/silver_evm__contracts.yml new file mode 100644 index 0000000..b721e88 --- /dev/null +++ b/models/evm/silver/core/silver_evm__contracts.yml @@ -0,0 +1,24 @@ +version: 2 +models: + - name: silver_evm__contracts + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - CONTRACT_ADDRESS + columns: + - name: CONTRACT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ diff --git a/models/evm/silver/core/silver_evm__created_contracts.sql b/models/evm/silver/core/silver_evm__created_contracts.sql new file mode 100644 index 0000000..e41569b --- /dev/null +++ b/models/evm/silver/core/silver_evm__created_contracts.sql @@ -0,0 +1,44 @@ +{{ config ( + materialized = "incremental", + unique_key = "created_contract_address", + merge_exclude_columns = ["inserted_timestamp"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(block_timestamp, tx_hash, created_contract_address, creator_address), SUBSTRING(created_contract_address, creator_address)", + tags = ['recent_evm_test','core'] +) }} + +SELECT + block_number, + block_timestamp, + tx_hash, + to_address AS created_contract_address, + from_address AS creator_address, + input AS created_contract_input, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['to_address'] + ) }} AS created_contracts_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('silver_evm__traces') }} +WHERE + TYPE ILIKE 'create%' + AND to_address IS NOT NULL + AND input IS NOT NULL + AND input != '0x' + AND tx_status = 'SUCCESS' + AND trace_status = 'SUCCESS' + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '24 hours' + FROM + {{ this }} +) +{% endif %} + +qualify(ROW_NUMBER() over(PARTITION BY created_contract_address +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/evm/silver/core/silver_evm__created_contracts.yml b/models/evm/silver/core/silver_evm__created_contracts.yml new file mode 100644 index 0000000..a654ccc --- /dev/null +++ b/models/evm/silver/core/silver_evm__created_contracts.yml @@ -0,0 +1,29 @@ +version: 2 +models: + - name: silver_evm__created_contracts + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - created_contract_address + columns: + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 3 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - TIMESTAMP_LTZ + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 3 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - TIMESTAMP_LTZ + \ No newline at end of file diff --git a/models/evm/silver/core/silver_evm__logs.sql b/models/evm/silver/core/silver_evm__logs.sql new file mode 100644 index 0000000..711dfb4 --- /dev/null +++ b/models/evm/silver/core/silver_evm__logs.sql @@ -0,0 +1,188 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = "block_timestamp::date, _inserted_timestamp::date", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['core'] +) }} + +WITH base AS ( + + SELECT + block_number, + tx_hash, + from_address AS origin_from_address, + to_address AS origin_to_address, + tx_status, + logs, + _inserted_timestamp + FROM + {{ ref('silver_evm__receipts') }} + WHERE + ARRAY_SIZE(logs) > 0 + +{% if is_incremental() %} +AND _INSERTED_TIMESTAMP >= ( + SELECT + MAX(_INSERTED_TIMESTAMP) _INSERTED_TIMESTAMP + FROM + {{ this }} +) +{% endif %} +), +flat_logs AS ( + SELECT + block_number, + tx_hash, + origin_from_address, + origin_to_address, + tx_status, + VALUE :address :: STRING AS contract_address, + VALUE :blockHash :: STRING AS block_hash, + VALUE :data :: STRING AS DATA, + utils.udf_hex_to_int( + VALUE :logIndex :: STRING + ) :: INT AS event_index, + VALUE :removed :: BOOLEAN AS event_removed, + VALUE :topics AS topics, + _inserted_timestamp + FROM + base, + LATERAL FLATTEN( + input => logs + ) +), +new_records AS ( + SELECT + l.block_number, + txs.block_timestamp, + l.tx_hash, + l.origin_from_address, + l.origin_to_address, + txs.origin_function_signature, + l.tx_status, + l.contract_address, + l.block_hash, + l.data, + l.event_index, + l.event_removed, + l.topics, + l._inserted_timestamp, + CASE + WHEN txs.block_timestamp IS NULL + OR txs.origin_function_signature IS NULL THEN TRUE + ELSE FALSE + END AS is_pending, + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id + FROM + flat_logs l + LEFT OUTER JOIN {{ ref('silver_evm__transactions') }} + txs + ON l.block_number = txs.block_number + AND l.tx_hash = txs.tx_hash + +{% if is_incremental() %} +AND txs._INSERTED_TIMESTAMP >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 1 + FROM + {{ this }} +) +{% endif %} +) + +{% if is_incremental() %}, +missing_data AS ( + SELECT + t.block_number, + txs.block_timestamp, + t.tx_hash, + t.origin_from_address, + t.origin_to_address, + txs.origin_function_signature, + t.tx_status, + t.contract_address, + t.block_hash, + t.data, + t.event_index, + t.event_removed, + t.topics, + GREATEST( + t._inserted_timestamp, + txs._inserted_timestamp + ) AS _inserted_timestamp, + _log_id, + FALSE AS is_pending + FROM + {{ this }} + t + INNER JOIN {{ ref('silver_evm__transactions') }} + txs USING ( + block_number, + tx_hash + ) + WHERE + t.is_pending +) +{% endif %}, +FINAL AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + origin_from_address, + origin_to_address, + origin_function_signature, + tx_status, + contract_address, + block_hash, + DATA, + event_index, + event_removed, + topics, + _inserted_timestamp, + _log_id, + is_pending + FROM + new_records + +{% if is_incremental() %} +UNION +SELECT + block_number, + block_timestamp, + tx_hash, + origin_from_address, + origin_to_address, + origin_function_signature, + tx_status, + contract_address, + block_hash, + DATA, + event_index, + event_removed, + topics, + _inserted_timestamp, + _log_id, + is_pending +FROM + missing_data +{% endif %} +) +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} AS logs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_hash, event_index +ORDER BY + _inserted_timestamp DESC, is_pending ASC)) = 1 diff --git a/models/evm/silver/core/silver_evm__native_transfers.sql b/models/evm/silver/core/silver_evm__native_transfers.sql new file mode 100644 index 0000000..d6b4b10 --- /dev/null +++ b/models/evm/silver/core/silver_evm__native_transfers.sql @@ -0,0 +1,110 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'block_number', + cluster_by = ['block_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['core'] +) }} + +WITH base AS ( + + SELECT + tx_hash, + block_number, + block_timestamp, + from_address, + to_address, + VALUE, + identifier, + _call_id, + input, + _INSERTED_TIMESTAMP, + value_precise_raw, + value_precise, + tx_position, + trace_index + FROM + {{ ref('silver_evm__traces') }} + WHERE + VALUE > 0 + AND tx_status = 'SUCCESS' + AND trace_status = 'SUCCESS' + AND TYPE NOT IN ( + 'DELEGATECALL', + 'STATICCALL' + ) + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '12 hours' + FROM + {{ this }} +) +{% endif %} +), +tx_table AS ( + SELECT + block_number, + tx_hash, + from_address AS origin_from_address, + to_address AS origin_to_address, + origin_function_signature + FROM + {{ ref('silver_evm__transactions') }} + WHERE + tx_hash IN ( + SELECT + DISTINCT tx_hash + FROM + base + ) + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '12 hours' + FROM + {{ this }} +) +{% endif %} +) +SELECT + tx_hash AS tx_hash, + block_number AS block_number, + block_timestamp AS block_timestamp, + identifier AS identifier, + origin_from_address, + origin_to_address, + origin_function_signature, + from_address, + to_address, + VALUE AS amount, + value_precise_raw AS amount_precise_raw, + value_precise AS amount_precise, + ROUND( + VALUE * price, + 2 + ) AS amount_usd, + _call_id, + _inserted_timestamp, + tx_position, + trace_index, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'trace_index'] + ) }} AS native_transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + base A + LEFT JOIN {{ ref('silver__complete_native_prices') }} + ON DATE_TRUNC( + 'hour', + A.block_timestamp + ) = HOUR + JOIN tx_table USING ( + tx_hash, + block_number + ) diff --git a/models/evm/silver/core/silver_evm__receipts.sql b/models/evm/silver/core/silver_evm__receipts.sql new file mode 100644 index 0000000..af391a6 --- /dev/null +++ b/models/evm/silver/core/silver_evm__receipts.sql @@ -0,0 +1,105 @@ +-- depends_on: {{ ref('bronze_evm__streamline_receipts') }} +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(block_hash, tx_hash, from_address, to_address)", + tags = ['core'] +) }} + +WITH base AS ( + + SELECT + VALUE :BLOCK_NUMBER :: INT AS block_number, + DATA, + inserted_timestamp + FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__streamline_receipts') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(inserted_timestamp) inserted_timestamp + FROM + {{ this }} + ) + AND IS_OBJECT(DATA) +{% else %} + {{ ref('bronze_evm__streamline_FR_receipts') }} +WHERE + IS_OBJECT(DATA) +{% endif %} +), +FINAL AS ( + SELECT + DATA, + block_number, + DATA :blockHash :: STRING AS block_hash, + utils.udf_hex_to_int( + DATA :blockNumber :: STRING + ) :: INT AS blockNumber, + DATA :contractAddress :: STRING AS contractAddress, + utils.udf_hex_to_int( + DATA :cumulativeGasUsed :: STRING + ) :: INT AS cumulative_gas_used, + utils.udf_hex_to_int( + DATA :effectiveGasPrice :: STRING + ) :: INT / pow( + 10, + 9 + ) AS effective_gas_price, + DATA :from :: STRING AS from_address, + COALESCE( + utils.udf_hex_to_int( + DATA :gasUsed :: STRING + ) :: INT, + 0 + ) AS gas_used, + DATA :logs AS logs, + DATA :logsBloom :: STRING AS logs_bloom, + utils.udf_hex_to_int( + DATA :status :: STRING + ) :: INT AS status, + CASE + WHEN status = 1 THEN TRUE + ELSE FALSE + END AS tx_success, + CASE + WHEN status = 1 THEN 'SUCCESS' + ELSE 'FAIL' + END AS tx_status, + DATA :to :: STRING AS to_address1, + CASE + WHEN to_address1 = '' THEN NULL + ELSE to_address1 + END AS to_address, + DATA :transactionHash :: STRING AS tx_hash, + CASE + WHEN block_number <> blockNumber THEN NULL + ELSE utils.udf_hex_to_int( + DATA :transactionIndex :: STRING + ) :: INT + END AS POSITION, + utils.udf_hex_to_int( + DATA :type :: STRING + ) :: INT AS TYPE, + inserted_timestamp + FROM + base +) +SELECT + * +FROM + FINAL +WHERE + tx_hash IS NOT NULL + AND POSITION IS NOT NULL qualify( + (ROW_NUMBER() over (PARTITION BY block_number, POSITION + ORDER BY + inserted_timestamp DESC)) = 1 + AND (ROW_NUMBER() over (PARTITION BY tx_hash + ORDER BY + block_number DESC)) = 1 + ) diff --git a/models/evm/silver/core/silver_evm__relevant_contracts.sql b/models/evm/silver/core/silver_evm__relevant_contracts.sql new file mode 100644 index 0000000..ef350c4 --- /dev/null +++ b/models/evm/silver/core/silver_evm__relevant_contracts.sql @@ -0,0 +1,145 @@ +{{ config( + materialized = 'incremental', + unique_key = "contract_address", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(contract_address)", + tags = ['recent_evm_test', 'core'], + merge_exclude_columns = ["inserted_timestamp"] +) }} + +WITH emitted_events AS ( + + SELECT + contract_address, + COUNT(*) AS event_count, + MAX(_inserted_timestamp) AS max_inserted_timestamp_logs, + MAX(block_number) AS latest_event_block + FROM + {{ ref('silver_evm__logs') }} + +{% if is_incremental() %} +WHERE + _inserted_timestamp > ( + SELECT + MAX(max_inserted_timestamp_logs) + FROM + {{ this }} + ) +{% endif %} +GROUP BY + contract_address +), +function_calls AS ( + SELECT + IFF( + TYPE = 'DELEGATECALL', + from_address, + to_address + ) AS contract_address, + COUNT(*) AS function_call_count, + MAX(_inserted_timestamp) AS max_inserted_timestamp_traces, + MAX(block_number) AS latest_call_block + FROM + {{ ref('silver_evm__traces') }} + WHERE + tx_status = 'SUCCESS' + AND trace_status = 'SUCCESS' + AND to_address IS NOT NULL + AND input IS NOT NULL + AND input <> '0x' + +{% if is_incremental() %} +AND _inserted_timestamp > ( + SELECT + MAX(max_inserted_timestamp_traces) + FROM + {{ this }} +) +{% endif %} +GROUP BY + 1 +), +active_contracts AS ( + SELECT + contract_address + FROM + emitted_events + UNION + SELECT + contract_address + FROM + function_calls +), +previous_totals AS ( + +{% if is_incremental() %} +SELECT + contract_address, total_event_count, total_call_count, max_inserted_timestamp_logs, latest_event_block, max_inserted_timestamp_traces, latest_call_block +FROM + {{ this }} +{% else %} +SELECT + NULL AS contract_address, 0 AS total_event_count, 0 AS total_call_count, '1970-01-01 00:00:00' AS max_inserted_timestamp_logs, 0 AS latest_event_block, '1970-01-01 00:00:00' AS max_inserted_timestamp_traces, 0 AS latest_call_block +{% endif %}) +SELECT + C.contract_address, + COALESCE( + p.total_event_count, + 0 + ) + COALESCE( + e.event_count, + 0 + ) AS total_event_count, + COALESCE( + p.total_call_count, + 0 + ) + COALESCE( + f.function_call_count, + 0 + ) AS total_call_count, + COALESCE( + p.total_event_count, + 0 + ) + COALESCE( + e.event_count, + 0 + ) + COALESCE( + p.total_call_count, + 0 + ) + COALESCE( + f.function_call_count, + 0 + ) AS total_interaction_count, + COALESCE( + e.max_inserted_timestamp_logs, + p.max_inserted_timestamp_logs, + '1970-01-01 00:00:00' + ) AS max_inserted_timestamp_logs, + COALESCE( + f.max_inserted_timestamp_traces, + p.max_inserted_timestamp_traces, + '1970-01-01 00:00:00' + ) AS max_inserted_timestamp_traces, + COALESCE( + e.latest_event_block, + p.latest_event_block, + 0 + ) AS latest_event_block, + COALESCE( + f.latest_call_block, + p.latest_call_block, + 0 + ) AS latest_call_block, + {{ dbt_utils.generate_surrogate_key( + ['c.contract_address'] + ) }} AS relevant_contracts_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + active_contracts C + LEFT JOIN emitted_events e + ON C.contract_address = e.contract_address + LEFT JOIN function_calls f + ON C.contract_address = f.contract_address + LEFT JOIN previous_totals p + ON C.contract_address = p.contract_address diff --git a/models/evm/silver/core/silver_evm__relevant_contracts.yml b/models/evm/silver/core/silver_evm__relevant_contracts.yml new file mode 100644 index 0000000..88fb318 --- /dev/null +++ b/models/evm/silver/core/silver_evm__relevant_contracts.yml @@ -0,0 +1,7 @@ +version: 2 +models: + - name: silver_evm__relevant_contracts + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - CONTRACT_ADDRESS \ No newline at end of file diff --git a/models/evm/silver/core/silver_evm__traces.sql b/models/evm/silver/core/silver_evm__traces.sql new file mode 100644 index 0000000..920190b --- /dev/null +++ b/models/evm/silver/core/silver_evm__traces.sql @@ -0,0 +1,429 @@ +-- depends_on: {{ ref('bronze_evm__streamline_traces') }} +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = "block_timestamp::date, _inserted_timestamp::date", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + full_refresh = false, + tags = ['core'] +) }} + +{% if execute %} + {% set query_traces %} + CREATE + OR REPLACE temporary TABLE silver_evm.traces__traces_intermediate_tmp AS + + SELECT + VALUE :BLOCK_NUMBER :: INT AS block_number, + VALUE :array_index :: INT AS tx_position, + DATA :result AS full_traces, + DATA :txHash :: STRING AS tx_hash, + _inserted_timestamp + FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__streamline_traces') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) + AND DATA :result IS NOT NULL +{% else %} + {{ ref('bronze_evm__streamline_FR_traces') }} +WHERE + DATA :result IS NOT NULL +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_position +ORDER BY + _inserted_timestamp DESC)) = 1 {% endset %} + {% do run_query( + query_traces + ) %} +{% endif %} + +WITH flatten_traces AS ( + SELECT + block_number, + tx_hash, + IFF( + path IN ( + 'result', + 'result.value', + 'result.type', + 'result.to', + 'result.input', + 'result.gasUsed', + 'result.gas', + 'result.from', + 'result.output', + 'result.error', + 'result.revertReason', + 'gasUsed', + 'gas', + 'type', + 'to', + 'from', + 'value', + 'input', + 'error', + 'output', + 'revertReason' + ), + 'ORIGIN', + REGEXP_REPLACE(REGEXP_REPLACE(path, '[^0-9]+', '_'), '^_|_$', '') + ) AS trace_address, + _inserted_timestamp, + OBJECT_AGG( + key, + VALUE + ) AS trace_json, + CASE + WHEN trace_address = 'ORIGIN' THEN NULL + WHEN POSITION( + '_' IN trace_address + ) = 0 THEN 'ORIGIN' + ELSE REGEXP_REPLACE( + trace_address, + '_[0-9]+$', + '', + 1, + 1 + ) + END AS parent_trace_address, + SPLIT( + trace_address, + '_' + ) AS str_array + FROM + silver_evm.traces__traces_intermediate_tmp txs, + TABLE( + FLATTEN( + input => PARSE_JSON( + txs.full_traces + ), + recursive => TRUE + ) + ) f + WHERE + f.index IS NULL + AND f.key != 'calls' + AND f.path != 'result' + GROUP BY + block_number, + tx_hash, + trace_address, + _inserted_timestamp +), +sub_traces AS ( + SELECT + block_number, + tx_hash, + parent_trace_address, + COUNT(*) AS sub_traces + FROM + flatten_traces + GROUP BY + block_number, + tx_hash, + parent_trace_address +), +num_array AS ( + SELECT + block_number, + tx_hash, + trace_address, + ARRAY_AGG(flat_value) AS num_array + FROM + ( + SELECT + block_number, + tx_hash, + trace_address, + IFF( + VALUE :: STRING = 'ORIGIN', + -1, + VALUE :: INT + ) AS flat_value + FROM + flatten_traces, + LATERAL FLATTEN ( + input => str_array + ) + ) + GROUP BY + block_number, + tx_hash, + trace_address +), +cleaned_traces AS ( + SELECT + b.block_number, + b.tx_hash, + b.trace_address, + IFNULL( + sub_traces, + 0 + ) AS sub_traces, + num_array, + ROW_NUMBER() over ( + PARTITION BY b.block_number, + b.tx_hash + ORDER BY + num_array ASC + ) - 1 AS trace_index, + trace_json, + b._inserted_timestamp + FROM + flatten_traces b + LEFT JOIN sub_traces s + ON b.block_number = s.block_number + AND b.tx_hash = s.tx_hash + AND b.trace_address = s.parent_trace_address + JOIN num_array n + ON b.block_number = n.block_number + AND b.tx_hash = n.tx_hash + AND b.trace_address = n.trace_address +), +final_traces AS ( + SELECT + tx_hash, + trace_index, + block_number, + trace_address, + trace_json :error :: STRING AS error_reason, + trace_json :from :: STRING AS from_address, + trace_json :to :: STRING AS to_address, + IFNULL( + utils.udf_hex_to_int( + trace_json :value :: STRING + ), + '0' + ) AS value_precise_raw, + utils.udf_decimal_adjust( + value_precise_raw, + 18 + ) AS value_precise, + value_precise :: FLOAT AS VALUE, + utils.udf_hex_to_int( + trace_json :gas :: STRING + ) :: INT AS gas, + utils.udf_hex_to_int( + trace_json :gasUsed :: STRING + ) :: INT AS gas_used, + trace_json :input :: STRING AS input, + trace_json :output :: STRING AS output, + trace_json :type :: STRING AS TYPE, + concat_ws( + '_', + TYPE, + trace_address + ) AS identifier, + _inserted_timestamp, + trace_json AS DATA, + sub_traces + FROM + cleaned_traces +), +new_records AS ( + SELECT + f.block_number, + f.tx_hash, + t.block_timestamp, + t.tx_status, + t.position AS tx_position, + f.trace_index, + f.from_address, + f.to_address, + f.value_precise_raw, + f.value_precise, + f.value, + f.gas, + f.gas_used, + f.input, + f.output, + f.type, + f.identifier, + f.sub_traces, + f.error_reason, + IFF( + f.error_reason IS NULL, + 'SUCCESS', + 'FAIL' + ) AS trace_status, + f.data, + IFF( + t.tx_hash IS NULL + OR t.block_timestamp IS NULL + OR t.tx_status IS NULL, + TRUE, + FALSE + ) AS is_pending, + concat_ws( + '-', + f.block_number, + t.position, + f.identifier + ) AS _call_id, + f._inserted_timestamp + FROM + final_traces f + LEFT OUTER JOIN {{ ref('silver_evm__transactions') }} + t + ON f.tx_hash = t.tx_hash + AND f.block_number = t.block_number + +{% if is_incremental() %} +AND t._INSERTED_TIMESTAMP >= ( + SELECT + DATEADD('hour', -24, MAX(_inserted_timestamp)) + FROM + {{ this }}) + {% endif %} +) + +{% if is_incremental() %}, +missing_data AS ( + SELECT + t.block_number, + t.tx_hash, + txs.block_timestamp, + txs.tx_status, + txs.position AS tx_position, + t.trace_index, + t.from_address, + t.to_address, + t.value_precise_raw, + t.value_precise, + t.value, + t.gas, + t.gas_used, + t.input, + t.output, + t.type, + t.identifier, + t.sub_traces, + t.error_reason, + t.trace_status, + t.data, + FALSE AS is_pending, + concat_ws( + '-', + t.block_number, + txs.position, + t.identifier + ) AS _call_id, + GREATEST( + t._inserted_timestamp, + txs._inserted_timestamp + ) AS _inserted_timestamp + FROM + {{ this }} + t + INNER JOIN {{ ref('silver_evm__transactions') }} + txs + ON t.tx_hash = txs.tx_hash + AND t.block_number = txs.block_number + WHERE + t.is_pending +) +{% endif %}, +FINAL AS ( + SELECT + block_number, + tx_hash, + block_timestamp, + tx_status, + tx_position, + trace_index, + from_address, + to_address, + value_precise_raw, + value_precise, + VALUE, + gas, + gas_used, + input, + output, + TYPE, + identifier, + sub_traces, + error_reason, + trace_status, + DATA, + is_pending, + _call_id, + _inserted_timestamp + FROM + new_records + +{% if is_incremental() %} +UNION +SELECT + block_number, + tx_hash, + block_timestamp, + tx_status, + tx_position, + trace_index, + from_address, + to_address, + value_precise_raw, + value_precise, + VALUE, + gas, + gas_used, + input, + output, + TYPE, + identifier, + sub_traces, + error_reason, + trace_status, + DATA, + is_pending, + _call_id, + _inserted_timestamp +FROM + missing_data +{% endif %} +) +SELECT + block_number, + tx_hash, + block_timestamp, + tx_status, + tx_position, + trace_index, + from_address, + to_address, + value_precise, + VALUE, + gas, + gas_used, + input, + output, + TYPE, + identifier, + sub_traces, + error_reason, + trace_status, + DATA, + is_pending, + _call_id, + _inserted_timestamp, + value_precise_raw, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'trace_index'] + ) }} AS traces_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_hash, trace_index +ORDER BY + _inserted_timestamp DESC, is_pending ASC)) = 1 diff --git a/models/evm/silver/core/silver_evm__transactions.sql b/models/evm/silver/core/silver_evm__transactions.sql new file mode 100644 index 0000000..121a26c --- /dev/null +++ b/models/evm/silver/core/silver_evm__transactions.sql @@ -0,0 +1,395 @@ +-- depends_on: {{ ref('bronze_evm__streamline_transactions') }} +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = "block_timestamp::date, _inserted_timestamp::date", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['core'] +) }} + +WITH base AS ( + + SELECT + VALUE :BLOCK_NUMBER :: INT AS block_number, + DATA, + _inserted_timestamp + FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__streamline_transactions') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) + AND IS_OBJECT(DATA) +{% else %} + {{ ref('bronze_evm__streamline_FR_transactions') }} +WHERE + IS_OBJECT(DATA) +{% endif %} +), +base_tx AS ( + SELECT + block_number, + DATA :blockHash :: STRING AS block_hash, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + DATA :blockNumber :: STRING + ) + ) AS blockNumber, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + DATA: depositReceiptVersion :: STRING + ) + ) AS deposit_receipt_version, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + DATA :chainId :: STRING + ) + ) AS chain_id, + DATA :from :: STRING AS from_address, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + DATA :gas :: STRING + ) + ) AS gas, + COALESCE( + TRY_TO_NUMBER( + utils.udf_hex_to_int( + DATA :gasPrice :: STRING + ) + ) / pow( + 10, + 9 + ), + 0 + ) AS gas_price, + DATA :hash :: STRING AS tx_hash, + DATA :input :: STRING AS input_data, + SUBSTR( + input_data, + 1, + 10 + ) AS origin_function_signature, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + DATA :maxFeePerGas :: STRING + ) + ) / pow( + 10, + 9 + ) AS max_fee_per_gas, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + DATA :maxPriorityFeePerGas :: STRING + ) + ) / pow( + 10, + 9 + ) AS max_priority_fee_per_gas, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + DATA :nonce :: STRING + ) + ) AS nonce, + DATA :r :: STRING AS r, + DATA :s :: STRING AS s, + DATA :sourceHash :: STRING AS source_hash, + DATA :to :: STRING AS to_address1, + CASE + WHEN to_address1 = '' THEN NULL + ELSE to_address1 + END AS to_address, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + DATA :transactionIndex :: STRING + ) + ) AS POSITION, + DATA :type :: STRING AS TYPE, + DATA :v :: STRING AS v, + utils.udf_hex_to_int( + DATA :value :: STRING + ) AS value_precise_raw, + utils.udf_decimal_adjust( + value_precise_raw, + 18 + ) AS value_precise, + value_precise :: FLOAT AS VALUE, + DATA :yParity :: STRING AS y_parity, + _INSERTED_TIMESTAMP, + DATA + FROM + base +), +new_records AS ( + SELECT + t.block_number, + t.block_hash, + t.chain_id, + t.from_address, + t.gas, + t.gas_price, + t.tx_hash, + t.input_data, + t.origin_function_signature, + t.max_fee_per_gas, + t.max_priority_fee_per_gas, + t.nonce, + t.r, + t.s, + t.source_hash, + t.to_address, + t.position, + t.type, + t.v, + t.value_precise_raw, + t.value_precise, + t.value, + t.y_parity, + block_timestamp, + CASE + WHEN block_timestamp IS NULL + OR tx_status IS NULL THEN TRUE + ELSE FALSE + END AS is_pending, + r.gas_used, + utils.udf_decimal_adjust( + r.gas_used * utils.udf_hex_to_int( + t.data :gasPrice :: STRING + ) :: bigint, + 18 + ) AS tx_fee_precise, + COALESCE( + tx_fee_precise :: FLOAT, + 0 + ) AS tx_fee, + tx_success, + tx_status, + cumulative_gas_used, + effective_gas_price, + r.type AS tx_type, + t._inserted_timestamp, + t.data + FROM + base_tx t + LEFT OUTER JOIN {{ ref('silver_evm__blocks') }} + b + ON t.block_number = b.block_number + LEFT OUTER JOIN {{ ref('silver_evm__receipts') }} + r + ON t.block_number = r.block_number + AND t.tx_hash = r.tx_hash + +{% if is_incremental() %} +AND r._INSERTED_TIMESTAMP >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 1 + FROM + {{ this }} +) +{% endif %} +) + +{% if is_incremental() %}, +missing_data AS ( + SELECT + t.block_number, + t.block_hash, + t.chain_id, + t.from_address, + t.gas, + t.gas_price, + t.tx_hash, + t.input_data, + t.origin_function_signature, + t.max_fee_per_gas, + t.max_priority_fee_per_gas, + t.nonce, + t.r, + t.s, + t.source_hash, + t.to_address, + t.position, + t.type, + t.v, + t.value_precise_raw, + t.value_precise, + t.value, + t.y_parity, + b.block_timestamp, + FALSE AS is_pending, + r.gas_used, + r.tx_success, + r.tx_status, + r.cumulative_gas_used, + r.effective_gas_price, + utils.udf_decimal_adjust( + r.gas_used * utils.udf_hex_to_int( + t.data :gasPrice :: STRING + ) :: bigint, + 18 + ) AS tx_fee_precise_heal, + COALESCE( + tx_fee_precise_heal :: FLOAT, + 0 + ) AS tx_fee, + r.type AS tx_type, + GREATEST( + t._inserted_timestamp, + b._inserted_timestamp, + r._inserted_timestamp + ) AS _inserted_timestamp, + t.data + FROM + {{ this }} + t + INNER JOIN {{ ref('silver_evm__blocks') }} + b + ON t.block_number = b.block_number + INNER JOIN {{ ref('silver_evm__receipts') }} + r + ON t.tx_hash = r.tx_hash + AND t.block_number = r.block_number + WHERE + t.is_pending +) +{% endif %}, +FINAL AS ( + SELECT + block_number, + block_hash, + chain_id, + from_address, + gas, + gas_price, + tx_hash, + input_data, + origin_function_signature, + max_fee_per_gas, + max_priority_fee_per_gas, + nonce, + r, + s, + source_hash, + to_address, + POSITION, + TYPE, + v, + y_parity, + VALUE, + value_precise_raw, + value_precise, + block_timestamp, + is_pending, + gas_used, + tx_success, + tx_status, + cumulative_gas_used, + effective_gas_price, + tx_fee, + tx_fee_precise, + tx_type, + _inserted_timestamp, + DATA + FROM + new_records + +{% if is_incremental() %} +UNION +SELECT + block_number, + block_hash, + chain_id, + from_address, + gas, + gas_price, + tx_hash, + input_data, + origin_function_signature, + max_fee_per_gas, + max_priority_fee_per_gas, + nonce, + r, + s, + source_hash, + to_address, + POSITION, + TYPE, + v, + y_parity, + VALUE, + value_precise_raw, + value_precise, + block_timestamp, + is_pending, + gas_used, + tx_success, + tx_status, + cumulative_gas_used, + effective_gas_price, + tx_fee, + tx_fee_precise_heal AS tx_fee_precise, + tx_type, + _inserted_timestamp, + DATA +FROM + missing_data +{% endif %} +) +SELECT + block_number, + block_hash, + chain_id, + from_address, + gas, + gas_price, + tx_hash, + input_data, + origin_function_signature, + max_fee_per_gas, + max_priority_fee_per_gas, + nonce, + r, + s, + source_hash, + to_address, + POSITION, + TYPE, + v, + y_parity, + VALUE, + value_precise_raw, + value_precise, + block_timestamp, + is_pending, + gas_used, + tx_success, + tx_status, + cumulative_gas_used, + effective_gas_price, + tx_fee, + tx_fee_precise, + tx_type, + _inserted_timestamp, + DATA, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS transactions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL +WHERE + block_hash IS NOT NULL qualify( + (ROW_NUMBER() over (PARTITION BY block_number, POSITION + ORDER BY + _inserted_timestamp DESC, is_pending ASC)) = 1 + AND (ROW_NUMBER() over (PARTITION BY tx_hash + ORDER BY + block_number DESC)) = 1 + ) diff --git a/models/evm/silver/core/silver_evm__transfers.sql b/models/evm/silver/core/silver_evm__transfers.sql new file mode 100644 index 0000000..f5fb656 --- /dev/null +++ b/models/evm/silver/core/silver_evm__transfers.sql @@ -0,0 +1,110 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = ['block_timestamp::DATE', '_inserted_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash, origin_function_signature, origin_from_address, origin_to_address, contract_address, from_address, to_address, symbol), SUBSTRING(origin_function_signature, contract_address, from_address, to_address, symbol)", + tags = ['core'] +) }} + +WITH logs AS ( + + SELECT + _log_id, + block_number, + tx_hash, + block_timestamp, + origin_function_signature, + origin_from_address, + origin_to_address, + contract_address :: STRING AS contract_address, + CONCAT('0x', SUBSTR(topics [1], 27, 40)) :: STRING AS from_address, + CONCAT('0x', SUBSTR(topics [2], 27, 40)) :: STRING AS to_address, + utils.udf_hex_to_int(SUBSTR(DATA, 3, 64)) AS raw_amount_precise, + raw_amount_precise :: FLOAT AS raw_amount, + event_index, + _inserted_timestamp + FROM + {{ ref('silver_evm__logs') }} + WHERE + topics [0] :: STRING = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' + AND tx_status = 'SUCCESS' + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '36 hours' + FROM + {{ this }} +) +{% endif %} +), +token_transfers AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + t.contract_address, + from_address, + to_address, + raw_amount_precise, + raw_amount, + IFF( + C.token_decimals IS NOT NULL, + utils.udf_decimal_adjust( + raw_amount_precise, + C.token_decimals + ), + NULL + ) AS amount_precise, + amount_precise :: FLOAT AS amount, + C.token_decimals AS decimals, + C.token_symbol AS symbol, + CASE + WHEN C.token_decimals IS NULL THEN 'false' + ELSE 'true' + END AS has_decimal, + _log_id, + _inserted_timestamp + FROM + logs t + LEFT JOIN {{ ref('silver_evm__contracts') }} C USING (contract_address) + WHERE + raw_amount IS NOT NULL + AND to_address IS NOT NULL + AND from_address IS NOT NULL +) +SELECT + block_number, + block_timestamp, + tx_hash, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + contract_address, + from_address, + to_address, + raw_amount_precise, + raw_amount, + amount_precise, + amount, + decimals, + symbol, + has_decimal, + _log_id, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} AS transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + token_transfers diff --git a/models/evm/silver/core/tests/blocks/test_silver_evm__blocks_full.sql b/models/evm/silver/core/tests/blocks/test_silver_evm__blocks_full.sql new file mode 100644 index 0000000..4f6c200 --- /dev/null +++ b/models/evm/silver/core/tests/blocks/test_silver_evm__blocks_full.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view', + tags = ['full_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('silver_evm__blocks') }} +WHERE + block_timestamp < DATEADD('hour', -1, SYSDATE()) diff --git a/models/evm/silver/core/tests/blocks/test_silver_evm__blocks_full.yml b/models/evm/silver/core/tests/blocks/test_silver_evm__blocks_full.yml new file mode 100644 index 0000000..9a3e541 --- /dev/null +++ b/models/evm/silver/core/tests/blocks/test_silver_evm__blocks_full.yml @@ -0,0 +1,95 @@ +version: 2 +models: + - name: test_silver_evm__blocks_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - fsc_utils.sequence_gaps: + column_name: BLOCK_NUMBER + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: DIFFICULTY + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TOTAL_DIFFICULTY + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EXTRA_DATA + tests: + - not_null + - name: GAS_LIMIT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: PARENT_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: MINER + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: NONCE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: RECEIPTS_ROOT + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: SHA3_UNCLES + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: SIZE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + + diff --git a/models/evm/silver/core/tests/blocks/test_silver_evm__blocks_recent.sql b/models/evm/silver/core/tests/blocks/test_silver_evm__blocks_recent.sql new file mode 100644 index 0000000..f0e0341 --- /dev/null +++ b/models/evm/silver/core/tests/blocks/test_silver_evm__blocks_recent.sql @@ -0,0 +1,24 @@ +{{ config ( + materialized = 'view', + tags = ['recent_evm_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_evm_block_lookback") }} +) +SELECT + * +FROM + {{ ref('silver_evm__blocks') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + AND block_timestamp < DATEADD('hour', -1, SYSDATE()) diff --git a/models/evm/silver/core/tests/blocks/test_silver_evm__blocks_recent.yml b/models/evm/silver/core/tests/blocks/test_silver_evm__blocks_recent.yml new file mode 100644 index 0000000..4209801 --- /dev/null +++ b/models/evm/silver/core/tests/blocks/test_silver_evm__blocks_recent.yml @@ -0,0 +1,27 @@ +version: 2 +models: + - name: test_silver_evm__blocks_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - fsc_utils.sequence_gaps: + column_name: BLOCK_NUMBER + config: + severity: error + error_if: ">10" + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + \ No newline at end of file diff --git a/models/evm/silver/core/tests/confirmed_blocks/test_silver_evm__confirmed_blocks_full.sql b/models/evm/silver/core/tests/confirmed_blocks/test_silver_evm__confirmed_blocks_full.sql new file mode 100644 index 0000000..838aed6 --- /dev/null +++ b/models/evm/silver/core/tests/confirmed_blocks/test_silver_evm__confirmed_blocks_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('silver_evm__confirmed_blocks') }} diff --git a/models/evm/silver/core/tests/confirmed_blocks/test_silver_evm__confirmed_blocks_full.yml b/models/evm/silver/core/tests/confirmed_blocks/test_silver_evm__confirmed_blocks_full.yml new file mode 100644 index 0000000..201db92 --- /dev/null +++ b/models/evm/silver/core/tests/confirmed_blocks/test_silver_evm__confirmed_blocks_full.yml @@ -0,0 +1,34 @@ +version: 2 +models: + - name: test_silver_evm__confirmed_blocks_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - TIMESTAMP_LTZ \ No newline at end of file diff --git a/models/evm/silver/core/tests/confirmed_blocks/test_silver_evm__confirmed_blocks_recent.sql b/models/evm/silver/core/tests/confirmed_blocks/test_silver_evm__confirmed_blocks_recent.sql new file mode 100644 index 0000000..90794e9 --- /dev/null +++ b/models/evm/silver/core/tests/confirmed_blocks/test_silver_evm__confirmed_blocks_recent.sql @@ -0,0 +1,23 @@ +{{ config ( + materialized = 'view', + tags = ['recent_evm_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_evm_block_lookback") }} +) +SELECT + * +FROM + {{ ref('silver_evm__confirmed_blocks') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/evm/silver/core/tests/confirmed_blocks/test_silver_evm__confirmed_blocks_recent.yml b/models/evm/silver/core/tests/confirmed_blocks/test_silver_evm__confirmed_blocks_recent.yml new file mode 100644 index 0000000..a412954 --- /dev/null +++ b/models/evm/silver/core/tests/confirmed_blocks/test_silver_evm__confirmed_blocks_recent.yml @@ -0,0 +1,34 @@ +version: 2 +models: + - name: test_silver_evm__confirmed_blocks_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - TIMESTAMP_LTZ \ No newline at end of file diff --git a/models/evm/silver/core/tests/event_logs/test_silver_evm__logs_full.sql b/models/evm/silver/core/tests/event_logs/test_silver_evm__logs_full.sql new file mode 100644 index 0000000..e368387 --- /dev/null +++ b/models/evm/silver/core/tests/event_logs/test_silver_evm__logs_full.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view', + tags = ['full_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('silver_evm__logs') }} +WHERE + block_timestamp < DATEADD('hour', -1, SYSDATE()) diff --git a/models/evm/silver/core/tests/event_logs/test_silver_evm__logs_full.yml b/models/evm/silver/core/tests/event_logs/test_silver_evm__logs_full.yml new file mode 100644 index 0000000..fc01a01 --- /dev/null +++ b/models/evm/silver/core/tests/event_logs/test_silver_evm__logs_full.yml @@ -0,0 +1,77 @@ +version: 2 +models: + - name: test_silver_evm__logs_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _LOG_ID + - fsc_utils.sequence_gaps: + partition_by: + - BLOCK_NUMBER + - TX_HASH + column_name: EVENT_INDEX + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - fsc_utils.tx_block_count: + config: + severity: error + error_if: "!=0" + - name: EVENT_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CONTRACT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TOPICS + tests: + - not_null + - name: DATA + tests: + - not_null + - name: EVENT_REMOVED + tests: + - not_null + - name: _LOG_ID + tests: + - not_null + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null: + where: NOT IS_PENDING + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: ORIGIN_TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ \ No newline at end of file diff --git a/models/evm/silver/core/tests/event_logs/test_silver_evm__logs_recent.sql b/models/evm/silver/core/tests/event_logs/test_silver_evm__logs_recent.sql new file mode 100644 index 0000000..bbf3ba6 --- /dev/null +++ b/models/evm/silver/core/tests/event_logs/test_silver_evm__logs_recent.sql @@ -0,0 +1,24 @@ +{{ config ( + materialized = 'view', + tags = ['recent_evm_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_evm_block_lookback") }} +) +SELECT + * +FROM + {{ ref('silver_evm__logs') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + AND block_timestamp < DATEADD('hour', -1, SYSDATE()) diff --git a/models/evm/silver/core/tests/event_logs/test_silver_evm__logs_recent.yml b/models/evm/silver/core/tests/event_logs/test_silver_evm__logs_recent.yml new file mode 100644 index 0000000..394b56c --- /dev/null +++ b/models/evm/silver/core/tests/event_logs/test_silver_evm__logs_recent.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: test_silver_evm__logs_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _LOG_ID + - fsc_utils.sequence_gaps: + partition_by: + - BLOCK_NUMBER + - TX_HASH + column_name: EVENT_INDEX + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null diff --git a/models/evm/silver/core/tests/receipts/test_silver_evm__receipts_full.sql b/models/evm/silver/core/tests/receipts/test_silver_evm__receipts_full.sql new file mode 100644 index 0000000..62e1e3f --- /dev/null +++ b/models/evm/silver/core/tests/receipts/test_silver_evm__receipts_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('silver_evm__receipts') }} diff --git a/models/evm/silver/core/tests/receipts/test_silver_evm__receipts_full.yml b/models/evm/silver/core/tests/receipts/test_silver_evm__receipts_full.yml new file mode 100644 index 0000000..3537eed --- /dev/null +++ b/models/evm/silver/core/tests/receipts/test_silver_evm__receipts_full.yml @@ -0,0 +1,82 @@ +version: 2 +models: + - name: test_silver_evm__receipts_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - fsc_utils.sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: CUMULATIVE_GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EFFECTIVE_GAS_PRICE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_STATUS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ['SUCCESS', 'FAIL'] + - name: TYPE + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + + diff --git a/models/evm/silver/core/tests/receipts/test_silver_evm__receipts_recent.sql b/models/evm/silver/core/tests/receipts/test_silver_evm__receipts_recent.sql new file mode 100644 index 0000000..1c27714 --- /dev/null +++ b/models/evm/silver/core/tests/receipts/test_silver_evm__receipts_recent.sql @@ -0,0 +1,23 @@ +{{ config ( + materialized = 'view', + tags = ['recent_evm_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_evm_block_lookback") }} +) +SELECT + * +FROM + {{ ref('silver_evm__receipts') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/evm/silver/core/tests/receipts/test_silver_evm__receipts_recent.yml b/models/evm/silver/core/tests/receipts/test_silver_evm__receipts_recent.yml new file mode 100644 index 0000000..57c23fe --- /dev/null +++ b/models/evm/silver/core/tests/receipts/test_silver_evm__receipts_recent.yml @@ -0,0 +1,28 @@ +version: 2 +models: + - name: test_silver_evm__receipts_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - fsc_utils.sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + + diff --git a/models/evm/silver/core/tests/traces/test_silver_evm__traces_full.sql b/models/evm/silver/core/tests/traces/test_silver_evm__traces_full.sql new file mode 100644 index 0000000..212bf1d --- /dev/null +++ b/models/evm/silver/core/tests/traces/test_silver_evm__traces_full.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view', + tags = ['full_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('silver_evm__traces') }} +WHERE + block_timestamp < DATEADD('hour', -1, SYSDATE()) diff --git a/models/evm/silver/core/tests/traces/test_silver_evm__traces_full.yml b/models/evm/silver/core/tests/traces/test_silver_evm__traces_full.yml new file mode 100644 index 0000000..4a080bb --- /dev/null +++ b/models/evm/silver/core/tests/traces/test_silver_evm__traces_full.yml @@ -0,0 +1,59 @@ +version: 2 +models: + - name: test_silver_evm__traces_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - TX_POSITION + - TRACE_INDEX + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: FROM_ADDRESS + tests: + - not_null: + where: TYPE <> 'SELFDESTRUCT' + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: IDENTIFIER + tests: + - not_null + - name: VALUE + tests: + - not_null + - name: GAS + tests: + - not_null + - name: GAS_USED + tests: + - not_null + + diff --git a/models/evm/silver/core/tests/traces/test_silver_evm__traces_recent.sql b/models/evm/silver/core/tests/traces/test_silver_evm__traces_recent.sql new file mode 100644 index 0000000..3bec4ed --- /dev/null +++ b/models/evm/silver/core/tests/traces/test_silver_evm__traces_recent.sql @@ -0,0 +1,24 @@ +{{ config ( + materialized = 'view', + tags = ['recent_evm_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_evm_block_lookback") }} +) +SELECT + * +FROM + {{ ref('silver_evm__traces') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + AND block_timestamp < DATEADD('hour', -1, SYSDATE()) diff --git a/models/evm/silver/core/tests/traces/test_silver_evm__traces_recent.yml b/models/evm/silver/core/tests/traces/test_silver_evm__traces_recent.yml new file mode 100644 index 0000000..058449a --- /dev/null +++ b/models/evm/silver/core/tests/traces/test_silver_evm__traces_recent.yml @@ -0,0 +1,35 @@ +version: 2 +models: + - name: test_silver_evm__traces_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - TX_POSITION + - TRACE_INDEX + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + \ No newline at end of file diff --git a/models/evm/silver/core/tests/transactions/test_silver_evm__transactions_full.sql b/models/evm/silver/core/tests/transactions/test_silver_evm__transactions_full.sql new file mode 100644 index 0000000..19b15bd --- /dev/null +++ b/models/evm/silver/core/tests/transactions/test_silver_evm__transactions_full.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view', + tags = ['full_evm_test'] +) }} + +SELECT + * +FROM + {{ ref('silver_evm__transactions') }} +WHERE + block_timestamp < DATEADD('hour', -1, SYSDATE()) diff --git a/models/evm/silver/core/tests/transactions/test_silver_evm__transactions_full.yml b/models/evm/silver/core/tests/transactions/test_silver_evm__transactions_full.yml new file mode 100644 index 0000000..37f42e5 --- /dev/null +++ b/models/evm/silver/core/tests/transactions/test_silver_evm__transactions_full.yml @@ -0,0 +1,114 @@ +version: 2 +models: + - name: test_silver_evm__transactions_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: NONCE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: VALUE + tests: + - not_null + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: GAS_PRICE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS + tests: + - not_null + - name: INPUT_DATA + tests: + - not_null + - name: TX_STATUS + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ['SUCCESS', 'FAIL'] + where: NOT IS_PENDING + - name: GAS_USED + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CUMULATIVE_GAS_USED + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_FEE + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EFFECTIVE_GAS_PRICE + tests: + - not_null: + where: NOT IS_PENDING + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + + diff --git a/models/evm/silver/core/tests/transactions/test_silver_evm__transactions_recent.sql b/models/evm/silver/core/tests/transactions/test_silver_evm__transactions_recent.sql new file mode 100644 index 0000000..f7f5a22 --- /dev/null +++ b/models/evm/silver/core/tests/transactions/test_silver_evm__transactions_recent.sql @@ -0,0 +1,24 @@ +{{ config ( + materialized = 'view', + tags = ['recent_evm_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_evm_block_lookback") }} +) +SELECT + * +FROM + {{ ref('silver_evm__transactions') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + AND block_timestamp < DATEADD('hour', -1, SYSDATE()) diff --git a/models/evm/silver/core/tests/transactions/test_silver_evm__transactions_recent.yml b/models/evm/silver/core/tests/transactions/test_silver_evm__transactions_recent.yml new file mode 100644 index 0000000..8614a73 --- /dev/null +++ b/models/evm/silver/core/tests/transactions/test_silver_evm__transactions_recent.yml @@ -0,0 +1,18 @@ +version: 2 +models: + - name: test_silver_evm__transactions_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + \ No newline at end of file diff --git a/models/evm/streamline/_evm_block_lookback.sql b/models/evm/streamline/_evm_block_lookback.sql new file mode 100644 index 0000000..1ee4149 --- /dev/null +++ b/models/evm/streamline/_evm_block_lookback.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = "ephemeral" +) }} + +SELECT + MIN(block_number) AS block_number +FROM + {{ ref("silver_evm__blocks") }} +WHERE + block_timestamp >= DATEADD('hour', -72, TRUNCATE(SYSDATE(), 'HOUR')) + AND block_timestamp < DATEADD('hour', -71, TRUNCATE(SYSDATE(), 'HOUR')) diff --git a/models/evm/streamline/_max_evm_block_by_date.sql b/models/evm/streamline/_max_evm_block_by_date.sql new file mode 100644 index 0000000..e7926d9 --- /dev/null +++ b/models/evm/streamline/_max_evm_block_by_date.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = "ephemeral", + unique_key = "block_number", +) }} + +WITH base AS ( + + SELECT + block_timestamp :: DATE AS block_date, + MAX(block_number) as block_number + FROM + {{ ref("silver_evm__blocks") }} + GROUP BY + block_timestamp :: DATE +) +SELECT + block_date, + block_number +FROM + base +WHERE + block_date <> ( + SELECT + MAX(block_date) + FROM + base + ) \ No newline at end of file diff --git a/models/evm/streamline/_max_evm_block_by_hour.sql b/models/evm/streamline/_max_evm_block_by_hour.sql new file mode 100644 index 0000000..1b82a04 --- /dev/null +++ b/models/evm/streamline/_max_evm_block_by_hour.sql @@ -0,0 +1,37 @@ +{{ config ( + materialized = "ephemeral" +) }} + +WITH base AS ( + + SELECT + DATE_TRUNC( + 'hour', + block_timestamp + ) AS block_hour, + MAX(block_number) block_number + FROM + {{ ref("silver_evm__blocks") }} + WHERE + block_timestamp > DATEADD( + 'day', + -5, + CURRENT_DATE + ) + GROUP BY + 1 +) +SELECT + block_hour, + block_number +FROM + base +WHERE + block_hour <> ( + SELECT + MAX( + block_hour + ) + FROM + base + ) diff --git a/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_blocks.sql b/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_blocks.sql new file mode 100644 index 0000000..1f83892 --- /dev/null +++ b/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_blocks.sql @@ -0,0 +1,7 @@ +{{ config ( + materialized = 'view' +) }} +{{ streamline_external_table_FR_query_v2( + model = "evm_blocks", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_confirm_blocks.sql b/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_confirm_blocks.sql new file mode 100644 index 0000000..fe75640 --- /dev/null +++ b/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_confirm_blocks.sql @@ -0,0 +1,7 @@ +{{ config ( + materialized = 'view' +) }} +{{ streamline_external_table_FR_query_v2( + model = "evm_confirm_blocks", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_receipts.sql b/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_receipts.sql new file mode 100644 index 0000000..f5c0682 --- /dev/null +++ b/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_receipts.sql @@ -0,0 +1,7 @@ +{{ config ( + materialized = 'view' +) }} +{{ streamline_external_table_FR_query_v2( + model = "evm_receipts", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_traces.sql b/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_traces.sql new file mode 100644 index 0000000..c8aa12d --- /dev/null +++ b/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_traces.sql @@ -0,0 +1,7 @@ +{{ config ( + materialized = 'view' +) }} +{{ streamline_external_table_FR_query_v2( + model = "evm_traces", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_transactions.sql b/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_transactions.sql new file mode 100644 index 0000000..ce54050 --- /dev/null +++ b/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_transactions.sql @@ -0,0 +1,7 @@ +{{ config ( + materialized = 'view' +) }} +{{ streamline_external_table_FR_query_v2( + model = 'evm_transactions', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/streamline/bronze/core/bronze_evm__streamline_blocks.sql b/models/evm/streamline/bronze/core/bronze_evm__streamline_blocks.sql new file mode 100644 index 0000000..fa5bd5b --- /dev/null +++ b/models/evm/streamline/bronze/core/bronze_evm__streamline_blocks.sql @@ -0,0 +1,7 @@ +{{ config ( + materialized = 'view' +) }} +{{ streamline_external_table_query_v2( + model = "evm_blocks", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/streamline/bronze/core/bronze_evm__streamline_confirm_blocks.sql b/models/evm/streamline/bronze/core/bronze_evm__streamline_confirm_blocks.sql new file mode 100644 index 0000000..3d04f3d --- /dev/null +++ b/models/evm/streamline/bronze/core/bronze_evm__streamline_confirm_blocks.sql @@ -0,0 +1,7 @@ +{{ config ( + materialized = 'view' +) }} +{{ streamline_external_table_query_v2( + model = "evm_confirm_blocks", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/streamline/bronze/core/bronze_evm__streamline_receipts.sql b/models/evm/streamline/bronze/core/bronze_evm__streamline_receipts.sql new file mode 100644 index 0000000..71c690c --- /dev/null +++ b/models/evm/streamline/bronze/core/bronze_evm__streamline_receipts.sql @@ -0,0 +1,7 @@ +{{ config ( + materialized = 'view' +) }} +{{ streamline_external_table_query_v2( + model = "evm_receipts", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/streamline/bronze/core/bronze_evm__streamline_traces.sql b/models/evm/streamline/bronze/core/bronze_evm__streamline_traces.sql new file mode 100644 index 0000000..e68e330 --- /dev/null +++ b/models/evm/streamline/bronze/core/bronze_evm__streamline_traces.sql @@ -0,0 +1,7 @@ +{{ config ( + materialized = 'view' +) }} +{{ streamline_external_table_query_v2( + model = "evm_traces", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/streamline/bronze/core/bronze_evm__streamline_transactions.sql b/models/evm/streamline/bronze/core/bronze_evm__streamline_transactions.sql new file mode 100644 index 0000000..10d3202 --- /dev/null +++ b/models/evm/streamline/bronze/core/bronze_evm__streamline_transactions.sql @@ -0,0 +1,7 @@ +{{ config ( + materialized = 'view' +) }} +{{ streamline_external_table_query_v2( + model = "evm_transactions", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" +) }} diff --git a/models/evm/streamline/bronze/decoder/bronze_evm__decoded_logs.sql b/models/evm/streamline/bronze/decoder/bronze_evm__decoded_logs.sql new file mode 100644 index 0000000..94f8fc1 --- /dev/null +++ b/models/evm/streamline/bronze/decoder/bronze_evm__decoded_logs.sql @@ -0,0 +1,41 @@ +{{ config ( + materialized = 'view' +) }} + +WITH meta AS ( + + SELECT + last_modified AS _inserted_timestamp, + file_name, + CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 6), '_', 1) AS INTEGER) AS _partition_by_block_number, + TO_DATE( + concat_ws('-', SPLIT_PART(file_name, '/', 3), SPLIT_PART(file_name, '/', 4), SPLIT_PART(file_name, '/', 5)) + ) AS _partition_by_created_date + FROM + TABLE( + information_schema.external_table_file_registration_history( + start_time => DATEADD('day', -3, CURRENT_TIMESTAMP()), + table_name => '{{ source( "bronze_streamline", "evm_decoded_logs") }}') + ) A + ) + SELECT + block_number, + id :: STRING AS id, + DATA, + _inserted_timestamp, + s._partition_by_block_number AS _partition_by_block_number, + s._partition_by_created_date AS _partition_by_created_date + FROM + {{ source( + "bronze_streamline", + "evm_decoded_logs" + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b._partition_by_block_number = s._partition_by_block_number + AND b._partition_by_created_date = s._partition_by_created_date + WHERE + b._partition_by_block_number = s._partition_by_block_number + AND b._partition_by_created_date = s._partition_by_created_date + AND s._partition_by_created_date >= DATEADD('day', -2, CURRENT_TIMESTAMP()) diff --git a/models/evm/streamline/bronze/decoder/bronze_evm__fr_decoded_logs.sql b/models/evm/streamline/bronze/decoder/bronze_evm__fr_decoded_logs.sql new file mode 100644 index 0000000..9f7145d --- /dev/null +++ b/models/evm/streamline/bronze/decoder/bronze_evm__fr_decoded_logs.sql @@ -0,0 +1,40 @@ +{{ config ( + materialized = 'view' +) }} + +WITH meta AS ( + + SELECT + registered_on AS _inserted_timestamp, + file_name, + CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 6), '_', 1) AS INTEGER) AS _partition_by_block_number, + TO_DATE( + concat_ws('-', SPLIT_PART(file_name, '/', 3), SPLIT_PART(file_name, '/', 4), SPLIT_PART(file_name, '/', 5)) + ) AS _partition_by_created_date + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", "evm_decoded_logs") }}' + ) + ) A +) +SELECT + block_number, + id :: STRING AS id, + DATA, + _inserted_timestamp, + s._partition_by_block_number AS _partition_by_block_number, + s._partition_by_created_date AS _partition_by_created_date +FROM + {{ source( + "bronze_streamline", + "evm_decoded_logs" + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b._partition_by_block_number = s._partition_by_block_number + AND b._partition_by_created_date = s._partition_by_created_date +WHERE + b._partition_by_block_number = s._partition_by_block_number + AND b._partition_by_created_date = s._partition_by_created_date diff --git a/models/evm/streamline/silver/core/complete/streamline__complete_evm_blocks.sql b/models/evm/streamline/silver/core/complete/streamline__complete_evm_blocks.sql new file mode 100644 index 0000000..399aab3 --- /dev/null +++ b/models/evm/streamline/silver/core/complete/streamline__complete_evm_blocks.sql @@ -0,0 +1,34 @@ +-- depends_on: {{ ref('bronze_evm__streamline_blocks') }} +{{ config ( + materialized = "incremental", + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", + tags = ['streamline_core_evm_complete'] +) }} + +SELECT + VALUE :BLOCK_NUMBER :: INT AS block_number, + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS complete_blocks_evm_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__streamline_blocks') }} +WHERE + inserted_timestamp >= ( + SELECT + COALESCE(MAX(inserted_timestamp), '1970-01-01' :: TIMESTAMP) inserted_timestamp + FROM + {{ this }}) + {% else %} + {{ ref('bronze_evm__streamline_FR_blocks') }} + {% endif %} + + qualify(ROW_NUMBER() over (PARTITION BY block_number + ORDER BY + inserted_timestamp DESC)) = 1 diff --git a/models/evm/streamline/silver/core/complete/streamline__complete_evm_confirmed_blocks.sql b/models/evm/streamline/silver/core/complete/streamline__complete_evm_confirmed_blocks.sql new file mode 100644 index 0000000..4944900 --- /dev/null +++ b/models/evm/streamline/silver/core/complete/streamline__complete_evm_confirmed_blocks.sql @@ -0,0 +1,34 @@ +-- depends_on: {{ ref('bronze_evm__streamline_confirm_blocks') }} +{{ config ( + materialized = "incremental", + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", + tags = ['streamline_core_evm_complete'] +) }} + +SELECT + VALUE :BLOCK_NUMBER :: INT AS block_number, + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS complete_blocks_evm_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__streamline_confirm_blocks') }} +WHERE + inserted_timestamp >= ( + SELECT + COALESCE(MAX(inserted_timestamp), '1970-01-01' :: TIMESTAMP) inserted_timestamp + FROM + {{ this }}) + {% else %} + {{ ref('bronze_evm__streamline_FR_confirm_blocks') }} + {% endif %} + + qualify(ROW_NUMBER() over (PARTITION BY block_number + ORDER BY + inserted_timestamp DESC)) = 1 diff --git a/models/evm/streamline/silver/core/complete/streamline__complete_evm_receipts.sql b/models/evm/streamline/silver/core/complete/streamline__complete_evm_receipts.sql new file mode 100644 index 0000000..db1d3b5 --- /dev/null +++ b/models/evm/streamline/silver/core/complete/streamline__complete_evm_receipts.sql @@ -0,0 +1,34 @@ +-- depends_on: {{ ref('bronze_evm__streamline_receipts') }} +{{ config ( + materialized = "incremental", + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", + tags = ['streamline_core_evm_complete'] +) }} + +SELECT + VALUE :BLOCK_NUMBER :: INT AS block_number, + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS complete_evm_receipts_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__streamline_receipts') }} +WHERE + inserted_timestamp >= ( + SELECT + COALESCE(MAX(inserted_timestamp), '1970-01-01' :: TIMESTAMP) inserted_timestamp + FROM + {{ this }}) + {% else %} + {{ ref('bronze_evm__streamline_FR_receipts') }} + {% endif %} + + qualify(ROW_NUMBER() over (PARTITION BY block_number + ORDER BY + inserted_timestamp DESC)) = 1 diff --git a/models/evm/streamline/silver/core/complete/streamline__complete_evm_traces.sql b/models/evm/streamline/silver/core/complete/streamline__complete_evm_traces.sql new file mode 100644 index 0000000..fa84c69 --- /dev/null +++ b/models/evm/streamline/silver/core/complete/streamline__complete_evm_traces.sql @@ -0,0 +1,34 @@ +-- depends_on: {{ ref('bronze_evm__streamline_traces') }} +{{ config ( + materialized = "incremental", + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", + tags = ['streamline_core_evm_complete'] +) }} + +SELECT + VALUE :BLOCK_NUMBER :: INT AS block_number, + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS complete_evm_traces_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__streamline_traces') }} +WHERE + inserted_timestamp >= ( + SELECT + COALESCE(MAX(inserted_timestamp), '1970-01-01' :: TIMESTAMP) inserted_timestamp + FROM + {{ this }}) + {% else %} + {{ ref('bronze_evm__streamline_FR_traces') }} + {% endif %} + + qualify(ROW_NUMBER() over (PARTITION BY block_number + ORDER BY + inserted_timestamp DESC)) = 1 diff --git a/models/evm/streamline/silver/core/complete/streamline__complete_evm_transactions.sql b/models/evm/streamline/silver/core/complete/streamline__complete_evm_transactions.sql new file mode 100644 index 0000000..ee9f8ab --- /dev/null +++ b/models/evm/streamline/silver/core/complete/streamline__complete_evm_transactions.sql @@ -0,0 +1,34 @@ +-- depends_on: {{ ref('bronze_evm__streamline_transactions') }} +{{ config ( + materialized = "incremental", + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", + tags = ['streamline_core_evm_complete'] +) }} + +SELECT + VALUE :BLOCK_NUMBER :: INT AS block_number, + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS complete_evm_transactions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__streamline_transactions') }} +WHERE + inserted_timestamp >= ( + SELECT + COALESCE(MAX(inserted_timestamp), '1970-01-01' :: TIMESTAMP) inserted_timestamp + FROM + {{ this }}) + {% else %} + {{ ref('bronze_evm__streamline_FR_transactions') }} + {% endif %} + + qualify(ROW_NUMBER() over (PARTITION BY block_number + ORDER BY + inserted_timestamp DESC)) = 1 diff --git a/models/evm/streamline/silver/core/history/streamline__evm_blocks_history.sql b/models/evm/streamline/silver/core/history/streamline__evm_blocks_history.sql new file mode 100644 index 0000000..51b465c --- /dev/null +++ b/models/evm/streamline/silver/core/history/streamline__evm_blocks_history.sql @@ -0,0 +1,85 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"evm_blocks", + "sql_limit" :"100", + "producer_batch_size" :"100000", + "worker_batch_size" :"10000", + "sql_source" :"{{this.identifier}}" } + ), + tags = ['streamline_core_evm_history'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_evm_block_lookback") }} +), +to_do AS ( + SELECT + block_number + FROM + {{ ref("streamline__evm_blocks") }} + WHERE + ( + block_number < ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + block_number + FROM + {{ ref("streamline__complete_evm_blocks") }} + WHERE + block_number < ( + SELECT + block_number + FROM + last_3_days + ) +), +ready_blocks AS ( + SELECT + block_number + FROM + to_do +) +SELECT + block_number, + ROUND( + block_number, + -3 + ) :: INT AS partition_key, + {{ target.database }}.live.udf_api( + 'POST', + '{Service}', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json', + 'User-Agent', + 'Flipside_Crypto/0.1' + ), + OBJECT_CONSTRUCT( + 'id', + block_number :: STRING, + 'jsonrpc', + '2.0', + 'method', + 'eth_getBlockByNumber', + 'params', + ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), FALSE)), + 'Vault/prod/m1/evm' + ) AS request + FROM + ready_blocks + ORDER BY + block_number DESC diff --git a/models/evm/streamline/silver/core/history/streamline__evm_confirm_blocks_history.sql b/models/evm/streamline/silver/core/history/streamline__evm_confirm_blocks_history.sql new file mode 100644 index 0000000..12881df --- /dev/null +++ b/models/evm/streamline/silver/core/history/streamline__evm_confirm_blocks_history.sql @@ -0,0 +1,94 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"evm_confirm_blocks", + "sql_limit" :"100", + "producer_batch_size" :"10000", + "worker_batch_size" :"5000", + "sql_source" :"{{this.identifier}}" } + ), + tags = ['streamline_core_evm_history'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_evm_block_lookback") }} +), +look_back AS ( + SELECT + block_number + FROM + {{ ref("_max_evm_block_by_hour") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 6 +), +to_do AS ( + SELECT + block_number + FROM + {{ ref("streamline__evm_blocks") }} + WHERE + block_number IS NOT NULL + AND block_number < ( + SELECT + block_number + FROM + last_3_days + ) + EXCEPT + SELECT + block_number + FROM + {{ ref("streamline__complete_evm_confirmed_blocks") }} + WHERE + block_number IS NOT NULL + AND block_number < ( + SELECT + block_number + FROM + last_3_days + ) +), +ready_blocks AS ( + SELECT + block_number + FROM + to_do +) +SELECT + block_number, + ROUND( + block_number, + -3 + ) :: INT AS partition_key, + {{ target.database }}.live.udf_api( + 'POST', + '{Service}/{Authentication}', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json', + 'User-Agent', + 'Flipside_Crypto/0.1' + ), + OBJECT_CONSTRUCT( + 'id', + block_number :: STRING, + 'jsonrpc', + '2.0', + 'method', + 'eth_getBlockByNumber', + 'params', + ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), FALSE)), + 'Vault/prod/m1/evm' + ) AS request + FROM + ready_blocks + ORDER BY + block_number ASC diff --git a/models/evm/streamline/silver/core/history/streamline__evm_receipts_history.sql b/models/evm/streamline/silver/core/history/streamline__evm_receipts_history.sql new file mode 100644 index 0000000..4f43b68 --- /dev/null +++ b/models/evm/streamline/silver/core/history/streamline__evm_receipts_history.sql @@ -0,0 +1,86 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"evm_receipts", + "sql_limit" :"100", + "producer_batch_size" :"100000", + "worker_batch_size" :"10000", + "sql_source" :"{{this.identifier}}", + "exploded_key": tojson(["result"]) } + ), + tags = ['streamline_core_evm_history'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_evm_block_lookback") }} +), +to_do AS ( + SELECT + block_number + FROM + {{ ref("streamline__evm_blocks") }} + WHERE + ( + block_number < ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + block_number + FROM + {{ ref("streamline__complete_evm_receipts") }} + WHERE + block_number < ( + SELECT + block_number + FROM + last_3_days + ) +), +ready_blocks AS ( + SELECT + block_number + FROM + to_do +) +SELECT + block_number, + ROUND( + block_number, + -3 + ) :: INT AS partition_key, + {{ target.database }}.live.udf_api( + 'POST', + '{Service}/{Authentication}', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json', + 'User-Agent', +' Flipside_Crypto/0.1' + ), + OBJECT_CONSTRUCT( + 'id', + block_number :: STRING, + 'jsonrpc', + '2.0', + 'method', + 'eth_getBlockReceipts', + 'params', + ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number))), + 'Vault/prod/m1/evm' + ) AS request + FROM + ready_blocks + ORDER BY + block_number DESC diff --git a/models/evm/streamline/silver/core/history/streamline__evm_traces_history.sql b/models/evm/streamline/silver/core/history/streamline__evm_traces_history.sql new file mode 100644 index 0000000..8207601 --- /dev/null +++ b/models/evm/streamline/silver/core/history/streamline__evm_traces_history.sql @@ -0,0 +1,87 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"evm_traces", + "sql_limit" :"100", + "producer_batch_size" :"1000", + "worker_batch_size" :"100", + "sql_source" :"{{this.identifier}}", + "exploded_key": tojson(["result"]) } + ), + tags = ['streamline_core_evm_history'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_evm_block_lookback") }} +), +to_do AS ( + SELECT + block_number + FROM + {{ ref("streamline__evm_blocks") }} + WHERE + ( + block_number < ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + block_number + FROM + {{ ref("streamline__complete_evm_traces") }} + WHERE + block_number < ( + SELECT + block_number + FROM + last_3_days + ) +), +ready_blocks AS ( + SELECT + block_number + FROM + to_do +) +SELECT + block_number, + ROUND( + block_number, + -3 + ) :: INT AS partition_key, + {{ target.database }}.live.udf_api( + 'POST', + '{Service}/{Authentication}', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json', + 'User-Agent', + 'Flipside_Crypto/0.1' + ), + OBJECT_CONSTRUCT( + 'id', + block_number :: STRING, + 'jsonrpc', + '2.0', + 'method', + 'debug_traceBlockByNumber', + 'params', + ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), OBJECT_CONSTRUCT('tracer', 'callTracer', 'timeout', '30s')) + ), + 'Vault/prod/m1/evm' + ) AS request +FROM + ready_blocks +ORDER BY + block_number DESC diff --git a/models/evm/streamline/silver/core/history/streamline__evm_transactions_history.sql b/models/evm/streamline/silver/core/history/streamline__evm_transactions_history.sql new file mode 100644 index 0000000..ccb96c3 --- /dev/null +++ b/models/evm/streamline/silver/core/history/streamline__evm_transactions_history.sql @@ -0,0 +1,86 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"evm_transactions", + "sql_limit" :"100", + "producer_batch_size" :"100000", + "worker_batch_size" :"10000", + "sql_source" :"{{this.identifier}}", + "exploded_key": tojson(["result.transactions"]) } + ), + tags = ['streamline_core_evm_history'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_evm_block_lookback") }} +), +to_do AS ( + SELECT + block_number + FROM + {{ ref("streamline__evm_blocks") }} + WHERE + ( + block_number < ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + block_number + FROM + {{ ref("streamline__complete_evm_transactions") }} + WHERE + block_number < ( + SELECT + block_number + FROM + last_3_days + ) +), +ready_blocks AS ( + SELECT + block_number + FROM + to_do +) +SELECT + block_number, + ROUND( + block_number, + -3 + ) :: INT AS partition_key, + {{ target.database }}.live.udf_api( + 'POST', + '{Service}/{Authentication}', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json', + 'User-Agent', + 'Flipside_Crypto/0.1' + ), + OBJECT_CONSTRUCT( + 'id', + block_number :: STRING, + 'jsonrpc', + '2.0', + 'method', + 'eth_getBlockByNumber', + 'params', + ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), TRUE)), + 'Vault/prod/m1/evm' + ) AS request + FROM + ready_blocks + ORDER BY + block_number DESC diff --git a/models/evm/streamline/silver/core/realtime/streamline__evm_blocks_realtime.sql b/models/evm/streamline/silver/core/realtime/streamline__evm_blocks_realtime.sql new file mode 100644 index 0000000..5e734bd --- /dev/null +++ b/models/evm/streamline/silver/core/realtime/streamline__evm_blocks_realtime.sql @@ -0,0 +1,64 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"evm_blocks", + "sql_limit" :"100", + "producer_batch_size" :"100000", + "worker_batch_size" :"10000", + "sql_source" :"{{this.identifier}}" } + ), + tags = ['streamline_core_evm_realtime'] +) }} + +WITH to_do AS ( + + SELECT + block_number + FROM + {{ ref("streamline__evm_blocks") }} + WHERE + block_number IS NOT NULL + EXCEPT + SELECT + block_number + FROM + {{ ref("streamline__complete_evm_blocks") }} +), +ready_blocks AS ( + SELECT + block_number + FROM + to_do +) +SELECT + block_number, + ROUND( + block_number, + -3 + ) :: INT AS partition_key, + {{ target.database }}.live.udf_api( + 'POST', + '{Service}', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json', + 'User-Agent', + 'Flipside_Crypto/0.1' + ), + OBJECT_CONSTRUCT( + 'id', + block_number :: STRING, + 'jsonrpc', + '2.0', + 'method', + 'eth_getBlockByNumber', + 'params', + ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), FALSE)), + 'Vault/prod/m1/evm' + ) AS request + FROM + ready_blocks + ORDER BY + block_number DESC diff --git a/models/evm/streamline/silver/core/realtime/streamline__evm_confirm_blocks_realtime.sql b/models/evm/streamline/silver/core/realtime/streamline__evm_confirm_blocks_realtime.sql new file mode 100644 index 0000000..123c764 --- /dev/null +++ b/models/evm/streamline/silver/core/realtime/streamline__evm_confirm_blocks_realtime.sql @@ -0,0 +1,64 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"evm_confirm_blocks", + "sql_limit" :"25000", + "producer_batch_size" :"10000", + "worker_batch_size" :"5000", + "sql_source" :"{{this.identifier}}" } + ), + tags = ['streamline_core_evm_realtime'] +) }} + +WITH to_do AS ( + + SELECT + block_number + FROM + {{ ref("streamline__evm_blocks") }} + WHERE + block_number IS NOT NULL + EXCEPT + SELECT + block_number + FROM + {{ ref("streamline__complete_evm_blocks") }} +), +ready_blocks AS ( + SELECT + block_number + FROM + to_do +) +SELECT + block_number, + ROUND( + block_number, + -3 + ) :: INT AS partition_key, + {{ target.database }}.live.udf_api( + 'POST', + '{Service}/{Authentication}', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json', + 'User-Agent', + 'Flipside_Crypto/0.1' + ), + OBJECT_CONSTRUCT( + 'id', + block_number :: STRING, + 'jsonrpc', + '2.0', + 'method', + 'eth_getBlockByNumber', + 'params', + ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), FALSE)), + 'Vault/prod/m1/evm' + ) AS request + FROM + ready_blocks + ORDER BY + block_number ASC diff --git a/models/evm/streamline/silver/core/realtime/streamline__evm_receipts_realtime.sql b/models/evm/streamline/silver/core/realtime/streamline__evm_receipts_realtime.sql new file mode 100644 index 0000000..092edda --- /dev/null +++ b/models/evm/streamline/silver/core/realtime/streamline__evm_receipts_realtime.sql @@ -0,0 +1,65 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"evm_receipts", + "sql_limit" :"25000", + "producer_batch_size" :"100000", + "worker_batch_size" :"10000", + "sql_source" :"{{this.identifier}}", + "exploded_key": tojson(["result"]) } + ), + tags = ['streamline_core_evm_realtime'] +) }} + +WITH to_do AS ( + + SELECT + block_number + FROM + {{ ref("streamline__evm_blocks") }} + WHERE + block_number IS NOT NULL + EXCEPT + SELECT + block_number + FROM + {{ ref("streamline__complete_evm_blocks") }} +), +ready_blocks AS ( + SELECT + block_number + FROM + to_do +) +SELECT + block_number, + ROUND( + block_number, + -3 + ) :: INT AS partition_key, + {{ target.database }}.live.udf_api( + 'POST', + '{Service}/{Authentication}', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json', + 'User-Agent', + 'Flipside_Crypto/0.1' + ), + OBJECT_CONSTRUCT( + 'id', + block_number :: STRING, + 'jsonrpc', + '2.0', + 'method', + 'eth_getBlockReceipts', + 'params', + ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number))), + 'Vault/prod/m1/evm' + ) AS request + FROM + ready_blocks + ORDER BY + block_number DESC diff --git a/models/evm/streamline/silver/core/realtime/streamline__evm_traces_realtime.sql b/models/evm/streamline/silver/core/realtime/streamline__evm_traces_realtime.sql new file mode 100644 index 0000000..c354605 --- /dev/null +++ b/models/evm/streamline/silver/core/realtime/streamline__evm_traces_realtime.sql @@ -0,0 +1,66 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"evm_traces", + "sql_limit" :"25000", + "producer_batch_size" :"1000", + "worker_batch_size" :"100", + "sql_source" :"{{this.identifier}}", + "exploded_key": tojson(["result"]) } + ), + tags = ['streamline_core_evm_realtime'] +) }} + +WITH to_do AS ( + + SELECT + block_number + FROM + {{ ref("streamline__evm_blocks") }} + WHERE + block_number IS NOT NULL + EXCEPT + SELECT + block_number + FROM + {{ ref("streamline__complete_evm_blocks") }} +), +ready_blocks AS ( + SELECT + block_number + FROM + to_do +) +SELECT + block_number, + ROUND( + block_number, + -3 + ) :: INT AS partition_key, + {{ target.database }}.live.udf_api( + 'POST', + '{Service}/{Authentication}', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json', + 'User-Agent', + 'Flipside_Crypto/0.1' + ), + OBJECT_CONSTRUCT( + 'id', + block_number :: STRING, + 'jsonrpc', + '2.0', + 'method', + 'debug_traceBlockByNumber', + 'params', + ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), OBJECT_CONSTRUCT('tracer', 'callTracer', 'timeout', '30s')) + ), + 'Vault/prod/m1/evm' + ) AS request +FROM + ready_blocks +ORDER BY + block_number DESC diff --git a/models/evm/streamline/silver/core/realtime/streamline__evm_transactions_realtime.sql b/models/evm/streamline/silver/core/realtime/streamline__evm_transactions_realtime.sql new file mode 100644 index 0000000..f13a925 --- /dev/null +++ b/models/evm/streamline/silver/core/realtime/streamline__evm_transactions_realtime.sql @@ -0,0 +1,65 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"evm_transactions", + "sql_limit" :"25000", + "producer_batch_size" :"100000", + "worker_batch_size" :"10000", + "sql_source" :"{{this.identifier}}", + "exploded_key": tojson(["result.transactions"]) } + ), + tags = ['streamline_core_evm_realtime'] +) }} + +WITH to_do AS ( + + SELECT + block_number + FROM + {{ ref("streamline__evm_blocks") }} + WHERE + block_number IS NOT NULL + EXCEPT + SELECT + block_number + FROM + {{ ref("streamline__complete_evm_blocks") }} +), +ready_blocks AS ( + SELECT + block_number + FROM + to_do +) +SELECT + block_number, + ROUND( + block_number, + -3 + ) :: INT AS partition_key, + {{ target.database }}.live.udf_api( + 'POST', + '{Service}/{Authentication}', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json', + 'User-Agent', + 'Flipside_Crypto/0.1' + ), + OBJECT_CONSTRUCT( + 'id', + block_number :: STRING, + 'jsonrpc', + '2.0', + 'method', + 'eth_getBlockByNumber', + 'params', + ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), TRUE)), + 'Vault/prod/m1/evm' + ) AS request + FROM + ready_blocks + ORDER BY + block_number DESC diff --git a/models/evm/streamline/silver/core/retry/_missing_receipts.sql b/models/evm/streamline/silver/core/retry/_missing_receipts.sql new file mode 100644 index 0000000..aa8c1dc --- /dev/null +++ b/models/evm/streamline/silver/core/retry/_missing_receipts.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "ephemeral" +) }} + +WITH lookback AS ( + + SELECT + block_number + FROM + {{ ref("_evm_block_lookback") }} +) +SELECT + DISTINCT t.block_number AS block_number +FROM + {{ ref("silver_evm__transactions") }} + t + LEFT JOIN {{ ref("silver_evm__receipts") }} + r USING ( + block_number, + block_hash, + tx_hash + ) +WHERE + r.tx_hash IS NULL + AND t.block_number >= ( + SELECT + block_number + FROM + lookback + ) + AND t.block_timestamp >= DATEADD('hour', -84, SYSDATE()) + AND ( + r._inserted_timestamp >= DATEADD('hour', -84, SYSDATE()) + OR r._inserted_timestamp IS NULL) diff --git a/models/evm/streamline/silver/core/retry/_missing_traces.sql b/models/evm/streamline/silver/core/retry/_missing_traces.sql new file mode 100644 index 0000000..b3bce2d --- /dev/null +++ b/models/evm/streamline/silver/core/retry/_missing_traces.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "ephemeral" +) }} + +WITH lookback AS ( + + SELECT + block_number + FROM + {{ ref("_evm_block_lookback") }} +) +SELECT + DISTINCT tx.block_number block_number +FROM + {{ ref("silver_evm__transactions") }} + tx + LEFT JOIN {{ ref("silver_evm__traces") }} + tr + ON tx.block_number = tr.block_number + AND tx.tx_hash = tr.tx_hash + AND tr.block_timestamp >= DATEADD('hour', -84, SYSDATE()) +WHERE + tx.block_timestamp >= DATEADD('hour', -84, SYSDATE()) + AND tr.tx_hash IS NULL + AND tx.block_number >= ( + SELECT + block_number + FROM + lookback + ) + AND tx.block_number NOT IN ( + 81772279, + 81772251 + ) diff --git a/models/evm/streamline/silver/core/retry/_unconfirmed_blocks.sql b/models/evm/streamline/silver/core/retry/_unconfirmed_blocks.sql new file mode 100644 index 0000000..de3feb4 --- /dev/null +++ b/models/evm/streamline/silver/core/retry/_unconfirmed_blocks.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "ephemeral" +) }} + +WITH lookback AS ( + + SELECT + block_number + FROM + {{ ref("_evm_block_lookback") }} +) +SELECT + DISTINCT cb.block_number AS block_number +FROM + {{ ref("silver_evm__confirmed_blocks") }} + cb + LEFT JOIN {{ ref("silver_evm__transactions") }} + txs USING ( + block_number, + block_hash, + tx_hash + ) +WHERE + txs.tx_hash IS NULL + AND cb.block_number >= ( + SELECT + block_number + FROM + lookback + ) + AND cb._inserted_timestamp >= DATEADD('hour', -84, SYSDATE()) + AND ( + txs._inserted_timestamp >= DATEADD('hour', -84, SYSDATE()) + OR txs._inserted_timestamp IS NULL) diff --git a/models/evm/streamline/silver/decoder/complete/streamline__complete_decode_logs.sql b/models/evm/streamline/silver/decoder/complete/streamline__complete_decode_logs.sql new file mode 100644 index 0000000..ed3eb3d --- /dev/null +++ b/models/evm/streamline/silver/decoder/complete/streamline__complete_decode_logs.sql @@ -0,0 +1,31 @@ +-- depends_on: {{ ref('bronze_evm__decoded_logs') }} +{{ config ( + materialized = "incremental", + unique_key = "_log_id", + cluster_by = "ROUND(block_number, -3)", + merge_update_columns = ["_log_id"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(_log_id)", + tags = ['streamline_decoded_logs_complete'] +) }} + +SELECT + block_number, + id AS _log_id, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze_evm__decoded_logs') }} +WHERE + TO_TIMESTAMP_NTZ(_inserted_timestamp) >= ( + SELECT + COALESCE(MAX(TO_TIMESTAMP_NTZ(_inserted_timestamp)), '1970-01-01 00:00:00') _inserted_timestamp + FROM + {{ this }}) + {% else %} + {{ ref('bronze_evm__fr_decoded_logs') }} + {% endif %} + + qualify(ROW_NUMBER() over (PARTITION BY id + ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/evm/streamline/silver/decoder/streamline__evm_blocks.sql b/models/evm/streamline/silver/decoder/streamline__evm_blocks.sql new file mode 100644 index 0000000..0c6e2df --- /dev/null +++ b/models/evm/streamline/silver/decoder/streamline__evm_blocks.sql @@ -0,0 +1,26 @@ +{{ config ( + materialized = "view", + tags = ['streamline_core_evm_complete'] +) }} + +SELECT + _id AS block_number, + REPLACE( + concat_ws('', '0x', to_char(block_number, 'XXXXXXXX')), + ' ', + '' + ) AS block_number_hex +FROM + {{ source( + 'crosschain_silver', + 'number_sequence' + ) }} +WHERE + _id <= ( + SELECT + MAX(block_number) + FROM + {{ ref('streamline__chainhead') }} + ) +ORDER BY + _id ASC diff --git a/models/github_actions/github_actions__current_task_status.sql b/models/github_actions/github_actions__current_task_status.sql new file mode 100644 index 0000000..577a226 --- /dev/null +++ b/models/github_actions/github_actions__current_task_status.sql @@ -0,0 +1,6 @@ +{{ config( + materialized = 'view', + tags = ['gha_tasks'] +) }} + +{{ fsc_utils.gha_task_current_status_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__current_task_status.yml b/models/github_actions/github_actions__current_task_status.yml new file mode 100644 index 0000000..e80ad12 --- /dev/null +++ b/models/github_actions/github_actions__current_task_status.yml @@ -0,0 +1,17 @@ +version: 2 +models: + - name: github_actions__current_task_status + columns: + - name: PIPELINE_ACTIVE + tests: + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: + - TRUE + - name: SUCCESSES + tests: + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: + - 204 + config: + severity: warn + warn_if: ">0" \ No newline at end of file diff --git a/models/github_actions/github_actions__task_history.sql b/models/github_actions/github_actions__task_history.sql new file mode 100644 index 0000000..9c35ce7 --- /dev/null +++ b/models/github_actions/github_actions__task_history.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_history_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__task_performance.sql b/models/github_actions/github_actions__task_performance.sql new file mode 100644 index 0000000..117ded5 --- /dev/null +++ b/models/github_actions/github_actions__task_performance.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_performance_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__task_schedule.sql b/models/github_actions/github_actions__task_schedule.sql new file mode 100644 index 0000000..ff95a44 --- /dev/null +++ b/models/github_actions/github_actions__task_schedule.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_schedule_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__tasks.sql b/models/github_actions/github_actions__tasks.sql new file mode 100644 index 0000000..feab82a --- /dev/null +++ b/models/github_actions/github_actions__tasks.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_tasks_view() }} \ No newline at end of file diff --git a/models/gold/core/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql new file mode 100644 index 0000000..efaca81 --- /dev/null +++ b/models/gold/core/core__fact_blocks.sql @@ -0,0 +1,33 @@ +{{ config( + materialized = 'incremental', + unique_key = "block_number", + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE'], + tags = ['core','full_test'] +) }} + +SELECT + block_number, + block_timestamp, + block_hash, + first_version, + last_version, + tx_count_from_versions AS tx_count, + blocks_id AS fact_blocks_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref( + 'silver__blocks' + ) }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/core/core__fact_blocks.yml b/models/gold/core/core__fact_blocks.yml new file mode 100644 index 0000000..0cc900d --- /dev/null +++ b/models/gold/core/core__fact_blocks.yml @@ -0,0 +1,26 @@ +version: 2 +models: + - name: core__fact_blocks + description: '{{ doc("core__fact_blocks") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("block_timestamp") }}' + - name: BLOCK_HASH + description: '{{ doc("block_hash") }}' + - name: FIRST_VERSION + description: '{{ doc("first_version") }}' + - name: LAST_VERSION + description: '{{ doc("last_version") }}' + - name: TX_COUNT + description: '{{ doc("tx_count") }}' + - name: FACT_BLOCKS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + + diff --git a/models/gold/core/core__fact_changes.sql b/models/gold/core/core__fact_changes.sql new file mode 100644 index 0000000..d4ee1cb --- /dev/null +++ b/models/gold/core/core__fact_changes.sql @@ -0,0 +1,48 @@ +{{ config( + materialized = 'incremental', + unique_key = ['tx_hash','change_index'], + incremental_strategy = 'merge', + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(version,tx_hash, change_type,inner_change_type,change_address,change_module,change_resource,payload_function);", + tags = ['core','full_test'] +) }} + +SELECT + block_number, + block_timestamp, + version, + tx_hash, + success, + tx_type, + payload_function, + change_index, + change_data, + change_type, + address, + handle, + inner_change_type, + change_address, + change_module, + change_resource, + key, + VALUE, + state_key_hash, + changes_id AS fact_changes_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref( + 'silver__changes' + ) }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/core/core__fact_changes.yml b/models/gold/core/core__fact_changes.yml new file mode 100644 index 0000000..3e37015 --- /dev/null +++ b/models/gold/core/core__fact_changes.yml @@ -0,0 +1,49 @@ +version: 2 +models: + - name: core__fact_changes + description: '{{ doc("core__fact_changes") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("tx_hash") }}' + - name: VERSION + description: '{{ doc("version") }}' + - name: SUCCESS + description: '{{ doc("success") }}' + - name: TX_TYPE + description: '{{ doc("tx_type") }}' + - name: PAYLOAD_FUNCTION + description: '{{ doc("payload_function") }}' + - name: CHANGE_INDEX + description: '{{ doc("change_index") }}' + - name: CHANGE_DATA + description: '{{ doc("change_data") }}' + - name: CHANGE_TYPE + description: '{{ doc("change_type") }}' + - name: ADDRESS + description: '{{ doc("address_change") }}' + - name: HANDLE + description: '{{ doc("handle_change") }}' + - name: INNER_CHANGE_TYPE + description: '{{ doc("inner_change_type") }}' + - name: CHANGE_ADDRESS + description: '{{ doc("change_address") }}' + - name: CHANGE_MODULE + description: '{{ doc("change_module") }}' + - name: CHANGE_RESOURCE + description: '{{ doc("change_resource") }}' + - name: KEY + description: '{{ doc("key_change") }}' + - name: VALUE + description: '{{ doc("value_change") }}' + - name: STATE_KEY_HASH + - name: FACT_CHANGES_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/core/core__fact_events.sql b/models/gold/core/core__fact_events.sql new file mode 100644 index 0000000..b49a482 --- /dev/null +++ b/models/gold/core/core__fact_events.sql @@ -0,0 +1,45 @@ +{{ config( + materialized = 'incremental', + unique_key = ['tx_hash','event_index'], + incremental_strategy = 'merge', + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(version,tx_hash, event_type,event_address,event_module,event_resource,payload_function);", + tags = ['core','full_test'] +) }} + +SELECT + block_number, + block_timestamp, + version, + tx_hash, + success, + tx_type, + payload_function, + event_index, + event_type, + event_address, + event_module, + event_resource, + event_data, + account_address, + creation_number, + sequence_number, + events_id AS fact_events_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref( + 'silver__events' + ) }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/core/core__fact_events.yml b/models/gold/core/core__fact_events.yml new file mode 100644 index 0000000..94fbce6 --- /dev/null +++ b/models/gold/core/core__fact_events.yml @@ -0,0 +1,44 @@ +version: 2 +models: + - name: core__fact_events + description: '{{ doc("core__fact_events") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("tx_hash") }}' + - name: VERSION + description: '{{ doc("version") }}' + - name: SUCCESS + description: '{{ doc("success") }}' + - name: TX_TYPE + description: '{{ doc("tx_type") }}' + - name: PAYLOAD_FUNCTION + description: '{{ doc("payload_function") }}' + - name: EVENT_INDEX + description: '{{ doc("event_index") }}' + - name: EVENT_TYPE + description: '{{ doc("event_type") }}' + - name: EVENT_ADDRESS + description: '{{ doc("event_address") }}' + - name: EVENT_MODULE + description: '{{ doc("event_module") }}' + - name: EVENT_RESOURCE + description: '{{ doc("event_resource") }}' + - name: EVENT_DATA + description: '{{ doc("event_data") }}' + - name: ACCOUNT_ADDRESS + description: '{{ doc("address_event") }}' + - name: CREATION_NUMBER + description: '{{ doc("creation_number") }}' + - name: SEQUENCE_NUMBER + description: '{{ doc("sequence_number") }}' + - name: FACT_EVENTS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql new file mode 100644 index 0000000..a9943e9 --- /dev/null +++ b/models/gold/core/core__fact_transactions.sql @@ -0,0 +1,61 @@ +{{ config( + materialized = 'incremental', + unique_key = ['tx_hash','block_timestamp::DATE'], + incremental_strategy = 'merge', + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(version,tx_hash,payload_function,sender);", + tags = ['core','full_test'] +) }} + +SELECT + b.block_number, + A.block_timestamp, + A.version, + A.tx_hash, + A.data :success :: BOOLEAN AS success, + A.tx_type, + A.data :sender :: STRING AS sender, + A.data :signature :: STRING AS signature, + A.data :payload AS payload, + A.data :payload :function :: STRING AS payload_function, + A.data :changes AS changes, + A.data :events AS events, + A.data: gas_unit_price :: bigint AS gas_unit_price, + A.data :gas_used :: INT AS gas_used, + A.data :max_gas_amount :: bigint AS max_gas_amount, + A.data :expiration_timestamp_secs :: bigint AS expiration_timestamp_secs, + A.data :vm_status :: STRING AS vm_status, + A.data :state_change_hash :: STRING AS state_change_hash, + A.data :accumulator_root_hash :: STRING AS accumulator_root_hash, + A.data :event_root_hash :: STRING AS event_root_hash, + A.data :state_checkpoint_hash :: STRING AS state_checkpoint_hash, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS fact_transactions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref( + 'silver__transactions' + ) }} A + JOIN {{ ref( + 'silver__blocks' + ) }} + b + ON A.version BETWEEN b.first_version + AND b.last_version + +{% if is_incremental() %} +WHERE + GREATEST( + A.modified_timestamp, + b.modified_timestamp + ) >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/core/core__fact_transactions.yml b/models/gold/core/core__fact_transactions.yml new file mode 100644 index 0000000..0b0807d --- /dev/null +++ b/models/gold/core/core__fact_transactions.yml @@ -0,0 +1,51 @@ +version: 2 +models: + - name: core__fact_transactions + description: '{{ doc("core__fact_transactions") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("tx_hash") }}' + - name: VERSION + description: '{{ doc("version") }}' + - name: SUCCESS + description: '{{ doc("success") }}' + - name: TX_TYPE + description: '{{ doc("tx_type") }}' + - name: SENDER + description: '{{ doc("sender") }}' + - name: SIGNATURE + description: '{{ doc("signature") }}' + - name: PAYLOAD + description: '{{ doc("payload") }}' + - name: PAYLOAD_FUNCTION + description: '{{ doc("payload_function") }}' + - name: CHANGES + description: '{{ doc("changes") }}' + - name: EVENTS + description: '{{ doc("events") }}' + - name: GAS_UNIT_PRICE + description: '{{ doc("gas_unit_price") }}' + - name: GAS_USED + description: '{{ doc("gas_used") }}' + - name: MAX_GAS_AMOUNT + description: '{{ doc("max_gas_amount") }}' + - name: EXPIRATION_TIMESTAMP_SECS + description: '{{ doc("expiration_timestamp_secs") }}' + - name: VM_STATUS + description: '{{ doc("vm_status") }}' + - name: STATE_CHANGE_HASH + - name: ACCUMULATOR_ROOT_HASH + description: '{{ doc("accumulator_root_hash") }}' + - name: EVENT_ROOT_HASH + description: '{{ doc("event_root_hash") }}' + - name: FACT_TRANSACTIONS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/core/core__fact_transactions_block_metadata.sql b/models/gold/core/core__fact_transactions_block_metadata.sql new file mode 100644 index 0000000..d7c5b6f --- /dev/null +++ b/models/gold/core/core__fact_transactions_block_metadata.sql @@ -0,0 +1,63 @@ +{{ config( + materialized = 'incremental', + unique_key = ['tx_hash','block_timestamp::DATE'], + incremental_strategy = 'merge', + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(version,tx_hash);", + tags = ['core','full_test'] +) }} + +SELECT + b.block_number, + A.block_timestamp, + A.version, + A.tx_hash, + A.data :success :: BOOLEAN AS success, + A.tx_type, + A.data :sender :: STRING AS sender, + A.data :signature :: STRING AS signature, + A.data :payload AS payload, + A.data :payload :function :: STRING AS payload_function, + A.data :changes AS changes, + A.data :events AS events, + A.data :failed_proposer_indices :: STRING AS failed_proposer_indices, + A.data :id :: STRING AS id, + A.data :previous_block_votes_bitvec :: STRING AS previous_block_votes_bitvec, + A.data :proposer :: STRING AS proposer, + A.data :ROUND :: INT AS ROUND, + A.data :vm_status :: STRING AS vm_status, + A.data :state_change_hash :: STRING AS state_change_hash, + A.data :accumulator_root_hash :: STRING AS accumulator_root_hash, + A.data :event_root_hash :: STRING AS event_root_hash, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS fact_transactions_block_metadata_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref( + 'silver__transactions' + ) }} A + JOIN {{ ref('silver__blocks') }} + b + ON A.version BETWEEN b.first_version + AND b.last_version +WHERE + LEFT( + tx_type, + 5 + ) = 'block' + +{% if is_incremental() %} +AND GREATEST( + A.modified_timestamp, + b.modified_timestamp +) >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} diff --git a/models/gold/core/core__fact_transactions_block_metadata.yml b/models/gold/core/core__fact_transactions_block_metadata.yml new file mode 100644 index 0000000..94180bc --- /dev/null +++ b/models/gold/core/core__fact_transactions_block_metadata.yml @@ -0,0 +1,44 @@ +version: 2 +models: + - name: core__fact_transactions_block_metadata + description: '{{ doc("core__fact_transactions_block_metadata") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("tx_hash") }}' + - name: VERSION + description: '{{ doc("version") }}' + - name: SUCCESS + description: '{{ doc("success") }}' + - name: TX_TYPE + description: '{{ doc("tx_type") }}' + - name: EPOCH + description: '{{ doc("epoch") }}' + - name: EVENTS + description: '{{ doc("events") }}' + - name: CHANGES + description: '{{ doc("changes") }}' + - name: FAILED_PROPOSER_INDICES + - name: ID + - name: PREVIOUS_BLOCK_VOTES_BITVEC + - name: PROPOSER + description: '{{ doc("proposer") }}' + - name: ROUND + description: '{{ doc("round") }}' + - name: VM_STATUS + description: '{{ doc("vm_status") }}' + - name: STATE_CHANGE_HASH + - name: ACCUMULATOR_ROOT_HASH + description: '{{ doc("accumulator_root_hash") }}' + - name: EVENT_ROOT_HASH + description: '{{ doc("event_root_hash") }}' + - name: FACT_TRANSACTIONS_BLOCK_METADATA_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/core/core__fact_transactions_state_checkpoint.sql b/models/gold/core/core__fact_transactions_state_checkpoint.sql new file mode 100644 index 0000000..6057bd4 --- /dev/null +++ b/models/gold/core/core__fact_transactions_state_checkpoint.sql @@ -0,0 +1,43 @@ +{{ config( + materialized = 'incremental', + unique_key = ['tx_hash','block_timestamp::DATE'], + incremental_strategy = 'merge', + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(version,tx_hash);", + tags = ['core','full_test'] +) }} + +SELECT + block_number, + block_timestamp, + version, + tx_hash, + success, + tx_type, + vm_status, + state_checkpoint_hash, + accumulator_root_hash, + event_root_hash, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS fact_transactions_state_checkpoint_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref('core__fact_transactions') }} +WHERE + LEFT( + tx_type, + 5 + ) = 'state' + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} diff --git a/models/gold/core/core__fact_transactions_state_checkpoint.yml b/models/gold/core/core__fact_transactions_state_checkpoint.yml new file mode 100644 index 0000000..a087b39 --- /dev/null +++ b/models/gold/core/core__fact_transactions_state_checkpoint.yml @@ -0,0 +1,31 @@ +version: 2 +models: + - name: core__fact_transactions_state_checkpoint + description: '{{ doc("core__fact_transactions_state_checkpoint") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("tx_hash") }}' + - name: VERSION + description: '{{ doc("version") }}' + - name: SUCCESS + description: '{{ doc("success") }}' + - name: TX_TYPE + description: '{{ doc("tx_type") }}' + - name: VM_STATUS + description: '{{ doc("vm_status") }}' + - name: STATE_CHECKPOINT_HASH + - name: ACCUMULATOR_ROOT_HASH + description: '{{ doc("accumulator_root_hash") }}' + - name: EVENT_ROOT_HASH + description: '{{ doc("event_root_hash") }}' + - name: FACT_TRANSACTIONS_STATE_CHECKPOINT_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/core/tests/blocks/test_core__blocks_full.sql b/models/gold/core/tests/blocks/test_core__blocks_full.sql new file mode 100644 index 0000000..e7c6bbf --- /dev/null +++ b/models/gold/core/tests/blocks/test_core__blocks_full.sql @@ -0,0 +1,11 @@ +{{ config( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref( + 'core__fact_blocks' + ) }} diff --git a/models/gold/core/tests/blocks/test_core__blocks_full.yml b/models/gold/core/tests/blocks/test_core__blocks_full.yml new file mode 100644 index 0000000..0987eaf --- /dev/null +++ b/models/gold/core/tests/blocks/test_core__blocks_full.yml @@ -0,0 +1,66 @@ +version: 2 +models: + - name: test_core__blocks_full + + tests: + - fsc_utils.sequence_gaps: + column_name: BLOCK_NUMBER + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - unique + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: FIRST_VERSION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: LAST_VERSION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_COUNT + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: FACT_BLOCKS_ID + tests: + - not_null + - unique + - name: INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null \ No newline at end of file diff --git a/models/gold/core/tests/blocks/test_core__blocks_recent.sql b/models/gold/core/tests/blocks/test_core__blocks_recent.sql new file mode 100644 index 0000000..eed3099 --- /dev/null +++ b/models/gold/core/tests/blocks/test_core__blocks_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('core__fact_blocks') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/gold/core/tests/blocks/test_core__blocks_recent.yml b/models/gold/core/tests/blocks/test_core__blocks_recent.yml new file mode 100644 index 0000000..1bb2a7b --- /dev/null +++ b/models/gold/core/tests/blocks/test_core__blocks_recent.yml @@ -0,0 +1,21 @@ +version: 2 +models: + - name: test_core__blocks_recent + + tests: + - fsc_utils.sequence_gaps: + column_name: BLOCK_NUMBER + config: + severity: error + error_if: ">100" + columns: + - name: BLOCK_NUMBER + tests: + - unique + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 \ No newline at end of file diff --git a/models/gold/core/tests/transactions/test_core__transactions_full.sql b/models/gold/core/tests/transactions/test_core__transactions_full.sql new file mode 100644 index 0000000..771d173 --- /dev/null +++ b/models/gold/core/tests/transactions/test_core__transactions_full.sql @@ -0,0 +1,11 @@ +{{ config( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref( + 'core__fact_transactions' + ) }} diff --git a/models/gold/core/tests/transactions/test_core__transactions_full.yml b/models/gold/core/tests/transactions/test_core__transactions_full.yml new file mode 100644 index 0000000..9d1ff5f --- /dev/null +++ b/models/gold/core/tests/transactions/test_core__transactions_full.yml @@ -0,0 +1,186 @@ +version: 2 +models: + - name: test_core__transactions_full + + tests: + - fsc_utils.sequence_gaps: + column_name: VERSION + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - unique + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: VERSION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: SUCCESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + - name: TX_TYPE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: ACCUMULATOR_ROOT_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: CHANGES + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - name: EPOCH + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EVENT_ROOT_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: EVENTS + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - name: EXPIRATION_TIMESTAMP_SECS + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: FAILED_PROPOSER_INDICES + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - name: GAS_UNIT_PRICE + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: ID + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: MAX_GAS_AMOUNT + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: PAYLOAD + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - name: PREVIOUS_BLOCK_VOTES_BITVEC + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - name: PROPOSER + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: ROUND + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: SENDER + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: SIGNATURE + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: STATE_CHANGE_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: STATE_CHECKPOINT_HASH + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: VM_STATUS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: FACT_TRANSACTIONS_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null diff --git a/models/gold/core/tests/transactions/test_core__transactions_recent.sql b/models/gold/core/tests/transactions/test_core__transactions_recent.sql new file mode 100644 index 0000000..bfe8640 --- /dev/null +++ b/models/gold/core/tests/transactions/test_core__transactions_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_date + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_date DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('core__fact_transactions') }} +WHERE + block_timestamp :: DATE >= ( + SELECT + block_date + FROM + last_3_days + ) diff --git a/models/gold/core/tests/transactions/test_core__transactions_recent.yml b/models/gold/core/tests/transactions/test_core__transactions_recent.yml new file mode 100644 index 0000000..5d48b53 --- /dev/null +++ b/models/gold/core/tests/transactions/test_core__transactions_recent.yml @@ -0,0 +1,29 @@ +version: 2 +models: + - name: test_core__transactions_recent + + tests: + - fsc_utils.sequence_gaps: + column_name: VERSION + config: + severity: error + error_if: ">100" + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + - name: TX_HASH + tests: + - not_null + - unique + - name: VERSION + tests: + - not_null + \ No newline at end of file diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.sql b/models/gold/stats/stats__ez_core_metrics_hourly.sql new file mode 100644 index 0000000..594e5dd --- /dev/null +++ b/models/gold/stats/stats__ez_core_metrics_hourly.sql @@ -0,0 +1,27 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STATS, METRICS, CORE, HOURLY', + }} }, + tags = ['noncore'] +) }} + +SELECT + block_timestamp_hour, + block_number_min, + block_number_max, + block_count, + transaction_count, + transaction_count_success, + transaction_count_failed, + unique_sender_count, + unique_payload_function_count, + total_fees AS total_fees_native, + NULL AS total_fees_usd, + core_metrics_hourly_id AS ez_core_metrics_hourly_id, + s.inserted_timestamp AS inserted_timestamp, + s.modified_timestamp AS modified_timestamp +FROM + {{ ref('silver__core_metrics_hourly') }} + s diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.yml b/models/gold/stats/stats__ez_core_metrics_hourly.yml new file mode 100644 index 0000000..d65016a --- /dev/null +++ b/models/gold/stats/stats__ez_core_metrics_hourly.yml @@ -0,0 +1,34 @@ +version: 2 +models: + - name: stats__ez_core_metrics_hourly + description: '{{ doc("ez_core_metrics_hourly_table_doc") }}' + + columns: + - name: BLOCK_TIMESTAMP_HOUR + description: '{{ doc("block_timestamp_hour") }}' + - name: BLOCK_NUMBER_MIN + description: '{{ doc("block_number_min") }}' + - name: BLOCK_NUMBER_MAX + description: '{{ doc("block_number_max") }}' + - name: BLOCK_COUNT + description: '{{ doc("block_count") }}' + - name: TRANSACTION_COUNT + description: '{{ doc("transaction_count") }}' + - name: TRANSACTION_COUNT_SUCCESS + description: '{{ doc("transaction_count_success") }}' + - name: TRANSACTION_COUNT_FAILED + description: '{{ doc("transaction_count_failed") }}' + - name: UNIQUE_SENDER_COUNT + description: '{{ doc("unique_sender_count") }}' + - name: UNIQUE_PAYLOAD_FUNCTION_COUNT + description: '{{ doc("unique_payload_function_count") }}' + - name: TOTAL_FEES_NATIVE + description: '{{ doc("total_fees_native") }}' + - name: TOTAL_FEES_USD + description: '{{ doc("total_fees_usd") }}' + - name: EZ_CORE_METRICS_HOURLY_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/silver/_observability/silver_observability__blocks_completeness.sql b/models/silver/_observability/silver_observability__blocks_completeness.sql new file mode 100644 index 0000000..8cf1c78 --- /dev/null +++ b/models/silver/_observability/silver_observability__blocks_completeness.sql @@ -0,0 +1,168 @@ +{{ config( + materialized = 'incremental', + unique_key = 'test_timestamp', + full_refresh = false, + tags = ['observability','recent_test'], + enabled = true +) }} + +WITH summary_stats AS ( + + SELECT + MIN(block_number) AS min_block, + MAX(block_number) AS max_block, + MIN(block_timestamp) AS min_block_timestamp, + MAX(block_timestamp) AS max_block_timestamp, + COUNT(1) AS blocks_tested + FROM + {{ ref('core__fact_blocks') }} + WHERE + block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + +{% if is_incremental() %} +AND ( + block_number >= ( + SELECT + MIN(block_number) + FROM + ( + SELECT + MIN(block_number) AS block_number + FROM + {{ ref('core__fact_blocks') }} + WHERE + block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) + AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + UNION + SELECT + MIN(VALUE) - 1 AS block_number + FROM + ( + SELECT + blocks_impacted_array + FROM + {{ this }} + qualify ROW_NUMBER() over ( + ORDER BY + test_timestamp DESC + ) = 1 + ), + LATERAL FLATTEN( + input => blocks_impacted_array + ) + ) + ) {% if var('OBSERV_FULL_TEST') %} + OR block_number >= 0 + {% endif %} +) +{% endif %} +), +block_range AS ( + SELECT + _id AS block_number + FROM + {{ source( + 'crosschain_silver', + 'number_sequence' + ) }} + WHERE + _id BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) +), +blocks AS ( + SELECT + l.block_number, + block_timestamp, + LAG( + l.block_number, + 1 + ) over ( + ORDER BY + l.block_number ASC + ) AS prev_BLOCK_NUMBER + FROM + {{ ref("core__fact_blocks") }} + l + INNER JOIN block_range b + ON l.block_number = b.block_number + AND l.block_number >= ( + SELECT + MIN(block_number) + FROM + block_range + ) +), +block_gen AS ( + SELECT + _id AS block_number + FROM + {{ source( + 'crosschain_silver', + 'number_sequence' + ) }} + WHERE + _id BETWEEN ( + SELECT + MIN(block_number) + FROM + blocks + ) + AND ( + SELECT + MAX(block_number) + FROM + blocks + ) +) +SELECT + 'blocks' AS test_name, + MIN( + b.block_number + ) AS min_block, + MAX( + b.block_number + ) AS max_block, + MIN( + b.block_timestamp + ) AS min_block_timestamp, + MAX( + b.block_timestamp + ) AS max_block_timestamp, + COUNT(1) AS blocks_tested, + COUNT( + CASE + WHEN C.block_number IS NOT NULL THEN A.block_number + END + ) AS blocks_impacted_count, + ARRAY_AGG( + CASE + WHEN C.block_number IS NOT NULL THEN A.block_number + END + ) within GROUP ( + ORDER BY + A.block_number + ) AS blocks_impacted_array, + SYSDATE() AS test_timestamp +FROM + block_gen A + LEFT JOIN blocks b + ON A.block_number = b.block_number + LEFT JOIN blocks C + ON A.block_number > C.prev_block_number + AND A.block_number < C.block_number + AND C.block_number - C.prev_block_number <> 1 +WHERE + COALESCE( + b.block_number, + C.block_number + ) IS NOT NULL diff --git a/models/silver/_observability/silver_observability__blocks_completeness.yml b/models/silver/_observability/silver_observability__blocks_completeness.yml new file mode 100644 index 0000000..082deef --- /dev/null +++ b/models/silver/_observability/silver_observability__blocks_completeness.yml @@ -0,0 +1,67 @@ +version: 2 +models: + - name: silver_observability__blocks_completeness + description: Records of all blocks block gaps (missing blocks) with a timestamp the test was run + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TEST_TIMESTAMP + columns: + - name: MIN_BLOCK + description: The lowest block id in the test + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: MAX_BLOCK + description: The highest block id in the test + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: MIN_BLOCK_TIMESTAMP + description: The lowest block timestamp in the test + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: MAX_BLOCK_TIMESTAMP + description: The highest block timestamp in the test + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: BLOCKS_TESTED + description: Count of blocks in the test + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCKS_IMPACTED_COUNT + description: Count of block gaps in the test + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCKS_IMPACTED_ARRAY + description: Array of affected blocks + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - ARRAY + - name: TEST_TIMESTAMP + description: When the test was run + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ diff --git a/models/silver/_observability/silver_observability__transactions_completeness.sql b/models/silver/_observability/silver_observability__transactions_completeness.sql new file mode 100644 index 0000000..52c5b52 --- /dev/null +++ b/models/silver/_observability/silver_observability__transactions_completeness.sql @@ -0,0 +1,137 @@ +{{ config( + materialized = 'incremental', + unique_key = 'test_timestamp', + full_refresh = false, + tags = ['observability','recent_test'] +) }} + +WITH summary_stats AS ( + + SELECT + MIN(block_number) AS min_block, + MAX(block_number) AS max_block, + MIN(block_timestamp) AS min_block_timestamp, + MAX(block_timestamp) AS max_block_timestamp, + COUNT(1) AS blocks_tested + FROM + {{ ref('core__fact_blocks') }} + WHERE + block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + +{% if is_incremental() %} +AND ( + block_number >= ( + SELECT + MIN(block_number) + FROM + ( + SELECT + MIN(block_number) AS block_number + FROM + {{ ref('core__fact_blocks') }} + WHERE + block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) + AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + UNION + SELECT + MIN(VALUE) - 1 AS block_number + FROM + ( + SELECT + blocks_impacted_array + FROM + {{ this }} + qualify ROW_NUMBER() over ( + ORDER BY + test_timestamp DESC + ) = 1 + ), + LATERAL FLATTEN( + input => blocks_impacted_array + ) + ) + ) {% if var('OBSERV_FULL_TEST') %} + OR block_number >= 0 + {% endif %} +) +{% endif %} +), +base_blocks AS ( + SELECT + block_number, + tx_count AS transaction_count + FROM + {{ ref('core__fact_blocks') }} + WHERE + block_number BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) +), +actual_tx_counts AS ( + SELECT + block_number, + COUNT(1) AS transaction_count + FROM + {{ ref('core__fact_transactions') }} + WHERE + block_number BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) + GROUP BY + block_number +), +potential_missing_txs AS ( + SELECT + e.block_number + FROM + base_blocks e + LEFT OUTER JOIN actual_tx_counts A + ON e.block_number = A.block_number + WHERE + COALESCE( + A.transaction_count, + 0 + ) <> e.transaction_count +), +impacted_blocks AS ( + SELECT + COUNT(1) AS blocks_impacted_count, + ARRAY_AGG(block_number) within GROUP ( + ORDER BY + block_number + ) AS blocks_impacted_array + FROM + potential_missing_txs +) +SELECT + 'transactions' AS test_name, + min_block, + max_block, + min_block_timestamp, + max_block_timestamp, + blocks_tested, + blocks_impacted_count, + blocks_impacted_array, + SYSDATE() AS test_timestamp +FROM + summary_stats + JOIN impacted_blocks + ON 1 = 1 diff --git a/models/silver/_observability/silver_observability__transactions_completeness.yml b/models/silver/_observability/silver_observability__transactions_completeness.yml new file mode 100644 index 0000000..e04bdbe --- /dev/null +++ b/models/silver/_observability/silver_observability__transactions_completeness.yml @@ -0,0 +1,69 @@ +version: 2 +models: + - name: silver_observability__transactions_completeness + description: Records of all blocks with missing transactions with a timestamp the test was run + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TEST_TIMESTAMP + columns: + - name: TEST_NAME + description: Name for the test + - name: MIN_BLOCK + description: The lowest block id in the test + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: MAX_BLOCK + description: The highest block id in the test + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: MIN_BLOCK_TIMESTAMP + description: The lowest block timestamp in the test + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: MAX_BLOCK_TIMESTAMP + description: The highest block timestamp in the test + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: BLOCKS_TESTED + description: Count of blocks in the test + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCKS_IMPACTED_COUNT + description: Count of block gaps in the test + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCKS_IMPACTED_ARRAY + description: Array of affected blocks + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - ARRAY + - name: TEST_TIMESTAMP + description: When the test was run + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/silver/core/silver__blocks.sql b/models/silver/core/silver__blocks.sql new file mode 100644 index 0000000..efa9667 --- /dev/null +++ b/models/silver/core/silver__blocks.sql @@ -0,0 +1,54 @@ +{{ config( + materialized = 'incremental', + unique_key = "block_number", + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['modified_timestamp::DATE'], + tags = ['core','full_test'] +) }} +-- depends_on: {{ ref('bronze__blocks_tx') }} + +SELECT + VALUE, + DATA :block_height :: INT AS block_number, + DATA :block_hash :: STRING AS block_hash, + DATA :block_timestamp :: bigint AS block_timestamp_num, + TO_TIMESTAMP( + block_timestamp_num :: STRING + ) AS block_timestamp, + DATA :first_version :: bigint AS first_version, + DATA :last_version :: bigint AS last_version, + ARRAY_SIZE( + DATA :transactions + ) AS tx_count_from_transactions_array, + last_version - first_version + 1 AS tx_count_from_versions, + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS blocks_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + +{% if is_incremental() %} +{{ ref('bronze__blocks_tx') }} +WHERE + inserted_timestamp >= ( + SELECT + MAX( + DATEADD( + 'minute', + -5, + modified_timestamp + ) + ) + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze__blocks_tx_FR') }} +{% endif %} + +qualify(ROW_NUMBER() over(PARTITION BY block_number +ORDER BY + inserted_timestamp DESC)) = 1 diff --git a/models/silver/core/silver__blocks.yml b/models/silver/core/silver__blocks.yml new file mode 100644 index 0000000..9c2e951 --- /dev/null +++ b/models/silver/core/silver__blocks.yml @@ -0,0 +1,39 @@ +version: 2 +models: + - name: silver__blocks + + tests: + - dbt_constraints.primary_key: + column_name: BLOCKS_ID + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP_NUM + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - name: BLOCK_HASH + tests: + - not_null + - name: FIRST_VERSION + tests: + - not_null + - name: LAST_VERSION + tests: + - not_null + - name: TX_COUNT_FROM_TRANSACTIONS_ARRAY + tests: + - not_null + - name: TX_COUNT_FROM_VERSIONS + tests: + - not_null + - name: BLOCKS_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + - name: MODIFIED_TIMESTAMP + - name: _INVOCATION_ID diff --git a/models/silver/core/silver__changes.sql b/models/silver/core/silver__changes.sql new file mode 100644 index 0000000..6841f0d --- /dev/null +++ b/models/silver/core/silver__changes.sql @@ -0,0 +1,43 @@ +{{ config( + materialized = 'view', + tags = ['core'] +) }} + +SELECT + A.block_number, + A.block_timestamp, + A.tx_hash, + version, + success, + A.tx_type, + A.payload_function, + b.index AS change_index, + b.value :data :data AS change_data, + b.value :type :: STRING AS change_type, + b.value :address :: STRING AS address, + b.value :handle :: STRING AS handle, + b.value :data: "type" :: STRING AS inner_change_type, + SPLIT_PART( + inner_change_type, + '::', + 1 + ) :: STRING AS change_address, + SPLIT_PART( + inner_change_type, + '::', + 2 + ) :: STRING AS change_module, + SUBSTRING(inner_change_type, len(change_address) + len(change_module) + 5) AS change_resource, + b.value :key :: STRING AS key, + b.value :value :: STRING AS VALUE, + b.value :state_key_hash :: STRING AS state_key_hash, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash','change_index'] + ) }} AS changes_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref( + 'core__fact_transactions' + ) }} A, + LATERAL FLATTEN (changes) b diff --git a/models/silver/core/silver__changes.yml b/models/silver/core/silver__changes.yml new file mode 100644 index 0000000..10ab9d3 --- /dev/null +++ b/models/silver/core/silver__changes.yml @@ -0,0 +1,26 @@ +version: 2 +models: + - name: silver__changes + + columns: + - name: BLOCK_NUMBER + - name: BLOCK_TIMESTAMP + - name: TX_HASH + - name: VERSION + - name: SUCCESS + - name: TX_TYPE + - name: CHANGE_INDEX + - name: CHANGE_DATA + - name: CHANGE_TYPE + - name: ADDRESS + - name: HANDLE + - name: INNER_CHANGE_TYPE + - name: CHANGE_ADDRESS + - name: CHANGE_MODULE + - name: CHANGE_RESOURCE + - name: KEY + - name: VALUE + - name: STATE_KEY_HASH + - name: CHANGES_ID + - name: INSERTED_TIMESTAMP + - name: MODIFIED_TIMESTAMP \ No newline at end of file diff --git a/models/silver/core/silver__events.sql b/models/silver/core/silver__events.sql new file mode 100644 index 0000000..69d9a78 --- /dev/null +++ b/models/silver/core/silver__events.sql @@ -0,0 +1,55 @@ +{{ config( + materialized = 'view', + tags = ['core'] +) }} + +SELECT + A.block_number, + A.block_timestamp, + A.tx_hash, + version, + success, + A.tx_type, + A.payload_function, + b.index AS event_index, + b.value :type :: STRING AS event_type, + SPLIT_PART( + event_type, + '::', + 1 + ) :: STRING AS event_address, + SPLIT_PART( + event_type, + '::', + 2 + ) :: STRING AS event_module, + SPLIT_PART( + event_type, + '::', + 3 + ) :: STRING AS event_resource, + b.value :data AS event_data, + -- b.value :guid :: STRING AS event_guid, -- extract into account_address + creation_number + b.value :guid :account_address :: STRING AS account_address, + b.value :guid :creation_number :: bigint AS creation_number, + b.value :sequence_number :: bigint AS sequence_number, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash','event_index'] + ) }} AS events_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref( + 'core__fact_transactions' + ) }} A, + LATERAL FLATTEN (events) b + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/core/silver__events.yml b/models/silver/core/silver__events.yml new file mode 100644 index 0000000..91db325 --- /dev/null +++ b/models/silver/core/silver__events.yml @@ -0,0 +1,23 @@ +version: 2 +models: + - name: silver__events + + columns: + - name: BLOCK_NUMBER + - name: BLOCK_TIMESTAMP + - name: TX_HASH + - name: VERSION + - name: SUCCESS + - name: TX_TYPE + - name: EVENT_INDEX + - name: EVENT_TYPE + - name: EVENT_ADDRESS + - name: EVENT_MODULE + - name: EVENT_RESOURCE + - name: EVENT_DATA + - name: ACCOUNT_ADDRESS + - name: CREATION_NUMBER + - name: SEQUENCE_NUMBER + - name: EVENTS_ID + - name: INSERTED_TIMESTAMP + - name: MODIFIED_TIMESTAMP diff --git a/models/silver/core/silver__transactions.sql b/models/silver/core/silver__transactions.sql new file mode 100644 index 0000000..2ddd43f --- /dev/null +++ b/models/silver/core/silver__transactions.sql @@ -0,0 +1,112 @@ +{{ config( + materialized = 'incremental', + unique_key = ['tx_hash','block_timestamp::DATE'], + incremental_strategy = 'merge', + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['modified_timestamp::DATE','tx_type'], + tags = ['core','full_test'] +) }} +-- depends_on: {{ ref('bronze__blocks_tx') }} +-- depends_on: {{ ref('bronze__transactions') }} +WITH from_blocks AS ( + + SELECT + TO_TIMESTAMP( + b.value :timestamp :: STRING + ) AS block_timestamp, + b.value :hash :: STRING AS tx_hash, + b.value :version :: INT AS version, + b.value :type :: STRING AS tx_type, + b.value AS DATA, + inserted_timestamp AS file_last_updated + FROM + +{% if is_incremental() %} +{{ ref('bronze__blocks_tx') }} +{% else %} + {{ ref('bronze__blocks_tx_FR') }} +{% endif %} + +A, +LATERAL FLATTEN (DATA :transactions) b + +{% if is_incremental() %} +WHERE + A.inserted_timestamp >= ( + SELECT + DATEADD('minute', -15, MAX(modified_timestamp)) + FROM + {{ this }}) + {% endif %} + ), + from_transactions AS ( + SELECT + TO_TIMESTAMP( + b.value :timestamp :: STRING + ) AS block_timestamp, + b.value :hash :: STRING AS tx_hash, + b.value :version :: INT AS version, + b.value :type :: STRING AS tx_type, + b.value AS DATA, + inserted_timestamp AS file_last_updated + FROM + +{% if is_incremental() %} +{{ ref('bronze__transactions') }} +{% else %} + {{ ref('bronze__transactions_FR') }} +{% endif %} + +A, +LATERAL FLATTEN(A.data) b + +{% if is_incremental() %} +WHERE + A.inserted_timestamp >= ( + SELECT + DATEADD('minute', -15, MAX(modified_timestamp)) + FROM + {{ this }}) + {% endif %} + ), + combo AS ( + SELECT + block_timestamp, + tx_hash, + version, + tx_type, + DATA, + file_last_updated + FROM + from_blocks + UNION ALL + SELECT + block_timestamp, + tx_hash, + version, + tx_type, + DATA, + file_last_updated + FROM + from_transactions A + ) +SELECT + COALESCE( + block_timestamp, + '1970-01-01 00:00:00.000' + ) AS block_timestamp, + tx_hash, + version, + tx_type, + DATA, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS transactions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + combo qualify(ROW_NUMBER() over (PARTITION BY tx_hash +ORDER BY + file_last_updated DESC)) = 1 diff --git a/models/silver/core/silver__transactions.yml b/models/silver/core/silver__transactions.yml new file mode 100644 index 0000000..48c448d --- /dev/null +++ b/models/silver/core/silver__transactions.yml @@ -0,0 +1,37 @@ +version: 2 +models: + - name: silver__transactions + + tests: + - dbt_constraints.primary_key: + column_name: TRANSACTIONS_ID + + + columns: + - name: BLOCK_TIMESTAMP + tests: + - not_null + - name: TX_HASH + tests: + - not_null + - name: VERSION + tests: + - not_null + - name: TX_TYPE + tests: + - not_null + - name: TRANSACTIONS_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + + - name: _INVOCATION_ID + tests: + - name: not_null_silver__transactions_INVOCATION_ID + test_name: not_null + \ No newline at end of file diff --git a/models/silver/core/tests/blocks/test_silver__blocks_full.sql b/models/silver/core/tests/blocks/test_silver__blocks_full.sql new file mode 100644 index 0000000..89acf46 --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_full.sql @@ -0,0 +1,11 @@ +{{ config( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref( + 'silver__blocks' + ) }} diff --git a/models/silver/core/tests/blocks/test_silver__blocks_full.yml b/models/silver/core/tests/blocks/test_silver__blocks_full.yml new file mode 100644 index 0000000..f0c255f --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_full.yml @@ -0,0 +1,84 @@ +version: 2 +models: + - name: test_silver__blocks_full + + tests: + - fsc_utils.sequence_gaps: + column_name: BLOCK_NUMBER + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - unique + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP_NUM + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: FIRST_VERSION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: LAST_VERSION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_COUNT_FROM_TRANSACTIONS_ARRAY + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_COUNT_FROM_VERSIONS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCKS_ID + tests: + - not_null + - unique + - name: INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: _INVOCATION_ID + tests: + - name: not_null_test_silver__blocks_INVOCATION_ID + test_name: not_null diff --git a/models/silver/core/tests/blocks/test_silver__blocks_recent.sql b/models/silver/core/tests/blocks/test_silver__blocks_recent.sql new file mode 100644 index 0000000..c27a8a9 --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__blocks') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/blocks/test_silver__blocks_recent.yml b/models/silver/core/tests/blocks/test_silver__blocks_recent.yml new file mode 100644 index 0000000..6dcf11f --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_recent.yml @@ -0,0 +1,21 @@ +version: 2 +models: + - name: test_silver__blocks_recent + + tests: + - fsc_utils.sequence_gaps: + column_name: BLOCK_NUMBER + config: + severity: error + error_if: ">100" + columns: + - name: BLOCK_NUMBER + tests: + - unique + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 \ No newline at end of file diff --git a/models/silver/core/tests/transactions/test_silver__transactions_full.sql b/models/silver/core/tests/transactions/test_silver__transactions_full.sql new file mode 100644 index 0000000..dcbd02c --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_full.sql @@ -0,0 +1,11 @@ +{{ config( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref( + 'silver__transactions' + ) }} diff --git a/models/silver/core/tests/transactions/test_silver__transactions_full.yml b/models/silver/core/tests/transactions/test_silver__transactions_full.yml new file mode 100644 index 0000000..5d20f52 --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_full.yml @@ -0,0 +1,58 @@ +version: 2 +models: + - name: test_silver__transactions_full + + tests: + - fsc_utils.sequence_gaps: + column_name: VERSION + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + + columns: + + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - unique + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: VERSION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_TYPE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: TRANSACTIONS_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: _INVOCATION_ID + tests: + - name: not_null_test_silver__transactions_INVOCATION_ID + test_name: not_null \ No newline at end of file diff --git a/models/silver/core/tests/transactions/test_silver__transactions_recent.sql b/models/silver/core/tests/transactions/test_silver__transactions_recent.sql new file mode 100644 index 0000000..acac88f --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_date + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_date DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__transactions') }} +WHERE + block_timestamp :: DATE >= ( + SELECT + block_date + FROM + last_3_days + ) diff --git a/models/silver/core/tests/transactions/test_silver__transactions_recent.yml b/models/silver/core/tests/transactions/test_silver__transactions_recent.yml new file mode 100644 index 0000000..9544c3e --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_recent.yml @@ -0,0 +1,26 @@ +version: 2 +models: + - name: test_silver__transactions_recent + + tests: + - fsc_utils.sequence_gaps: + column_name: VERSION + config: + severity: error + error_if: ">100" + + columns: + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + - name: TX_HASH + tests: + - not_null + - unique + - name: VERSION + tests: + - not_null + \ No newline at end of file diff --git a/models/silver/price/silver__complete_native_prices.sql b/models/silver/price/silver__complete_native_prices.sql new file mode 100644 index 0000000..bb798ba --- /dev/null +++ b/models/silver/price/silver__complete_native_prices.sql @@ -0,0 +1,40 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_native_prices_id', + tags = ['core'] +) }} + +SELECT + HOUR, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_native_prices_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_native_prices' + ) }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/price/silver__complete_native_prices.yml b/models/silver/price/silver__complete_native_prices.yml new file mode 100644 index 0000000..bb72433 --- /dev/null +++ b/models/silver/price/silver__complete_native_prices.yml @@ -0,0 +1,37 @@ +version: 2 +models: + - name: silver__complete_native_prices + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - HOUR + - SYMBOL + + columns: + - name: HOUR + tests: + - not_null + - name: SYMBOL + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: PROVIDER + tests: + - not_null + - name: PRICE + tests: + - not_null + - name: IS_IMPUTED + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_NATIVE_PRICES_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/stats/silver__core_metrics_hourly.sql b/models/silver/stats/silver__core_metrics_hourly.sql new file mode 100644 index 0000000..db3e7d0 --- /dev/null +++ b/models/silver/stats/silver__core_metrics_hourly.sql @@ -0,0 +1,83 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_timestamp_hour", + cluster_by = ['block_timestamp_hour::DATE'], + tags = ['noncore'] +) }} +/* run incremental timestamp value first then use it as a static value */ +{% if execute %} + +{% if is_incremental() %} +{% set query %} + +SELECT + MIN(DATE_TRUNC('hour', block_timestamp)) block_timestamp_hour +FROM + {{ ref('core__fact_transactions') }} +WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) {% endset %} + {% set min_block_timestamp_hour = run_query(query).columns [0].values() [0] %} +{% endif %} +{% endif %} +SELECT + DATE_TRUNC( + 'hour', + block_timestamp + ) AS block_timestamp_hour, + MIN(block_number) :: FLOAT AS block_number_min, + MAX(block_number) :: FLOAT AS block_number_max, + COUNT( + DISTINCT block_number + ) AS block_count, + COUNT( + DISTINCT tx_hash + ) AS transaction_count, + COUNT( + DISTINCT CASE + WHEN success THEN tx_hash + END + ) AS transaction_count_success, + COUNT( + DISTINCT CASE + WHEN NOT success THEN tx_hash + END + ) AS transaction_count_failed, + COUNT( + DISTINCT sender + ) AS unique_sender_count, + COUNT( + DISTINCT payload_function + ) AS unique_payload_function_count, + SUM(gas_used) AS total_fees, + -- in Octa = 10^-8 Aptos + {{ dbt_utils.generate_surrogate_key( + ['block_timestamp_hour'] + ) }} AS core_metrics_hourly_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('core__fact_transactions') }} +WHERE + block_timestamp_hour < DATE_TRUNC( + 'hour', + CURRENT_TIMESTAMP + ) + +{% if is_incremental() %} +AND DATE_TRUNC( + 'hour', + block_timestamp +) >= NULLIF( + '{{ min_block_timestamp_hour }}', + 'None' +) +{% endif %} +GROUP BY + 1 diff --git a/models/silver/stats/silver__core_metrics_hourly.yml b/models/silver/stats/silver__core_metrics_hourly.yml new file mode 100644 index 0000000..b381d08 --- /dev/null +++ b/models/silver/stats/silver__core_metrics_hourly.yml @@ -0,0 +1,70 @@ +version: 2 +models: + - name: silver__core_metrics_hourly + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_TIMESTAMP_HOUR + columns: + - name: BLOCK_TIMESTAMP_HOUR + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: BLOCK_NUMBER_MIN + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_NUMBER_MAX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_COUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRANSACTION_COUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRANSACTION_COUNT_SUCCESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRANSACTION_COUNT_FAILED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TOTAL_FEES + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - DECIMAL + - FLOAT + - NUMBER + - name: _INSERTED_TIMESTAMP + tests: + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml new file mode 100644 index 0000000..d76bb95 --- /dev/null +++ b/models/sources.yml @@ -0,0 +1,45 @@ +version: 2 + +sources: + - name: crosschain + database: "{{ 'crosschain' if target.database == 'MOVEMENT' else 'crosschain_dev' }}" + schema: core + tables: + - name: dim_date_hours + - name: address_tags + - name: dim_dates + - name: crosschain_silver + database: "{{ 'crosschain' if target.database == 'MOVEMENT' else 'crosschain_dev' }}" + schema: silver + tables: + - name: asset_metadata_coin_market_cap + - name: asset_metadata_coin_gecko + - name: hourly_prices_coin_market_cap + - name: hourly_prices_coin_gecko + - name: number_sequence + - name: labels_combined + - name: complete_native_asset_metadata + - name: complete_native_prices + - name: bronze_streamline + database: streamline + schema: | + {{ "MOVEMENT_DEV" if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else "MOVEMENT" }} + tables: + - name: blocks_tx + - name: transactions + - name: evm_blocks + - name: evm_transactions + - name: evm_confirm_blocks + - name: evm_receipts + - name: evm_traces + - name: evm_decoded_logs + - name: github_actions + database: MOVEMENT + schema: github_actions + tables: + - name: workflows + - name: crosschain_public + database: crosschain + schema: bronze_public + tables: + - name: user_abis \ No newline at end of file diff --git a/models/streamline/complete/streamline__blocks_tx_complete.sql b/models/streamline/complete/streamline__blocks_tx_complete.sql new file mode 100644 index 0000000..d602314 --- /dev/null +++ b/models/streamline/complete/streamline__blocks_tx_complete.sql @@ -0,0 +1,41 @@ +{{ config ( + materialized = "incremental", + incremental_strategy = 'merge', + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + merge_exclude_columns = ["inserted_timestamp"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)" +) }} +-- depends_on: {{ ref('bronze__blocks_tx') }} + +SELECT + DATA :block_height :: INT AS block_number, + ARRAY_SIZE( + DATA :transactions + ) AS tx_count_from_transactions_array, + DATA :last_version :: bigint - DATA :first_version :: bigint + 1 AS tx_count_from_versions, + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS blocks_tx_complete_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + file_name, + '{{ invocation_id }}' AS _invocation_id +FROM + +{% if is_incremental() %} +{{ ref('bronze__blocks_tx') }} +WHERE + inserted_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze__blocks_tx_FR') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number +ORDER BY + inserted_timestamp DESC)) = 1 diff --git a/models/streamline/complete/streamline__transactions_complete.sql b/models/streamline/complete/streamline__transactions_complete.sql new file mode 100644 index 0000000..7e5a5ca --- /dev/null +++ b/models/streamline/complete/streamline__transactions_complete.sql @@ -0,0 +1,46 @@ +{{ config ( + materialized = "incremental", + incremental_strategy = 'merge', + unique_key = "transactions_complete_id", + cluster_by = "ROUND(block_number, -3)", + merge_exclude_columns = ["inserted_timestamp"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)" +) }} +-- depends_on: {{ ref('bronze__transactions') }} + +SELECT + A.value :BLOCK_NUMBER :: INT AS block_number, + A.value :MULTIPLIER :: INT AS multiplier_no, + {{ dbt_utils.generate_surrogate_key( + ['block_number','multiplier_no'] + ) }} AS transactions_complete_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + file_name, + '{{ invocation_id }}' AS _invocation_id +FROM + +{% if is_incremental() %} +{{ ref('bronze__transactions') }} +{% else %} + {{ ref('bronze__transactions_FR') }} +{% endif %} + +A +JOIN {{ ref('silver__blocks') }} +b +ON DATA [0] :version :: INT BETWEEN b.first_version +AND b.last_version + +{% if is_incremental() %} +WHERE + A.inserted_timestamp >= ( + SELECT + COALESCE(MAX(modified_timestamp), '1970-01-01' :: DATE) + FROM + {{ this }}) + {% endif %} + + qualify(ROW_NUMBER() over (PARTITION BY block_number + ORDER BY + A.inserted_timestamp DESC)) = 1 diff --git a/models/streamline/silver/_max_block_by_date.sql b/models/streamline/silver/_max_block_by_date.sql new file mode 100644 index 0000000..7609b67 --- /dev/null +++ b/models/streamline/silver/_max_block_by_date.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = "ephemeral", + unique_key = "block_number", +) }} + +WITH base AS ( + + SELECT + block_timestamp :: DATE AS block_date, + MAX(block_number) block_number + FROM + {{ ref("silver__blocks") }} + GROUP BY + block_timestamp :: DATE +) +SELECT + block_date, + block_number +FROM + base +WHERE + block_date <> ( + SELECT + MAX(block_date) + FROM + base + ) diff --git a/models/streamline/silver/realtime/streamline__blocks_tx_realtime.sql b/models/streamline/silver/realtime/streamline__blocks_tx_realtime.sql new file mode 100644 index 0000000..7ae77fc --- /dev/null +++ b/models/streamline/silver/realtime/streamline__blocks_tx_realtime.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"blocks_tx", + "sql_limit" :"8000", + "producer_batch_size" :"50", + "worker_batch_size" :"50", + "sql_source" :"{{this.identifier}}" } + ) +) }} + +WITH blocks AS ( + + SELECT + block_number + FROM + {{ ref('streamline__blocks') }} + EXCEPT + SELECT + block_number + FROM + {{ ref('streamline__blocks_tx_complete') }} +) +SELECT + block_number, + ROUND( + block_number, + -4 + ) :: INT AS partition_key, + {{ target.database }}.live.udf_api( + 'GET', + '{Service}/v1/blocks/by_height/' || block_number || '?with_transactions=true', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + PARSE_JSON('{}'), + 'Vault/prod/movement/devnet' + ) AS request +FROM + blocks +ORDER BY + block_number diff --git a/models/streamline/silver/realtime/streamline__transactions_realtime.sql b/models/streamline/silver/realtime/streamline__transactions_realtime.sql new file mode 100644 index 0000000..f918e02 --- /dev/null +++ b/models/streamline/silver/realtime/streamline__transactions_realtime.sql @@ -0,0 +1,94 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"transactions", + "sql_limit" :"1000", + "producer_batch_size" :"50", + "worker_batch_size" :"50", + "sql_source" :"{{this.identifier}}" } + ) +) }} + +WITH blocks AS ( + + SELECT + A.block_number, + tx_count_from_versions -100 AS tx_count, + first_version + 100 version_start + FROM + {{ ref('silver__blocks') }} A + WHERE + tx_count_from_versions > 100 +), +numbers AS ( + -- Recursive CTE to generate numbers. We'll use the maximum txcount value to limit our recursion. + SELECT + 1 AS n + UNION ALL + SELECT + n + 1 + FROM + numbers + WHERE + n < ( + SELECT + CEIL(MAX(tx_count) / 100.0) + FROM + blocks) + ), + blocks_with_page_numbers AS ( + SELECT + tt.block_number :: INT AS block_number, + n.n - 1 AS multiplier, + version_start, + tx_count + FROM + blocks tt + JOIN numbers n + ON n.n <= CASE + WHEN tt.tx_count % 100 = 0 THEN tt.tx_count / 100 + ELSE FLOOR( + tt.tx_count / 100 + ) + 1 + END + ), + WORK AS ( + SELECT + A.block_number, + version_start +( + 100 * multiplier + ) AS tx_version, + multiplier + FROM + blocks_with_page_numbers A + LEFT JOIN {{ ref('streamline__transactions_complete') }} + b + ON A.block_number = b.block_number + AND multiplier = b.multiplier_no + WHERE + b.block_number IS NULL + ) + SELECT + tx_version, + ROUND( + tx_version, + -4 + ) :: INT AS partition_key, + {{ target.database }}.live.udf_api( + 'GET', + '{Service}/v1/transactions?start=' || tx_version || '&limit=100', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + PARSE_JSON('{}'), + 'Vault/prod/movement/devnet' + ) AS request, + block_number, + multiplier + FROM + WORK + ORDER BY + block_number diff --git a/models/streamline/silver/streamline__blocks.sql b/models/streamline/silver/streamline__blocks.sql new file mode 100644 index 0000000..009fa64 --- /dev/null +++ b/models/streamline/silver/streamline__blocks.sql @@ -0,0 +1,22 @@ +{{ config ( + materialized = "view", + tags = ['streamline_view'] +) }} + +SELECT + _id AS block_number +FROM + {{ source( + 'crosschain_silver', + 'number_sequence' + ) }} +WHERE + _id <= ( + SELECT + MAX(block_number) + FROM + {{ ref('streamline__chainhead') }} + ) +UNION ALL +SELECT + 0 AS block_number diff --git a/models/streamline/silver/streamline__chainhead.sql b/models/streamline/silver/streamline__chainhead.sql new file mode 100644 index 0000000..998a11a --- /dev/null +++ b/models/streamline/silver/streamline__chainhead.sql @@ -0,0 +1,20 @@ +{{ config ( + materialized = "view", + tags = ['streamline_view'] +) }} + +SELECT + {{ target.database }}.live.udf_api( + 'GET', + '{Service}/v1', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json', + 'fsc-quantum-state', + 'livequery', + 'User-Agent', + 'Flipside_Crypto/0.1' + ), + OBJECT_CONSTRUCT(), + 'Vault/prod/movement/devnet' + ) :data :block_height :: INT AS block_number diff --git a/package-lock.yml b/package-lock.yml new file mode 100644 index 0000000..fc976d2 --- /dev/null +++ b/package-lock.yml @@ -0,0 +1,16 @@ +packages: +- package: calogica/dbt_expectations + version: 0.8.5 +- package: dbt-labs/dbt_utils + version: 1.0.0 +- git: https://github.com/FlipsideCrypto/fsc-utils.git + revision: eb33ac727af26ebc8a8cc9711d4a6ebc3790a107 +- package: get-select/dbt_snowflake_query_tags + version: 2.5.0 +- package: Snowflake-Labs/dbt_constraints + version: 0.6.3 +- package: calogica/dbt_date + version: 0.7.2 +- git: https://github.com/FlipsideCrypto/livequery-models.git + revision: b024188be4e9c6bc00ed77797ebdc92d351d620e +sha1_hash: d3219b9c206b5988189dcdafae0ec22ca9b4056c diff --git a/packages.yml b/packages.yml new file mode 100644 index 0000000..047bf86 --- /dev/null +++ b/packages.yml @@ -0,0 +1,11 @@ +packages: + - package: calogica/dbt_expectations + version: [">=0.4.0", "<0.9.0"] + - package: dbt-labs/dbt_utils + version: 1.0.0 + - git: https://github.com/FlipsideCrypto/fsc-utils.git + revision: v1.30.0 + - package: get-select/dbt_snowflake_query_tags + version: [">=2.0.0", "<3.0.0"] + - package: Snowflake-Labs/dbt_constraints + version: [">=0.4.0"] \ No newline at end of file diff --git a/profiles.yml b/profiles.yml new file mode 100644 index 0000000..588ba2a --- /dev/null +++ b/profiles.yml @@ -0,0 +1,15 @@ +movement: + target: dev + outputs: + dev: + type: snowflake + account: vna27887.us-east-1 + role: INTERNAL_DEV + user: mike.stepanovic@flipsidecrypto.com + authenticator: externalbrowser + region: us-east-1 + database: MOVEMENT_DEV + warehouse: DBT + schema: SILVER + threads: 4 + client_session_keep_alive: False \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..39b82bb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +dbt-snowflake>=1.7,<1.8 +protobuf==4.25.3 \ No newline at end of file diff --git a/snapshots/.gitkeep b/snapshots/.gitkeep new file mode 100644 index 0000000..e69de29